From 9a046c2f035af130878aabc2c5d35bf2d2ccca02 Mon Sep 17 00:00:00 2001 From: eleksir Date: Wed, 25 Mar 2026 21:46:15 +0300 Subject: [PATCH] Initial commit --- .gitignore | 29 ++ .golangci.yml | 331 ++++++++++++++++ LICENSE | 21 + Makefile | 53 +++ README.md | 79 ++++ TODO.org | 14 + cmd/backer/main.go | 76 ++++ contrib/backer | 18 + contrib/backer.freebsd | 30 ++ contrib/backer.service | 14 + data/config_example.json | 37 ++ go.mod | 8 + go.sum | 4 + internal/backer/getFilesFromDirectories.go | 53 +++ .../backer/getFilesFromDirectories_test.go | 269 +++++++++++++ internal/backer/globals.go | 14 + internal/backer/loadConfig.go | 148 +++++++ internal/backer/loadConfig_test.go | 223 +++++++++++ internal/backer/newServer.go | 194 +++++++++ internal/backer/newServer_test.go | 374 ++++++++++++++++++ internal/backer/types.go | 23 ++ internal/backer/utils.go | 177 +++++++++ internal/backer/utils_test.go | 359 +++++++++++++++++ internal/log/main.go | 148 +++++++ llms.txt | 207 ++++++++++ test_data/example.crt | 24 ++ test_data/example.key | 28 ++ test_data/test1/bar/goodbye.txt/text.txt | 1 + test_data/test1/bar/hello.txt | 0 test_data/test1/foo/hello_breakfast.txt | 2 + test_data/test1/foo/mydir/myfile.txt | 1 + test_data/test1/foo/some_text.txt | 0 test_data/test_config.json | 43 ++ 33 files changed, 3002 insertions(+) create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 TODO.org create mode 100644 cmd/backer/main.go create mode 100644 contrib/backer create mode 100644 contrib/backer.freebsd create mode 100644 contrib/backer.service create mode 100644 data/config_example.json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/backer/getFilesFromDirectories.go create mode 100644 internal/backer/getFilesFromDirectories_test.go create mode 100644 internal/backer/globals.go create mode 100644 internal/backer/loadConfig.go create mode 100644 internal/backer/loadConfig_test.go create mode 100644 internal/backer/newServer.go create mode 100644 internal/backer/newServer_test.go create mode 100644 internal/backer/types.go create mode 100644 internal/backer/utils.go create mode 100644 internal/backer/utils_test.go create mode 100644 internal/log/main.go create mode 100644 llms.txt create mode 100644 test_data/example.crt create mode 100644 test_data/example.key create mode 100644 test_data/test1/bar/goodbye.txt/text.txt create mode 100644 test_data/test1/bar/hello.txt create mode 100644 test_data/test1/foo/hello_breakfast.txt create mode 100644 test_data/test1/foo/mydir/myfile.txt create mode 100644 test_data/test1/foo/some_text.txt create mode 100644 test_data/test_config.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb29651 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Binary +/backer +/backer.exe + +# Local config (may contain secrets) +/config.json + +# Test coverage +/coverage.out +/coverage.html + +# Accidental log files in source +/internal/backer/info + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Build artifacts +*.exe +*.test +*.out diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..4991f88 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,331 @@ +version: "2" +run: + go: "1.25" + modules-download-mode: readonly + tests: false + allow-parallel-runners: true +linters: + default: none + enable: + - asasalint + - asciicheck + - bodyclose + - containedctx + - contextcheck + - decorder + - durationcheck + - errcheck + - errchkjson + - errorlint + - exhaustive + - exptostd + - ginkgolinter + - gocheckcompilerdirectives + - gocritic + - godoclint + - godot + - godox + - goprintffuncname + - gosec + - gosmopolitan + - govet + - grouper + - importas + - ineffassign + - intrange + - ireturn + - makezero + - mirror + - misspell + - modernize + - nakedret + - nilerr + - nilnil + - nlreturn + - noctx + - nolintlint + - nonamedreturns + - perfsprint + - predeclared + - promlinter + - reassign + - revive + - rowserrcheck + - staticcheck + - unconvert + - unused + - usestdlibvars + - wastedassign + - wsl_v5 + settings: + decorder: + dec-order: + - type + - const + - var + - func + gocritic: + disable-all: false + enable-all: true + disabled-checks: + - singleCaseSwitch + - octalLiteral + - regexpSimplify + - paramTypeCombine + - whyNoLint + - hugeParam + godoclint: + default: all + ignore-unexported: false + ignore-exported: false + include-unexported: true + disable: + - max-len + godot: + scope: all + exclude: + # Exclude todo and fixme comments. + - "^fixme:" + - "^todo:" + period: true + capital: false + godox: + keywords: + - FIXME + gosec: # go-fucking-sic + excludes: + # Боже, как же вы затрахали своей крипто-ебучкой, наркоманы хреновы! + - G405 # Detect the usage of DES or RC4 + - G406 # Detect the usage of MD4 or RIPEMD160 + # Оно не используется для повышения безопасности или каких-то таких секъюрных вещей, оно используется для выборки + # произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью. + - G404 # Insecure random number source (rand) + - G501 # Import blocklist: crypto/md5 + - G502 # Import blocklist: crypto/des + - G503 # Import blocklist: crypto/rc4 + - G504 # Import blocklist: net/http/cgi + - G505 # Import blocklist: crypto/sha1 + - G506 # Import blocklist: golang.org/x/crypto/md4 + - G507 # Import blocklist: golang.org/x/crypto/ripemd160 + config: + global: + audit: true + # Ваша сраная паранойя при создании файлов и каталогов мешает нормальной работе сервисов. Группы придуманы для + # удобства, группе удобно давать право чтения. Поэтому идите нахуй со своими ограничениями на чтение для + # группы, "безопасные" клоуны. + G301: "0755" + G306: "0644" + govet: + disable-all: false + enable-all: true + disable: + - shadow + - fieldalignment + grouper: + # Require the use of a single global 'const' declaration only. + # Default: false + const-require-single-const: true + # Require the use of grouped global 'const' declarations. + # Default: false + const-require-grouping: true + # Require the use of a single 'import' declaration only. + # Default: false + import-require-single-import: true + # Require the use of grouped 'import' declarations. + # Default: false + import-require-grouping: true + # Require the use of a single global 'type' declaration only. + # Default: false + type-require-single-type: true + # Require the use of grouped global 'type' declarations. + # Default: false + type-require-grouping: true + # Require the use of a single global 'var' declaration only. + # Default: false + var-require-single-var: true + # Require the use of grouped global 'var' declarations. + # Default: false + var-require-grouping: true + misspell: + # Correct spellings using locale preferences for US or UK. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + # Default is to use a neutral variety of English. + locale: US + # Typos to ignore. + # Should be in lower case. + # Default: [] + ignore-rules: + - someword + # Extra word corrections. + # `typo` and `correction` should only contain letters. + # The words are case-insensitive. + # Default: [] + extra-words: + - typo: "iff" + correction: "if" + - typo: "cancelation" + correction: "cancellation" + # Mode of the analysis: + # - default: checks all the file content. + # - restricted: checks only comments. + # Default: "" + mode: restricted + nonamedreturns: + # Report named error if it is assigned inside defer. + # Default: false + report-error-in-defer: true + perfsprint: + integer-format: true + int-conversion: true + error-format: true + err-error: false + errorf: true + string-format: true + sprintf1: true + strconcat: true + bool-format: true + hex-format: true + concat-loop: true + loop-other-ops: false + revive: + enable-all-rules: false + staticcheck: + # https://staticcheck.dev/docs/configuration/options/#dot_import_whitelist + # Default: ["github.com/mmcloughlin/avo/build", "github.com/mmcloughlin/avo/operand", "github.com/mmcloughlin/avo/reg"] + dot-import-whitelist: + - fmt + # https://staticcheck.dev/docs/configuration/options/#initialisms + # Default: ["ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS", "SIP", "RTP", "AMQP", "DB", "TS"] + initialisms: + [ + "ACL", + "API", + "ASCII", + "CPU", + "CSS", + "DNS", + "EOF", + "GUID", + "HTML", + "HTTP", + "HTTPS", + "ID", + "IP", + "JSON", + "QPS", + "RAM", + "RPC", + "SLA", + "SMTP", + "SQL", + "SSH", + "TCP", + "TLS", + "TTL", + "UDP", + "UI", + "GID", + "UID", + "UUID", + "URI", + "URL", + "UTF8", + "VM", + "XML", + "XMPP", + "XSRF", + "XSS", + "SIP", + "RTP", + "AMQP", + "DB", + "TS", + ] + # https://staticcheck.dev/docs/configuration/options/#http_status_code_whitelist + # Default: ["200", "400", "404", "500"] + http-status-code-whitelist: ["200", "400", "404", "500"] + # SAxxxx checks in https://staticcheck.dev/docs/configuration/options/#checks + # Example (to disable some checks): [ "all", "-SA1000", "-SA1001"] + # Run `GL_DEBUG=staticcheck golangci-lint run --enable=staticcheck` to see all available checks and enabled by config checks. + # Default: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022"] + checks: + # Enable all checks, then disable specific ones + - all + # Disable ST1005 - allows capitalized error messages + - -ST1005 + unconvert: + # Remove conversions that force intermediate rounding. + # Default: false + fast-math: true + # Be more conservative (experimental). + # Default: false + safe: true + usestdlibvars: + # Suggest the use of http.MethodXX. + # Default: true + http-method: false + # Suggest the use of http.StatusXX. + # Default: true + http-status-code: false + # Suggest the use of time.Month in time.Date. + # Default: false + time-date-month: true + # Suggest the use of time.Weekday.String(). + # Default: true + time-weekday: true + # Suggest the use of time.Month.String(). + # Default: false + time-month: true + # Suggest the use of time.Layout. + # Default: false + time-layout: true + # Suggest the use of crypto.Hash.String(). + # Default: false + crypto-hash: true + # Suggest the use of rpc.DefaultXXPath. + # Default: false + default-rpc-path: true + # Suggest the use of sql.LevelXX.String(). + # Default: false + sql-isolation-level: true + # Suggest the use of tls.SignatureScheme.String(). + # Default: false + tls-signature-scheme: true + # Suggest the use of constant.Kind.String(). + # Default: false + constant-kind: true + wsl_v5: + allow-first-in-block: true + allow-whole-block: true + branch-max-lines: 2 + case-max-lines: 2 + default: all + disable: + - if + - err + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + # Allow capitalized error messages + - linters: + - staticcheck + text: "ST1005" + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..feb16c6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Backer Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9dfea54 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +#!/usr/bin/env gmake -f + +.PHONY: all build clean test upgrade help + +BINARY=backer +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") +LDFLAGS=-ldflags="-s -w -X main.version=$(VERSION)" +BUILDOPTS=-a -gcflags=all=-l -trimpath -buildvcs=false + +# Binary name with .exe for Windows +ifdef GOOS + ifeq ($(GOOS),windows) + BINARY=backer.exe + endif +else ifeq ($(OS),Windows_NT) + BINARY=backer.exe +endif + +all: clean build + +build: + CGO_ENABLED=0 go build $(BUILDOPTS) $(LDFLAGS) -o $(BINARY) ./cmd/backer + +clean: + rm -f $(BINARY) + +upgrade: + go get -u ./... + go mod tidy + go mod vendor + +test: + go test ./... + +help: + @echo "Backer build system" + @echo "" + @echo "Usage: make [target]" + @echo "" + @echo "Targets:" + @echo " build Build the binary" + @echo " clean Remove built binary" + @echo " test Run all tests" + @echo " upgrade Update dependencies" + @echo " help Show this help message" + @echo "" + @echo "Variables:" + @echo " VERSION Set version (default: git describe)" + @echo "" + @echo "Example:" + @echo " make build" + +# vim: set ft=make noet ai ts=4 sw=4 sts=4: diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd46ef7 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# Backer + +Kind of backup utility. It makes tar.gz on-the-fly out of given in config.json files. + +## How to build it + +```bash +make # Build binary +make build VERSION=0.1.0 # Build with version +make test # Run tests +backer # Run +backer --version # Show version +backer -c /path/to/config.json # Custom config +``` + +That will produce binary named `backer`. Copy it somewhere in `/usr/local/sbin/backer`, for example. In `data` +directory resides `config_example.json`. Using this file make working config, put it into `/etc/backer.json`. + +Service ready to run. + +You can copy systemd unit from contrib directory to /etc/systemd/system and make usual rituals required for systemd. + +If you running alpine linux or gentoo, you can use openrc init script from contrib dir. + +If you running FreeBSD, use the rc.d script `contrib/backer.freebsd` and place it in `/usr/local/etc/rc.d/`. + +## How to use it + +Build, install and configure backer. On your backup storage server you can do + +```bash +curl -u username:password https://your_server:8086/archive -o "backup-$(date +%Y%m%d-%H%M%S).tar.gz" +``` + +and you'll get your backup with timestamp in filename, like `backup-20260325-105100.tar.gz` + +Of course, something can go wrong, so you must check http status code after downloading backup. + +## About config options + +AI suggests to fill this section with useful information, despite that all options described in example config. + +| Option | Default value | Wtf | Notes | +| :--- | :---- | :--- | :--- | +| address | "0.0.0.0" | Address at which server listens. | In many cases 0.0.0.0 is okay. | +| port | 8086 | Port at which server binds | Number picked after intel first comercially successful cpu. | +| cert | "/path/to/ssl.crt" | Path to ssl certificate file | It is not enforced in obvious form, but you should limit access to certificate at least with file permissions. | +| key | "/path/to/ssl.key" | Path to ssl certificates key file | It is not enforced in obvious form, but you should limit access to certificates key at least with file permissions. | +| nohttps | false | ability to disable https and use plain http. | This option is for lab experiments or for development needs. In case of this utility you should consider using long enough certificate with well ciphers. | +| location | "/archive" | This is static location where curl (or more sophisticated client) should aim to download backup | /archive is okay, but you free to choose something more wild like "/backup", just for giggles. | +| user | | You should pick up some flashy username for basic auth to prevent unauthorized access to your backup. | Enough said. | +| password | | You should pick up something super-duper secretous to prevent unauthorized access to your backup. | Enough said. | +| log | stderr | Where log messages are directed. If no value set, logs drops to stderr | | +| loglevel | | Verbosity of logs. | Can be error, warn, info, debug. Pick whatever you like. | +| directories | | An array with directories. Backer can work only with directories, not single files. | Put here all the folders you'd like to backup! | +| backup_timeout | 60 | Timeout in minutes for backup streaming operations. | If a backup takes longer than this, the connection will be terminated. Useful for large backups. Range: 1-1440 minutes (1 day max). | +| compression_level | 9 | Gzip compression level for tar.gz archives. | 1 (fastest) to 9 (best compression). Higher values produce smaller files but take longer to compress. | +| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. Files matching any pattern are skipped. | + +## Best practices + +Traditional yada-yada-yada about internet threats. + +It is entirely possible to use this utility without https/certificate. But you should not do this, really. Validity +of certificate is up to you, it can be even expired, but it should be well secured with reasonable key length and +good enough cypher. + +And one more thing user and password are mandatory, we should have at least some minimal barriers, right? + +## Special Thanks + +This project was developed with the assistance of **opencode** (AI co-programmer powered by MIMO). The AI helped with: +- Code implementation and refactoring +- Test coverage +- Documentation +- Bug fixes +- Linting and code style consistency + +Big thanks to the AI for doing most of the boring stuff while I was drinking coffee. diff --git a/TODO.org b/TODO.org new file mode 100644 index 0000000..11966d2 --- /dev/null +++ b/TODO.org @@ -0,0 +1,14 @@ +#+TITLE: Backer TODO List +#+AUTHOR: Backer Project +#+DATE: <2026-03-25 Wed> + +* TODO Backer Tasks + +** TODO Consider different compression algorithms + - [ ] Evaluate alternatives to gzip: bz2, xz, lzo, lz4 + - [ ] lz4 for speed, xz for better compression + - [ ] Add compression_algorithm config option + +** TODO Configurable filename prefix + - [ ] Move "backup" prefix to config (currently hardcoded in newServer.go) + - [ ] Default: "backup", e.g., "backup-20260325-092341.tar.gz" diff --git a/cmd/backer/main.go b/cmd/backer/main.go new file mode 100644 index 0000000..86a46c3 --- /dev/null +++ b/cmd/backer/main.go @@ -0,0 +1,76 @@ +// Package main implements minimal wrapper to run backer as go application. +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "runtime" + + "backer/internal/backer" + "backer/internal/log" + + "github.com/pkg/errors" +) + +var ( + // Version is set at build time via ldflags. + version = "dev" +) + +func main() { + configPath := flag.String("c", "", "path to config file (default: OS-specific location)") + showVersion := flag.Bool("version", false, "show version and exit") + + flag.Parse() + + if *showVersion { + fmt.Printf("backer %s\n", version) + + os.Exit(0) + } + + cfgPath := *configPath + if cfgPath == "" { + cfgPath = defaultConfigPath() + } + + server, err := backer.NewServer(cfgPath) + + if err != nil { + log.Fatalf("Failed to create server: %v", errors.WithStack(err)) + } + + log.Infof("Starting server at %s", server.Addr) + + if backer.C.NoHTTPS { + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("Failed to start server: %v", errors.WithStack(err)) + } + } else { + if _, err := os.Stat(backer.C.Cert); err != nil { + log.Fatalf("Certificate file error: %v", err) + } + + if _, err := os.Stat(backer.C.Key); err != nil { + log.Fatalf("Certificate key file error: %v", err) + } + + if err := server.ListenAndServeTLS(backer.C.Cert, backer.C.Key); err != nil && err != http.ErrServerClosed { + log.Fatalf("Failed to start server: %v", errors.WithStack(err)) + } + } +} + +// defaultConfigPath returns the default config file path based on the OS. +func defaultConfigPath() string { + switch runtime.GOOS { + case "freebsd", "netbsd", "openbsd", "dragonfly": + return "/usr/local/etc/backer.json" + default: // Linux, macOS, and others. + return "/etc/backer.json" + } +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/contrib/backer b/contrib/backer new file mode 100644 index 0000000..9279fb8 --- /dev/null +++ b/contrib/backer @@ -0,0 +1,18 @@ +#!/sbin/openrc-run + +name="$SVCNAME" +command="/var/lib/$SVCNAME/$SVCNAME" +supervisor="supervise-daemon" +pidfile="/run/$SVCNAME/$SVCNAME.pid" + +depend() { + need net + use dns +} + +start_pre() { + mkdir -p /var/log/$SVCNAME + mkdir -p /run/$SVCNAME +} + +# vim: set ft=sh noet ai ts=4 sw=4 sts=4: \ No newline at end of file diff --git a/contrib/backer.freebsd b/contrib/backer.freebsd new file mode 100644 index 0000000..f76b0bf --- /dev/null +++ b/contrib/backer.freebsd @@ -0,0 +1,30 @@ +#!/bin/sh + +# PROVIDE: backer +# REQUIRE: NETWORKING +# BEFORE: DAEMON + +. /etc/rc.subr + +name="backer" +rcvar="backer_enable" +command="/usr/local/sbin/backer" +command_args="" +pidfile="/var/run/${name}.pid" + +load_rc_config $name + +: ${backer_enable:="no"} +: ${backer_user:="root"} +: ${backer_group:="wheel"} +: ${backer_logfile:="/var/log/backer.log"} + +start_cmd="${name}_start" + +backer_start() +{ + echo "Starting ${name}." + /usr/sbin/daemon -p ${pidfile} -f ${command} ${command_args} +} + +run_rc_command "$1" diff --git a/contrib/backer.service b/contrib/backer.service new file mode 100644 index 0000000..d7e4f8a --- /dev/null +++ b/contrib/backer.service @@ -0,0 +1,14 @@ +[Unit] +Description=Backer Backup Service +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/sbin/backer +Restart=on-failure +RestartSec=10 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/data/config_example.json b/data/config_example.json new file mode 100644 index 0000000..6386e15 --- /dev/null +++ b/data/config_example.json @@ -0,0 +1,37 @@ +{ + /* Listen on other addresses is pointless. */ + "address": "0.0.0.0", + + /* By default listen on port 8086, like first intel commercially successful cpu model. */ + "port": 8086, + + /* Path location of certificate file, never use example from test_data! */ + "cert": "path/to/certificate.crt", + + /* Path location of certificate key, never use example from test_data! */ + "key": "/path/to/certificate.key", + + /* We really want to use https for backup archive tranferring. */ + "nohttps": false, + + /* Which location we should query by our client curl, for example. */ + "location": "/archive", + + /* User for basic auth. */ + "user": "secret", + + /* Password for basic auth. */ + "password": "Hush-hush secret password", + + /* Where to store logs. */ + "log": "/var/log/backer.log", + + /* Verbosity of logs */ + "loglevel": "info", + + /* Diretories to backup */ + "directories": [ + "/path/to/directory1", + "/path/to/directory2" + ] +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..19948ed --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module backer + +go 1.25 + +require ( + github.com/hjson/hjson-go v3.3.0+incompatible + github.com/pkg/errors v0.9.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..42c8663 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI= +github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/internal/backer/getFilesFromDirectories.go b/internal/backer/getFilesFromDirectories.go new file mode 100644 index 0000000..d78beb8 --- /dev/null +++ b/internal/backer/getFilesFromDirectories.go @@ -0,0 +1,53 @@ +// Package backer implements simple backup server. +package backer + +import ( + "io/fs" + "path/filepath" + + "backer/internal/log" +) + +// isExcluded checks if path matches any of the configured exclude patterns. +func isExcluded(path string) bool { + for _, re := range excludePatterns { + if re.MatchString(path) { + return true + } + } + + return false +} + +// GetFilesFromDirectories makes a file list of given directories. +func GetFilesFromDirectories(directories []string) ([]string, error) { + var files []string + + for _, dir := range directories { + err := filepath.WalkDir(dir, func(path string, de fs.DirEntry, err error) error { //nolint: revive + if err != nil { + log.Warnf("Skipping %s: %v", path, err) + + return nil + } + + if isExcluded(path) { + log.Debugf("Excluding: %s", path) + + return nil + } + + files = append(files, path) + + return nil + }) + + if err != nil { + log.Warnf("Failed to walk directory %s: %v", dir, err) + } + } + + return files, nil +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/getFilesFromDirectories_test.go b/internal/backer/getFilesFromDirectories_test.go new file mode 100644 index 0000000..a5d139d --- /dev/null +++ b/internal/backer/getFilesFromDirectories_test.go @@ -0,0 +1,269 @@ +package backer + +import ( + "os" + "regexp" + "slices" + "strings" + "testing" +) + +// normalizePath converts backslashes to forward slashes for cross-platform consistency. +func normalizePath(p string) string { + return strings.ReplaceAll(p, "\\", "/") +} + +// TestGetFilesFromDirectories checks if GetFilesFromDirectories returns the proper amount of strings, correct strings and no error. +func TestGetFilesFromDirectories(t *testing.T) { + var ( + input = Config{ + Directories: []string{"../../test_data/test1/foo", "../../test_data/test1/bar"}, + } + expectedData = []string{ + "../../test_data/test1/foo", + "../../test_data/test1/foo/empty_dir", + "../../test_data/test1/foo/hello_breakfast.txt", + "../../test_data/test1/foo/mydir", + "../../test_data/test1/foo/mydir/myfile.txt", + "../../test_data/test1/foo/some_text.txt", + "../../test_data/test1/bar", + "../../test_data/test1/bar/goodbye.txt", + "../../test_data/test1/bar/goodbye.txt/text.txt", + "../../test_data/test1/bar/hello.txt", + } + ) + + output, err := GetFilesFromDirectories(input.Directories) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + outputLen := len(output) + expectedDataLen := len(expectedData) + + if outputLen != expectedDataLen { + t.Errorf("length of output of GetFilesFromDirectories() does not match length of expected data: %d vs %d", outputLen, expectedDataLen) + } + + for _, filepath := range output { + normalized := normalizePath(filepath) + if !slices.Contains(expectedData, normalized) { + t.Errorf("expectedData data does not contain string from output: %s", filepath) + } + } +} + +// TestGetFilesFromDirectoriesSingleDir tests with a single directory. +func TestGetFilesFromDirectoriesSingleDir(t *testing.T) { + output, err := GetFilesFromDirectories([]string{"../../test_data/test1/foo"}) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if len(output) == 0 { + t.Error("Expected non-empty output") + } + + // Should contain the directory itself + if !slices.Contains(output, "../../test_data/test1/foo") { + t.Error("Output should contain the root directory") + } +} + +// TestGetFilesFromDirectoriesEmptyDir tests with an empty directory. +func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) { + // Create temp empty dir + tmpDir := t.TempDir() + + output, err := GetFilesFromDirectories([]string{tmpDir}) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + // Should contain at least the directory itself + if len(output) != 1 { + t.Errorf("Expected 1 entry (the directory itself), got %d", len(output)) + } +} + +// TestGetFilesFromDirectoriesNonExistent tests with non-existent directory. +func TestGetFilesFromDirectoriesNonExistent(t *testing.T) { + output, err := GetFilesFromDirectories([]string{"/nonexistent/path/that/does/not/exist"}) + + if err != nil { + t.Errorf("Expected no error (logs warning instead), got %v", err) + } + + if len(output) != 0 { + t.Errorf("Expected empty output for non-existent directory, got %d files", len(output)) + } +} + +// TestGetFilesFromDirectoriesMultipleDirs tests multiple directories. +func TestGetFilesFromDirectoriesMultipleDirs(t *testing.T) { + output, err := GetFilesFromDirectories([]string{ + "../../test_data/test1/foo", + "../../test_data/test1/bar", + }) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + // Should contain entries from both directories + hasFoo := slices.ContainsFunc(output, func(s string) bool { + return s == "../../test_data/test1/foo" || s == "../../test_data/test1/foo/hello_breakfast.txt" + }) + hasBar := slices.ContainsFunc(output, func(s string) bool { + return s == "../../test_data/test1/bar" || s == "../../test_data/test1/bar/hello.txt" + }) + + if !hasFoo { + t.Error("Output should contain entries from foo directory") + } + if !hasBar { + t.Error("Output should contain entries from bar directory") + } +} + +// TestGetFilesFromDirectoriesSymlink tests symlink handling. +func TestGetFilesFromDirectoriesSymlink(t *testing.T) { + // Create temp dir with symlink + tmpDir := t.TempDir() + targetFile := tmpDir + "/target.txt" + symlinkFile := tmpDir + "/link.txt" + + // Create target file + if err := os.WriteFile(targetFile, []byte("test"), 0644); err != nil { + t.Fatal(err) + } + + // Create symlink + if err := os.Symlink(targetFile, symlinkFile); err != nil { + t.Skip("Symlinks not supported, skipping test") + } + + output, err := GetFilesFromDirectories([]string{tmpDir}) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + // Should contain the symlink + if !slices.ContainsFunc(output, func(s string) bool { + return s == symlinkFile + }) { + t.Error("Output should contain the symlink") + } +} + +func TestIsExcluded(t *testing.T) { + // Save original excludePatterns and restore after test + originalPatterns := excludePatterns + defer func() { excludePatterns = originalPatterns }() + + // Test empty patterns + excludePatterns = nil + if isExcluded("/some/path") { + t.Error("Expected no exclusion with empty patterns") + } + + // Test matching pattern + excludePatterns = []*regexp.Regexp{ + regexp.MustCompile(`\.tmp$`), + regexp.MustCompile(`/node_modules/`), + } + if !isExcluded("/tmp/foo.tmp") { + t.Error("Expected exclusion for .tmp file") + } + if !isExcluded("/project/node_modules/package") { + t.Error("Expected exclusion for node_modules path") + } + if isExcluded("/tmp/foo.txt") { + t.Error("Expected no exclusion for .txt file") + } +} + +func TestGetFilesFromDirectoriesWithExcludes(t *testing.T) { + // Save original excludePatterns and restore after test + originalPatterns := excludePatterns + defer func() { excludePatterns = originalPatterns }() + + // Create a temporary directory with some files + tmpDir := t.TempDir() + // Create files that should be included + err := os.WriteFile(tmpDir+"/keep.txt", []byte("keep"), 0644) + if err != nil { + t.Fatal(err) + } + // Create files that should be excluded + err = os.WriteFile(tmpDir+"/skip.tmp", []byte("skip"), 0644) + if err != nil { + t.Fatal(err) + } + // Create a subdirectory with file + subDir := tmpDir + "/sub" + err = os.Mkdir(subDir, 0755) + if err != nil { + t.Fatal(err) + } + err = os.WriteFile(subDir+"/file.txt", []byte("file"), 0644) + if err != nil { + t.Fatal(err) + } + // Create a .tmp file in subdirectory + err = os.WriteFile(subDir+"/skip.tmp", []byte("skip"), 0644) + if err != nil { + t.Fatal(err) + } + + // Set exclude pattern for .tmp files + excludePatterns = []*regexp.Regexp{ + regexp.MustCompile(`\.tmp$`), + } + + output, err := GetFilesFromDirectories([]string{tmpDir}) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Expect entries: tmpDir, keep.txt, subDir, sub/file.txt + // Should NOT contain any .tmp files + expected := []string{ + tmpDir, + tmpDir + "/keep.txt", + subDir, + subDir + "/file.txt", + } + + // Normalize paths for cross-platform comparison + normalizedOutput := make([]string, len(output)) + for i, p := range output { + normalizedOutput[i] = normalizePath(p) + } + normalizedExpected := make([]string, len(expected)) + for i, p := range expected { + normalizedExpected[i] = normalizePath(p) + } + + if len(normalizedOutput) != len(normalizedExpected) { + t.Errorf("Expected %d entries, got %d: %v", len(normalizedExpected), len(normalizedOutput), normalizedOutput) + } + for _, exp := range normalizedExpected { + found := slices.Contains(normalizedOutput, exp) + if !found { + t.Errorf("Expected entry %q not found", exp) + } + } + // Ensure no .tmp files appear + for _, path := range normalizedOutput { + if strings.HasSuffix(path, ".tmp") { + t.Errorf("Excluded file %q appeared in output", path) + } + } +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/globals.go b/internal/backer/globals.go new file mode 100644 index 0000000..1012969 --- /dev/null +++ b/internal/backer/globals.go @@ -0,0 +1,14 @@ +package backer + +import ( + "regexp" +) + +var ( + // C contains the global configuration loaded from config file. + C Config + // excludePatterns contains compiled regex patterns from C.ExcludePatterns. + excludePatterns []*regexp.Regexp +) + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/loadConfig.go b/internal/backer/loadConfig.go new file mode 100644 index 0000000..9dcebd6 --- /dev/null +++ b/internal/backer/loadConfig.go @@ -0,0 +1,148 @@ +package backer + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "regexp" + + "backer/internal/log" + + hjson "github.com/hjson/hjson-go" +) + +// LoadConfig loads config file to global variable C. +func LoadConfig(path string) error { + var tmp map[string]any + + buf, err := os.ReadFile(path) + + if err != nil { + return fmt.Errorf("Failed to read config file: %w", err) + } + + if err = hjson.Unmarshal(buf, &tmp); err != nil { + return fmt.Errorf("Unable to parse config: %w", err) + } + + buf, err = json.Marshal(tmp) + + if err != nil { + return fmt.Errorf("Unable to convert config: %w", err) + } + + if err = json.Unmarshal(buf, &C); err != nil { + return fmt.Errorf("Failed to decode config file: %w", err) + } + + // Address: default and no range validation needed. + if C.Address == "" { + C.Address = "0.0.0.0" + + log.Warnf("Config option address is not set, fallback to %s", C.Address) + } + + // Port: set default, then validate range. + if C.Port == 0 { + C.Port = 8086 + + log.Warnf("Config option port is not set, fallback to %d", C.Port) + } + + if C.Port < 1 || C.Port > 65535 { + return fmt.Errorf("Port must be between 1 and 65535, got %d", C.Port) + } + + // Location: default and no range validation needed. + if C.Location == "" { + C.Location = "/archive" + + log.Warnf("Config option location is not set, fallback to %s", C.Location) + } + + // Log: default only. + if C.Log == "" { + log.Warn("Config option log is not set, fallback to stderr") + } + + // LogLevel: default and no range validation needed. + if C.LogLevel == "" { + C.LogLevel = "info" + + log.Warnf("Config option loglevel is not set, fallback to %s", C.LogLevel) + } + + // Directories: required, then validate existence. + if len(C.Directories) == 0 { + return errors.New("You should configure at least one directory to backup!") //nolint: revive + } + + for _, dir := range C.Directories { + if _, err := os.Stat(dir); err != nil { + return fmt.Errorf("Backup directory does not exist: %s", dir) + } + } + + // BackupTimeout: set default, then validate range. + if C.BackupTimeout == 0 { + C.BackupTimeout = 60 + + log.Warnf("Config option backup_timeout is not set, fallback to %d minutes", C.BackupTimeout) + } + + if C.BackupTimeout < 1 || C.BackupTimeout > 1440 { + return fmt.Errorf("Backup_timeout must be between 1 and 1440 minutes, got %d", C.BackupTimeout) + } + + // CompressionLevel: set default, then validate range. + if C.CompressionLevel == 0 { + C.CompressionLevel = 9 + + log.Warnf("Config option compression_level is not set, fallback to %d", C.CompressionLevel) + } + + if C.CompressionLevel < 1 || C.CompressionLevel > 9 { + return fmt.Errorf("Compression_level must be between 1 and 9, got %d", C.CompressionLevel) + } + + // HTTPS: validate cert/key when HTTPS enabled. + if !C.NoHTTPS { + switch { + case C.Cert == "" && C.Key == "": + return errors.New("Config option nohttps set to false, but neither cert nor key options set") + case C.Cert == "": + return errors.New("Config option nohttps set to false, but no cert option set") + case C.Key == "": + return errors.New("Config option nohttps set to false, but no key option set") + } + } + + // User: required. + if C.User == "" { + return errors.New("Option user is not set") + } + + // Password: required. + if C.Password == "" { + return errors.New("Option password is not set") + } + + // Compile exclude patterns. + excludePatterns = make([]*regexp.Regexp, 0, len(C.ExcludePatterns)) + + for _, pattern := range C.ExcludePatterns { + re, err := regexp.Compile(pattern) + if err != nil { + log.Warnf("Invalid exclude pattern %q: %v", pattern, err) + + continue + } + + excludePatterns = append(excludePatterns, re) + } + + return nil +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/loadConfig_test.go b/internal/backer/loadConfig_test.go new file mode 100644 index 0000000..39ea0bb --- /dev/null +++ b/internal/backer/loadConfig_test.go @@ -0,0 +1,223 @@ +package backer + +import ( + "os" + "slices" + "testing" +) + +func TestLoadConfig(t *testing.T) { + // Reset global config + C = Config{} + + var ( + testConfigPath = "./../../test_data/test_config.json" + expectedData = Config{ + Address: "127.0.0.1", + Port: 8086, + Directories: []string{ + "../../test_data/test1/foo", + "../../test_data/test1/bar", + }, + Location: "/archive", + } + ) + + err := LoadConfig(testConfigPath) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if C.Address != expectedData.Address { + t.Errorf("C.Address should contain %s but contains %s", expectedData.Address, C.Address) + } + + if C.Port != expectedData.Port { + t.Errorf("C.Port should contain %d but containc %d", expectedData.Port, C.Port) + } + + if C.Location != expectedData.Location { + t.Errorf("C.StreamLocation should contain %s but contains %s", expectedData.Location, C.Location) + } + + outputDirectoriesLen := len(C.Directories) + expectedDataDirectoriesLen := len(expectedData.Directories) + + if outputDirectoriesLen != expectedDataDirectoriesLen { + t.Errorf("length of C.Directories must be %d but is %d", expectedDataDirectoriesLen, outputDirectoriesLen) + } + + for _, str := range expectedData.Directories { + if !slices.Contains(C.Directories, str) { + t.Errorf("C.Directories data does not contain expected string: %s", str) + } + } +} + +// TestLoadConfigMissingUser tests that config fails when user is missing. +func TestLoadConfigMissingUser(t *testing.T) { + config := `{ + "port": 8086, + "password": "test", + "directories": ["../../test_data/test1/foo"] + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err == nil { + t.Error("Expected error for missing user, got nil") + } +} + +// TestLoadConfigMissingPassword tests that config fails when password is missing. +func TestLoadConfigMissingPassword(t *testing.T) { + config := `{ + "port": 8086, + "user": "test", + "directories": ["../../test_data/test1/foo"] + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err == nil { + t.Error("Expected error for missing password, got nil") + } +} + +// TestLoadConfigMissingDirectories tests that config fails when directories is missing. +func TestLoadConfigMissingDirectories(t *testing.T) { + config := `{ + "port": 8086, + "user": "test", + "password": "test" + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err == nil { + t.Error("Expected error for missing directories, got nil") + } +} + +// TestLoadConfigInvalidPort tests that config fails with invalid port. +func TestLoadConfigInvalidPort(t *testing.T) { + config := `{ + "port": 99999, + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"] + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err == nil { + t.Error("Expected error for invalid port, got nil") + } +} + +// TestLoadConfigNonExistentDirectory tests that config fails with non-existent directory. +func TestLoadConfigNonExistentDirectory(t *testing.T) { + config := `{ + "port": 8086, + "user": "test", + "password": "test", + "directories": ["/nonexistent/path"] + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err == nil { + t.Error("Expected error for non-existent directory, got nil") + } +} + +// TestLoadConfigHTTPSWithoutCert tests that config fails when HTTPS enabled but no cert. +func TestLoadConfigHTTPSWithoutCert(t *testing.T) { + config := `{ + "port": 8086, + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"], + "nohttps": false + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err == nil { + t.Error("Expected error for HTTPS without cert, got nil") + } +} + +// TestLoadConfigNoHTTPS tests that config succeeds with nohttps=true and no cert. +func TestLoadConfigNoHTTPS(t *testing.T) { + config := `{ + "port": 8086, + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"], + "nohttps": true + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err != nil { + t.Errorf("Expected no error with nohttps=true, got %v", err) + } + + if !C.NoHTTPS { + t.Error("C.NoHTTPS should be true") + } +} + +// TestLoadConfigDefaultValues tests default values are set correctly. +func TestLoadConfigDefaultValues(t *testing.T) { + config := `{ + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"], + "cert": "../../test_data/example.crt", + "key": "../../test_data/example.key" + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if C.Address != "0.0.0.0" { + t.Errorf("C.Address should default to 0.0.0.0, got %s", C.Address) + } + + if C.Port != 8086 { + t.Errorf("C.Port should default to 8086, got %d", C.Port) + } + + if C.Location != "/archive" { + t.Errorf("C.Location should default to /archive, got %s", C.Location) + } + + if C.LogLevel != "info" { + t.Errorf("C.LogLevel should default to info, got %s", C.LogLevel) + } +} + +// writeTempConfig is a helper to write a temp config file for testing. +func writeTempConfig(t *testing.T, content string) string { + t.Helper() + tmpFile, err := os.CreateTemp(t.TempDir(), "test_config_*.json") + if err != nil { + t.Fatal(err) + } + if _, err := tmpFile.WriteString(content); err != nil { + t.Fatal(err) + } + tmpFile.Close() + // Reset global config before loading + C = Config{} + excludePatterns = nil + return tmpFile.Name() +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/newServer.go b/internal/backer/newServer.go new file mode 100644 index 0000000..684fb88 --- /dev/null +++ b/internal/backer/newServer.go @@ -0,0 +1,194 @@ +package backer + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "strings" + "time" + + "backer/internal/log" +) + +// withServerHeader wraps a handler to add Server header to all responses. +func withServerHeader(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "backer") + next.ServeHTTP(w, r) + }) +} + +// NewServer starts new server instance with given config file. +func NewServer(configPath string) (*http.Server, error) { + if err := LoadConfig(configPath); err != nil { + return nil, fmt.Errorf("Failed to load config: %w", err) + } + + if err := log.Init(C.LogLevel, C.Log); err != nil { + log.Errorf("Failed to initialize logger: %v", err) + } + + mux := http.NewServeMux() + + mux.HandleFunc(C.Location, func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + + // No auth? Good bye. + if !ok { + w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + + return + } + + // Both username AND password must match. Fuck De Morgan and his bullshit laws - we explicitly require + // that the whole condition fails. That *is* the point. + //nolint:staticcheck // QF1001: De Morgan's law is irrelevant here. + if !(username == C.User && password == C.Password) { + w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + + return + } + + clientIP := getClientIP(r) + backupStart := time.Now() + + log.Infof("Backup started: client=%s user=%s", clientIP, username) + + ctx := r.Context() + files, err := GetFilesFromDirectories(C.Directories) + + if err != nil { + log.Errorf("Failed to get files: %v", err) + + w.Header().Set("Server", "backer") + http.Error(w, "Failed to get files", http.StatusInternalServerError) + + return + } + + tarGz := CreateTarGzStream(ctx, files) + + defer func() { + if err := tarGz.Close(); err != nil { + log.Errorf("Failed to close tar.gz reader: %v", err) + } + }() + + timestamp := time.Now().Format("20060102-150405") + filename := fmt.Sprintf("backup-%s.tar.gz", timestamp) + + w.Header().Set("Content-Disposition", "attachment; filename="+filename) + w.Header().Set("Content-Type", "application/octet-stream") + + bytesWritten, err := io.Copy(w, tarGz) + duration := time.Since(backupStart) + + if err != nil { + log.Errorf("Backup failed: client=%s user=%s duration=%s error=%v", clientIP, username, duration, err) + + http.Error(w, "Failed to stream archive", http.StatusInternalServerError) + + return + } + + log.Infof("Backup completed: client=%s user=%s files=%d bytes=%d duration=%s", clientIP, username, len(files), bytesWritten, duration) + }) + + return &http.Server{ + Addr: fmt.Sprintf("%s:%d", C.Address, C.Port), + Handler: withServerHeader(mux), + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: time.Duration(C.BackupTimeout) * time.Minute, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS13, + }, + }, nil +} + +// writeWithContext executes fn in a goroutine and returns its result. +// If the context is canceled before fn completes, ctx.Err() is returned. +// fn should perform I/O that will unblock when the underlying pipe is closed +// on context cancellation (e.g., writes to a pipe writer). Panics in fn are +// recovered and returned as errors. +func writeWithContext(ctx context.Context, fn func() error) error { + done := make(chan error, 1) + + go func() { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("panic: %v", r) + } + }() + + select { + case <-ctx.Done(): + done <- ctx.Err() + default: + done <- fn() + } + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-done: + return err + } +} + +// copyWithContext copies data from src to dst using a 32KB buffer, respecting +// context cancellation. If the context is canceled, the copy stops and returns +// ctx.Err(). Panics are recovered and returned as errors. +func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (err error) { //nolint:nonamedreturns + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + + buf := make([]byte, 32*1024) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + n, readErr := src.Read(buf) + if n > 0 { + if _, werr := dst.Write(buf[:n]); werr != nil { + return werr + } + } + + if readErr != nil { + if readErr == io.EOF { + return nil + } + + return readErr + } + } +} + +// getClientIP extracts the client IP address from the request. +// It checks X-Forwarded-For header first (for proxied requests), then falls back to RemoteAddr. +func getClientIP(r *http.Request) string { + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + return strings.Split(xff, ",")[0] + } + + ip := r.RemoteAddr + if colon := strings.LastIndex(ip, ":"); colon != -1 { + return ip[:colon] + } + + return ip +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/newServer_test.go b/internal/backer/newServer_test.go new file mode 100644 index 0000000..152df50 --- /dev/null +++ b/internal/backer/newServer_test.go @@ -0,0 +1,374 @@ +package backer + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestNewServerSuccess tests successful server creation. +func TestNewServerSuccess(t *testing.T) { + C = Config{} // Reset global config. + + server, err := NewServer("./../../test_data/test_config.json") + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if server == nil { + t.Fatal("Expected non-nil server") + } + + if server.Addr != "127.0.0.1:8086" { + t.Errorf("Expected address '127.0.0.1:8086', got '%s'", server.Addr) + } +} + +// TestNewServerInvalidConfig tests server creation with invalid config. +func TestNewServerInvalidConfig(t *testing.T) { + C = Config{} // Reset global config. + + _, err := NewServer("/nonexistent/config.json") + if err == nil { + t.Error("Expected error for invalid config path, got nil") + } +} + +// TestHandlerNoAuth tests that requests without auth are rejected. +func TestHandlerNoAuth(t *testing.T) { + C = Config{ + Address: "0.0.0.0", + Port: 8086, + Location: "/archive", + User: "testuser", + Password: "testpass", + Directories: []string{"../../test_data/test1/foo"}, + } + excludePatterns = nil + + req := httptest.NewRequest("GET", "/archive", nil) + w := httptest.NewRecorder() + + // Create a handler that mimics the server's handler + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + + if !ok { + w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + if !(username == C.User && password == C.Password) { + w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Errorf("Expected status 401, got %d", w.Code) + } + + authHeader := w.Header().Get("WWW-Authenticate") + if !strings.Contains(authHeader, "Basic") { + t.Errorf("Expected WWW-Authenticate header with Basic, got '%s'", authHeader) + } +} + +// TestHandlerWrongCredentials tests that requests with wrong credentials are rejected. +func TestHandlerWrongCredentials(t *testing.T) { + C = Config{ + Address: "0.0.0.0", + Port: 8086, + Location: "/archive", + User: "testuser", + Password: "testpass", + Directories: []string{"../../test_data/test1/foo"}, + } + excludePatterns = nil + + req := httptest.NewRequest("GET", "/archive", nil) + req.SetBasicAuth("wronguser", "wrongpass") + w := httptest.NewRecorder() + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + + if !ok { + w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + if !(username == C.User && password == C.Password) { + w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Errorf("Expected status 401, got %d", w.Code) + } +} + +// TestHandlerCorrectCredentials tests that requests with correct credentials succeed. +func TestHandlerCorrectCredentials(t *testing.T) { + C = Config{ + Address: "0.0.0.0", + Port: 8086, + Location: "/archive", + User: "testuser", + Password: "testpass", + Directories: []string{"../../test_data/test1/foo"}, + } + excludePatterns = nil + + req := httptest.NewRequest("GET", "/archive", nil) + req.SetBasicAuth("testuser", "testpass") + w := httptest.NewRecorder() + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + + if !ok { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + if !(username == C.User && password == C.Password) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("Expected status 200, got %d", w.Code) + } +} + +// TestHandlerArchiveStream tests that the handler streams a valid tar.gz archive. +func TestHandlerArchiveStream(t *testing.T) { + C = Config{ + Address: "0.0.0.0", + Port: 8086, + Location: "/archive", + User: "testuser", + Password: "testpass", + Directories: []string{"../../test_data/test1/foo"}, + } + excludePatterns = nil + + // Create a test server with the actual handler logic + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + if !ok || !(username == C.User && password == C.Password) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + ctx := r.Context() + files, err := GetFilesFromDirectories(C.Directories) + if err != nil { + http.Error(w, "Failed to get files", http.StatusInternalServerError) + return + } + + tarGz := CreateTarGzStream(ctx, files) + defer tarGz.Close() + + w.Header().Set("Content-Disposition", "attachment; filename=archive.tar") + w.Header().Set("Content-Type", "application/octet-stream") + + io.Copy(w, tarGz) + })) + defer server.Close() + + // Make authenticated request + req, _ := http.NewRequest("GET", server.URL, nil) + req.SetBasicAuth("testuser", "testpass") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + // Verify it's a valid tar.gz + gzReader, err := gzip.NewReader(resp.Body) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + foundFile := false + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Failed to read tar entry: %v", err) + } + + if strings.Contains(header.Name, "hello_breakfast.txt") { + foundFile = true + } + } + + if !foundFile { + t.Error("Expected to find hello_breakfast.txt in archive") + } +} + +// TestWriteWithContextSuccess tests successful write with context. +func TestWriteWithContextSuccess(t *testing.T) { + ctx := context.Background() + called := false + + err := writeWithContext(ctx, func() error { + called = true + return nil + }) + + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if !called { + t.Error("Expected function to be called") + } +} + +// TestWriteWithContextError tests write with context returning error. +func TestWriteWithContextError(t *testing.T) { + ctx := context.Background() + testErr := io.ErrUnexpectedEOF + + err := writeWithContext(ctx, func() error { + return testErr + }) + + if err != testErr { + t.Errorf("Expected %v, got %v", testErr, err) + } +} + +// TestWriteWithContextCancellation tests write with cancelled context. +func TestWriteWithContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + err := writeWithContext(ctx, func() error { + t.Error("Function should not be called with cancelled context") + return nil + }) + + if err != context.Canceled { + t.Errorf("Expected context.Canceled, got %v", err) + } +} + +// TestCopyWithContextSuccess tests successful copy with context. +func TestCopyWithContextSuccess(t *testing.T) { + ctx := context.Background() + src := strings.NewReader("hello world") + dst := &strings.Builder{} + + err := copyWithContext(ctx, dst, src) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if dst.String() != "hello world" { + t.Errorf("Expected 'hello world', got '%s'", dst.String()) + } +} + +// TestCopyWithContextCancellation tests copy with cancelled context. +func TestCopyWithContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + // Create a reader with a lot of data that will take time to copy + data := make([]byte, 1024*1024) // 1MB + for i := range data { + data[i] = byte(i % 256) + } + src := strings.NewReader(string(data)) + dst := &strings.Builder{} + + // Start copy in goroutine + errCh := make(chan error, 1) + go func() { + errCh <- copyWithContext(ctx, dst, src) + }() + + // Cancel context immediately + cancel() + + err := <-errCh + if err != context.Canceled { + t.Errorf("Expected context.Canceled, got %v", err) + } +} + +func TestWriteWithContextPanic(t *testing.T) { + ctx := context.Background() + err := writeWithContext(ctx, func() error { + panic("test panic") + }) + if err == nil { + t.Error("Expected error from panic, got nil") + } + if !strings.Contains(err.Error(), "panic") { + t.Errorf("Expected panic error, got %v", err) + } +} + +func TestCopyWithContextPanic(t *testing.T) { + ctx := context.Background() + // Use a reader that panics on Read + panicReader := &panicReader{} + dst := &strings.Builder{} + err := copyWithContext(ctx, dst, panicReader) + if err == nil { + t.Error("Expected error from panic, got nil") + } + if !strings.Contains(err.Error(), "panic") { + t.Errorf("Expected panic error, got %v", err) + } +} + +type panicReader struct{} + +func (pr *panicReader) Read(p []byte) (n int, err error) { + panic("read panic") +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/types.go b/internal/backer/types.go new file mode 100644 index 0000000..0b8cf1d --- /dev/null +++ b/internal/backer/types.go @@ -0,0 +1,23 @@ +package backer + +type ( + // Config main config struct. + Config struct { + Address string `json:"address"` + Port int `json:"port"` + Cert string `json:"cert"` + Key string `json:"key"` + NoHTTPS bool `json:"nohttps"` + Location string `json:"location"` + User string `json:"user"` + Password string `json:"password"` + Log string `json:"log"` + LogLevel string `json:"loglevel"` + Directories []string `json:"directories"` + BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming. + CompressionLevel int `json:"compression_level"` // Gzip compression level (1-9, default 9). + ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup. + } +) + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/utils.go b/internal/backer/utils.go new file mode 100644 index 0000000..28ea288 --- /dev/null +++ b/internal/backer/utils.go @@ -0,0 +1,177 @@ +package backer + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "backer/internal/log" +) + +// CreateTarGzStream takes a list of files and returns a reader that reads from a tar archive containing these files. +func CreateTarGzStream(ctx context.Context, filepaths []string) io.ReadCloser { + pr, pw := io.Pipe() + + go func() { + var pipeErr error // Track if we close pipe with error. + + // Helper to close pipe with error, only once. + closePipeWithError := func(err error) { + if pipeErr != nil { + return // Already closed with error. + } + + pipeErr = err + pw.CloseWithError(err) + } + + defer func() { + // Only do normal close if we haven't closed with error. + if pipeErr == nil { + pw.Close() + } + }() + + gw, err := gzip.NewWriterLevel(pw, C.CompressionLevel) + if err != nil { + closePipeWithError(fmt.Errorf("gzip writer error: %w", err)) + + return + } + + defer func() { + // Only close gzip writer if pipe is still open. + if pipeErr == nil { + if err := gw.Close(); err != nil { + // Ignore pipe closed errors - reader may have closed early. + if !isPipeClosedError(err) { + log.Errorf("Failed to close gzip writer: %v", err) + } + } + } + }() + + tw := tar.NewWriter(gw) + + defer func() { + // Only close tar writer if pipe is still open. + if pipeErr == nil { + if err := tw.Close(); err != nil { + // Ignore pipe closed errors - reader may have closed early. + if !isPipeClosedError(err) { + log.Errorf("Failed to close tar writer: %v", err) + } + } + } + }() + + for _, fpath := range filepaths { + var ( + err error + linkTarget string + ) + + // Check for cancellation early. + select { + case <-ctx.Done(): + closePipeWithError(ctx.Err()) + + return + + default: + } + + log.Debugf("Adding file: %s", fpath) + + st, err := os.Lstat(fpath) + + if err != nil { + log.Warnf("Skipping %s: %v", fpath, err) + + continue + } + + if st.Mode()&os.ModeSymlink != 0 { + linkTarget, err = os.Readlink(fpath) + + if err != nil { + log.Warnf("Skipping %s: readlink failed: %s", fpath, err) + + continue + } + } + + // Skip sockets explicitly (they cannot be backed up). + if st.Mode()&os.ModeSocket != 0 { + log.Debugf("Skipping socket: %s", fpath) + + continue + } + + header, err := tar.FileInfoHeader(st, linkTarget) + + if err != nil { + log.Warnf("Skipping %s: tar header failed: %v", fpath, err) + + continue + } + + header.Format = tar.FormatGNU + header.Name = strings.TrimPrefix(filepath.Clean(fpath), string(filepath.Separator)) + + mode := st.Mode() + + // Write header with cancellation awareness. + if err := writeWithContext(ctx, func() error { + return tw.WriteHeader(header) + }); err != nil { + closePipeWithError(err) + + return + } + + // Only stream file contents for regular files. + if mode.IsRegular() { + f, err := os.Open(fpath) + if err != nil { + log.Warnf("Skipping %s: open failed: %v", fpath, err) + + continue + } + + err = copyWithContext(ctx, tw, f) + + if e := f.Close(); e != nil { + log.Warnf("Failed to close file %s: %v", fpath, e) + } + + if err != nil { + log.Warnf("Skipping %s: copy failed: %v", fpath, err) + + continue + } + } + + } + }() + + return pr +} + +// isPipeClosedError checks if the error is due to a closed pipe. +func isPipeClosedError(err error) bool { + if err == nil { + return false + } + + // Check for the specific pipe closed error message. + return err.Error() == "io: read/write on closed pipe" || + err.Error() == "io: read on closed pipe" +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/utils_test.go b/internal/backer/utils_test.go new file mode 100644 index 0000000..a9c85dc --- /dev/null +++ b/internal/backer/utils_test.go @@ -0,0 +1,359 @@ +package backer + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "os" + "path/filepath" + "testing" +) + +// TestCreateTarGzStreamBasic tests basic tar.gz creation with a single file. +func TestCreateTarGzStreamBasic(t *testing.T) { + // Create temp file + tmpDir := t.TempDir() + testFile := filepath.Join(tmpDir, "test.txt") + content := []byte("hello world") + if err := os.WriteFile(testFile, content, 0644); err != nil { + t.Fatal(err) + } + + ctx := context.Background() + reader := CreateTarGzStream(ctx, []string{testFile}) + defer reader.Close() + + // Decompress and read + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + // Read the file entry + header, err := tarReader.Next() + if err != nil { + t.Fatalf("Failed to read tar header: %v", err) + } + + if header.Name != testFile && filepath.Base(header.Name) != "test.txt" { + t.Errorf("Expected file name 'test.txt', got '%s'", header.Name) + } + + if header.Typeflag != tar.TypeReg { + t.Errorf("Expected regular file type, got %d", header.Typeflag) + } + + // Read content + data, err := io.ReadAll(tarReader) + if err != nil { + t.Fatalf("Failed to read file content: %v", err) + } + + if string(data) != string(content) { + t.Errorf("Expected content '%s', got '%s'", content, data) + } + + // Should be no more entries + if _, err := tarReader.Next(); err != io.EOF { + t.Error("Expected EOF after single file") + } +} + +// TestCreateTarGzStreamMultipleFiles tests with multiple files. +func TestCreateTarGzStreamMultipleFiles(t *testing.T) { + tmpDir := t.TempDir() + + // Create multiple files + files := []string{ + filepath.Join(tmpDir, "file1.txt"), + filepath.Join(tmpDir, "file2.txt"), + filepath.Join(tmpDir, "file3.txt"), + } + + for i, f := range files { + if err := os.WriteFile(f, []byte{byte('0' + i)}, 0644); err != nil { + t.Fatal(err) + } + } + + ctx := context.Background() + reader := CreateTarGzStream(ctx, files) + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + count := 0 + for { + _, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Failed to read tar entry: %v", err) + } + count++ + } + + if count != 3 { + t.Errorf("Expected 3 entries, got %d", count) + } +} + +// TestCreateTarGzStreamDirectory tests directory handling. +func TestCreateTarGzStreamDirectory(t *testing.T) { + tmpDir := t.TempDir() + subDir := filepath.Join(tmpDir, "subdir") + if err := os.Mkdir(subDir, 0755); err != nil { + t.Fatal(err) + } + + ctx := context.Background() + reader := CreateTarGzStream(ctx, []string{subDir}) + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + header, err := tarReader.Next() + if err != nil { + t.Fatalf("Failed to read tar header: %v", err) + } + + if header.Typeflag != tar.TypeDir { + t.Errorf("Expected directory type, got %d", header.Typeflag) + } +} + +// TestCreateTarGzStreamSymlink tests symlink handling. +func TestCreateTarGzStreamSymlink(t *testing.T) { + tmpDir := t.TempDir() + targetFile := filepath.Join(tmpDir, "target.txt") + symlinkFile := filepath.Join(tmpDir, "link.txt") + + if err := os.WriteFile(targetFile, []byte("target content"), 0644); err != nil { + t.Fatal(err) + } + + if err := os.Symlink(targetFile, symlinkFile); err != nil { + t.Skip("Symlinks not supported, skipping test") + } + + ctx := context.Background() + reader := CreateTarGzStream(ctx, []string{symlinkFile}) + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + header, err := tarReader.Next() + if err != nil { + t.Fatalf("Failed to read tar header: %v", err) + } + + if header.Typeflag != tar.TypeSymlink { + t.Errorf("Expected symlink type, got %c", header.Typeflag) + } + + if header.Linkname != targetFile { + t.Errorf("Expected linkname '%s', got '%s'", targetFile, header.Linkname) + } +} + +// TestCreateTarGzStreamCancellation tests context cancellation. +func TestCreateTarGzStreamCancellation(t *testing.T) { + tmpDir := t.TempDir() + + // Create a file + testFile := filepath.Join(tmpDir, "test.txt") + if err := os.WriteFile(testFile, []byte("test content"), 0644); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + + // Cancel before reading + cancel() + + reader := CreateTarGzStream(ctx, []string{testFile}) + defer reader.Close() + + // Read all - should complete without panic + data, err := io.ReadAll(reader) + if err != nil { + // Error is expected due to cancellation + t.Logf("Got expected error: %v", err) + } + + // We should get some data or an error, but not panic + _ = data +} + +// TestCreateTarGzStreamNonExistentFile tests handling of non-existent files. +func TestCreateTarGzStreamNonExistentFile(t *testing.T) { + ctx := context.Background() + reader := CreateTarGzStream(ctx, []string{"/nonexistent/path/file.txt"}) + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + // Should get EOF immediately since the file doesn't exist and is skipped + _, err = tarReader.Next() + if err != io.EOF { + t.Errorf("Expected EOF for non-existent file, got: %v", err) + } +} + +// TestCreateTarGzStreamEmptyList tests with empty file list. +func TestCreateTarGzStreamEmptyList(t *testing.T) { + ctx := context.Background() + reader := CreateTarGzStream(ctx, []string{}) + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + // Should get EOF immediately for empty list + _, err = tarReader.Next() + if err != io.EOF { + t.Errorf("Expected EOF for empty list, got: %v", err) + } +} + +// TestCreateTarGzStreamNestedFiles tests with nested directory structure. +func TestCreateTarGzStreamNestedFiles(t *testing.T) { + tmpDir := t.TempDir() + + // Create nested structure + nestedDir := filepath.Join(tmpDir, "a", "b", "c") + if err := os.MkdirAll(nestedDir, 0755); err != nil { + t.Fatal(err) + } + + nestedFile := filepath.Join(nestedDir, "deep.txt") + if err := os.WriteFile(nestedFile, []byte("deep content"), 0644); err != nil { + t.Fatal(err) + } + + ctx := context.Background() + // Pass the nested file directly, not the root + reader := CreateTarGzStream(ctx, []string{nestedFile}) + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + foundDeepFile := false + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Failed to read tar entry: %v", err) + } + + if filepath.Base(header.Name) == "deep.txt" { + foundDeepFile = true + + // Verify content + data, err := io.ReadAll(tarReader) + if err != nil { + t.Fatalf("Failed to read content: %v", err) + } + if string(data) != "deep content" { + t.Errorf("Expected 'deep content', got '%s'", data) + } + } + } + + if !foundDeepFile { + t.Error("Expected to find deep.txt in archive") + } +} + +// TestCreateTarGzStreamDeviceFile tests that device files are included in the archive. +func TestCreateTarGzStreamDeviceFile(t *testing.T) { + // Use /dev/null as a test device file (character device). + devNull := "/dev/null" + st, err := os.Lstat(devNull) + if err != nil { + t.Skipf("Skipping device test: %v", err) + } + if st.Mode()&os.ModeCharDevice == 0 { + t.Skipf("Skipping: %s is not a character device", devNull) + } + + ctx := context.Background() + reader := CreateTarGzStream(ctx, []string{devNull}) + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + t.Fatalf("Failed to create gzip reader: %v", err) + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + + // Read the device entry + header, err := tarReader.Next() + if err != nil { + t.Fatalf("Failed to read tar header: %v", err) + } + + if header.Typeflag != tar.TypeChar { + t.Errorf("Expected character device type, got %c", header.Typeflag) + } + + // Check that the name matches the file path (without leading slash) + expectedName := devNull + if expectedName[0] == '/' { + expectedName = expectedName[1:] + } + if header.Name != expectedName { + t.Errorf("Expected header name %q, got %q", expectedName, header.Name) + } + + // Ensure there are no more entries + if _, err := tarReader.Next(); err != io.EOF { + t.Error("Expected EOF after device entry") + } +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/log/main.go b/internal/log/main.go new file mode 100644 index 0000000..602b33e --- /dev/null +++ b/internal/log/main.go @@ -0,0 +1,148 @@ +// Package log implements logging facility as golang slog module wrapper. +package log + +import ( + "fmt" + "log/slog" + "os" + "time" +) + +var ( + // Log contains *[os.File] pointing to log file descriptor. + Log *os.File +) + +// Init sets up logger stuff. +// level can be error, warn, info, debug; if something other is supplied, info level is selected. +func Init(level, filename string) error { + var ( + err error + loglevel slog.Level + ) + + if filename != "" { + Log, err = os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + + if err != nil { + return fmt.Errorf("Unable to open log file: %w", err) + } + } else { + Log = os.Stderr + } + + // no panic, no trace. + switch level { + case "error": + loglevel = slog.LevelError + + case "warn": + loglevel = slog.LevelWarn + + case "info": + loglevel = slog.LevelInfo + + case "debug": + loglevel = slog.LevelDebug + + default: + loglevel = slog.LevelInfo + } + + opts := &slog.HandlerOptions{ + // Use the ReplaceAttr function on the handler options + // to be able to replace any single attribute in the log output. + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { //nolint: revive + // Check that we are handling the time key. + if a.Key != slog.TimeKey { + return a + } + + t := a.Value.Time() + + // Change the value from a time.Time to a String + // where the string has the correct time format. + a.Value = slog.StringValue(t.Format(time.DateTime)) + + return a + }, + + Level: loglevel, + } + + slog.SetDefault( + slog.New( + slog.NewTextHandler( + Log, + opts, + ), + ), + ) + + return err +} + +// Error passes message directly to [slog.Error()]. +func Error(message string) { + slog.Error(message) +} + +// Errorf logs message on error log level and allows to supply arguments in printf() style. +func Errorf(format string, a ...any) { + slog.Error(fmt.Sprintf(format, a...)) +} + +// Warn passes message directly to [slog.Warn()]. +func Warn(message string) { + slog.Warn(message) +} + +// Warnf logs message on warn log level and allows to supply arguments in printf() style. +func Warnf(format string, a ...any) { + slog.Warn(fmt.Sprintf(format, a...)) +} + +// Info passes message directly to [slog.Info()]. +func Info(message string) { + slog.Info(message) +} + +// Infof logs message on info log level and allows to supply arguments in printf() style. +func Infof(format string, a ...any) { + slog.Info(fmt.Sprintf(format, a...)) +} + +// Debug passes message directly to [slog.Debug()]. +func Debug(message string) { + slog.Debug(message) +} + +// Debugf logs message on debug log level and allows to supply arguments in printf() style. +func Debugf(format string, a ...any) { + slog.Debug(fmt.Sprintf(format, a...)) +} + +// Fatal passes message directly to [slog.Error()] and quits with status code 1. +func Fatal(message string) { + slog.Error(message) + Close() + os.Exit(1) +} + +// Fatalf logs message on error log level and allows to supply arguments in printf() style. After that it quits with +// status code 1. +func Fatalf(format string, a ...any) { + slog.Error(fmt.Sprintf(format, a...)) + Close() + os.Exit(1) +} + +// Close closes log file descriptor. +func Close() { + if err := Log.Close(); err != nil { + msg := fmt.Sprintln("Failed to close log file:", err) + slog.Error(msg) + } +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/llms.txt b/llms.txt new file mode 100644 index 0000000..4c6faba --- /dev/null +++ b/llms.txt @@ -0,0 +1,207 @@ +# llms.txt for Backer + +## Overview +Backer is a Go-based backup utility that creates tar.gz archives on-the-fly and serves them via HTTP/HTTPS with basic authentication. + +## Quick Start +```bash +make # Build binary +make build VERSION=0.1.0 # Build with version +backer # Run with default config (/etc/backer.json) +backer -c /path/to/config.json # Run with custom config +backer --version # Show version +backer -h # Show help +curl -u user:pass https://server:8086/archive -o backup.tar.gz +``` + +## Config File Locations +- Linux: `/etc/backer.json` +- FreeBSD/NetBSD/OpenBSD/DragonFly: `/usr/local/etc/backer.json` +- Override with `-c` flag + +## Directory Structure +``` +/ +├── cmd/backer/main.go # Entry point, CLI flags, server startup +├── internal/ +│ ├── backer/ # Core logic +│ │ ├── types.go # Config struct definition +│ │ ├── globals.go # Global C Config variable +│ │ ├── loadConfig.go # HJSON/JSON config parsing with validation +│ │ ├── newServer.go # HTTP server, auth handler, context helpers +│ │ ├── utils.go # CreateTarGzStream, isPipeClosedError +│ │ ├── getFilesFromDirectories.go # Directory walking, exclusion filtering +│ │ └── *_test.go # Test files +│ └── log/main.go # Logging wrapper around stdlib slog +├── contrib/ # Init scripts +│ ├── backer # OpenRC (Alpine/Gentoo) +│ ├── backer.service # systemd +│ └── backer.freebsd # FreeBSD rc.d +├── test_data/ # Test fixtures +├── data/config_example.json # Example config +├── Makefile # Build targets +├── TODO.org # Project backlog +├── README.md # User documentation +└── LICENSE # MIT License +``` + +## Config Options +| Option | Default | Description | +|--------|---------|-------------| +| address | 0.0.0.0 | Listen address | +| port | 8086 | Listen port | +| cert/key | - | TLS cert/key paths | +| nohttps | false | Disable HTTPS | +| location | /archive | API endpoint path | +| user | - | Basic auth username (required) | +| password | - | Basic auth password (required) | +| log | stderr | Log output | +| loglevel | info | Log verbosity (debug/info/warn/error) | +| directories | - | Directories to backup (required) | +| backup_timeout | 60 | Stream timeout in minutes (1-1440) | +| compression_level | 9 | Gzip level (1-9) | +| exclude_patterns | [] | Regex patterns to exclude from backup | + +## Key Patterns + +### Global Config +Intentional design - uses global `C` variable: +```go +var C Config // internal/backer/globals.go +``` + +### Config Parsing +HJSON requires double conversion (hjson → map → json → struct): +```go +hjson.Unmarshal(buf, &tmp) // tmp is map[string]any +json.Marshal(tmp) +json.Unmarshal(buf, &C) +``` + +### Stream Cancellation +Context-aware write and copy functions: +```go +func writeWithContext(ctx context.Context, fn func() error) error +func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error +``` + +### Pipe Error Handling +Uses `pipeErr` tracking to avoid noisy errors when reader closes early. +Skips files that can't be read (logs warning, continues backup). + +### Content-Disposition Filename +Backup filenames include timestamp: `backup-20060102-150405.tar.gz` + +### Special File Types +- **Device files** (character/block) are included with correct major/minor numbers. +- **Sockets** are skipped (no entry created). +- **Symlinks** are stored as symlinks (target path stored). +- **Named pipes** and other special files are included (tar header created). +- **Regular files** are compressed; other file types have no data section. + +### Error Recovery +- `writeWithContext` and `copyWithContext` recover from panics in callbacks and return an error. + +### Performance Notes +- Exclude patterns are precompiled after config load for faster matching. +- Compression level is configurable (1-9); default 9 uses more CPU. +- Large directory walks use `filepath.WalkDir` (efficient). + +## Coding Style & Conventions + +### Function Naming +```go +// Exported (public): VerbNoun pattern +NewServer() +LoadConfig() +CreateTarGzStream() +GetFilesFromDirectories() + +// Unexported (private): verbNoun or isNoun pattern +writeWithContext() +copyWithContext() +isPipeClosedError() // predicate helper +defaultConfigPath() +``` + +### JSON Tags +Struct fields use PascalCase, json tags use snake_case: +```go +BackupTimeout int `json:"backup_timeout"` +CompressionLevel int `json:"compression_level"` +``` + +### Error Messages +- Start with capital letter (ST1005 disabled) +- Use past tense: `"Failed to..."`, `"Unable to..."` +- Include context: `fmt.Errorf("Failed to close %s: %w", name, err)` +- Use `%v` for errors (not `%s`) + +### Logging +- Past tense: `"Failed to close gzip writer: %v", err` +- Context included: `"Skipping %s: %v", filename, err` +- Appropriate level: Warn for recoverable, Error for issues, Fatal for startup failures + +### Config Validation Pattern +For each config option: set default first, then immediately validate: +```go +// Port: set default, then validate range. +if C.Port == 0 { + C.Port = 8086 + log.Warnf("Config option port is not set, fallback to %d", C.Port) +} + +if C.Port < 1 || C.Port > 65535 { + return fmt.Errorf("Port must be between 1 and 65535, got %d", C.Port) +} +``` + +### Go Formatting +- Tabs for indentation +- Vim modeline at EOF: `/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */` + +### Linter Suppressions (intentional) +| Rule | Reason | +|------|--------| +| ST1005 | Capitalized error messages for readability | +| QF1001 | De Morgan's law disabled for auth condition clarity | +| revive (error strings) | Same reason as ST1005 | + +### Comment Style +- Profanity allowed (pet project) +- Use `//nolint:` only when absolutely necessary + +## Testing +```bash +go test ./... # Run tests +go test -v -run TestName ./... # Run specific test +golangci-lint run ./... # Lint +``` + +Coverage: ~69% on internal/backer (main and log packages not covered by design) + +### Test Conventions +- Reset global config: `C = Config{}` +- Use `t.TempDir()` for temporary files +- Helper function `writeTempConfig()` for config tests + +## Build +```bash +make # Build binary (version from git or "dev") +make build VERSION=0.1.0 # Build with specific version +make clean # Remove binary +make test # Run tests +make upgrade # Update dependencies +make help # Show targets +``` + +### Versioning +Version is injected at build time via ldflags: +```go +var version = "dev" // overridden by: -X main.version=$(VERSION) +``` + +## Known Issues / TODO +See TODO.org for current backlog: +- Consider different compression algorithms (bz2, xz, lzo, lz4) +- Consider configurable filename prefix for content-disposition diff --git a/test_data/example.crt b/test_data/example.crt new file mode 100644 index 0000000..3aa671e --- /dev/null +++ b/test_data/example.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID9zCCAt+gAwIBAgIUMYNatgQoPFvsrlyRU03P7mcWtA8wDQYJKoZIhvcNAQEL +BQAwgYkxCzAJBgNVBAYTAlJVMQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAcMBk1v +c2NvdzEYMBYGA1UECgwPSG9tZSBTd2VldCBIb21lMQswCQYDVQQLDAJJVDESMBAG +A1UEAwwJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5yb290QGxvY2FsaG9zdDAg +Fw0yNjAzMjQwODU3NDJaGA8yMTI2MDIyODA4NTc0MlowgYkxCzAJBgNVBAYTAlJV +MQ8wDQYDVQQIDAZNb3Njb3cxDzANBgNVBAcMBk1vc2NvdzEYMBYGA1UECgwPSG9t +ZSBTd2VldCBIb21lMQswCQYDVQQLDAJJVDESMBAGA1UEAwwJbG9jYWxob3N0MR0w +GwYJKoZIhvcNAQkBFg5yb290QGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBANxz73cHg+Hl/o+t023tafuYn9KgJGhwnjhtx1UYdZtE9lA8 +Y3Ae3VCgTZIdsTq7F0DmCYO/Bt8q7E+lAInQMzSwANJ2h97nKOQD7SlZFPZb7Pka +jjM7u/MOv944exyfu36b3qCk5YkqHOBTO+pJx3RLcXWhkZKn3DJafrZCa9N3Wwj1 +XTCtfbnnSWD3I/7F9JOLNJS6ZfPzDjgyPfKRKN5rs9ZVZq2t6OOlrC6TwhBOuC9U +GXnCM+rBM4c96EBv50ep9JbbrppfXB9jWxMgssAVIpvLaMRvVkNbWO+W3zsRDr5H +SyeVikz/kQ9EfCkWg8YiUU5VUSRzPKbR5S4H4uUCAwEAAaNTMFEwHQYDVR0OBBYE +FCiFjFTNly/CXi8BPhOymoOwBk6mMB8GA1UdIwQYMBaAFCiFjFTNly/CXi8BPhOy +moOwBk6mMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAIy0z5SY +AC4x46J/EGXe4OUmVkwyHkcgW7JNjcOrN1N9ijnuhNVAe2afvGTV0SV2yInY5pXB +LajbHoFBRAikvEmtPaCQXc7rHN51ywDLFsie/6Xr3if59I5MH+YukWZzwCN8PIFM ++PDF3cLT+Kae+66LLpUTn5X8mGQ7wx4qrpr8sxXj621+hz1FR18sSUZt9/OPJoQM +cIoaWDp3LfHEdQ8zfe/9V/1g4+CkpDstUDgFsS0LhFioY+9jtneglEMSu8kN2l3T +Lnzpt1tDnlxQm80EfA4yK5AmpeOq8XpBVxEDlvg7GdB1MTvUsezM8CoUCUrzRrL8 +ZMrJeUrD84DrZGk= +-----END CERTIFICATE----- diff --git a/test_data/example.key b/test_data/example.key new file mode 100644 index 0000000..af9692d --- /dev/null +++ b/test_data/example.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDcc+93B4Ph5f6P +rdNt7Wn7mJ/SoCRocJ44bcdVGHWbRPZQPGNwHt1QoE2SHbE6uxdA5gmDvwbfKuxP +pQCJ0DM0sADSdofe5yjkA+0pWRT2W+z5Go4zO7vzDr/eOHscn7t+m96gpOWJKhzg +UzvqScd0S3F1oZGSp9wyWn62QmvTd1sI9V0wrX2550lg9yP+xfSTizSUumXz8w44 +Mj3ykSjea7PWVWatrejjpawuk8IQTrgvVBl5wjPqwTOHPehAb+dHqfSW266aX1wf +Y1sTILLAFSKby2jEb1ZDW1jvlt87EQ6+R0snlYpM/5EPRHwpFoPGIlFOVVEkczym +0eUuB+LlAgMBAAECggEALuI6JIiU3+tDuOdJLpj9/+u827TCoXKH96zDJoIR6vBe +6P/6ztrFixLYhHFronetEZiKn/n1mmscwDBxfZ2pcjRXLGtYaIulSt7Z5oADQQSa +UHz5DoTO6zlxDS/aYONIovu/vE7tYP8zjnKtLJ29vSoZSmDO3o9z+UJ9LcJQ/7L+ +uU+HLGDiiWlCIfVPaF73cVF+sX5GCpjzhS4A2fxn+O0X1wBXmP2hDC1cb3kP8aL+ +RuBniiEKuWmV8l1SLfVBbdOwla/BCv0TyiWiPbIluDmIPst6IScAbjrx4hyh0Wyc +P04ZQDom2NyqHZqJD4qeih2I/4Ly9MlQ8Lp2DtI8GQKBgQDx/rzwN3Zv7KWJwNVJ +B/N4IiAxx2y294+hnceJgfS9XLAutu/ToomhmAdi0ZUjC508NSQLWO6HlcmU69hR +AydBR1aMd32RpAmO3QNjqQcZjsCXhWrChnInNOEW6Xc5azCo1Gjpt3CgDYR1mBUm +i/EhpP8mbdfj/d34sSxV79YhjQKBgQDpNgpVQPT87QgVVTsp2Bm5C6sfQNW8FY5x +bUPHeSkwNyCo4wUUW4lC/LkPP1AeaACW8Dvd8NcvHPRvdqS0c8IPFYrkQZRwjDl+ +F3vdLgW3nuC8zUolaTNQyLPBQiJZV80U5rLTcDziEwEJvsvq/o8JHeYUf/90SpLT +3y0Qcp00uQKBgHSb57dpQm0Uf4Atp5kVd8C/0PuruTtBd7jNKsWcFa7Z5xvbkx1M +CT4Q4B5CF7dXdWD5ynCdXfN/vi9ApnADMPtHwPDW828muGRbI7L2VDGAlFVvOM6b +so4p8Pz1azoD6OKx2AwnFOJLITT5H9f/PyaTN0/xRkLapgeceFzAwVJRAoGARN3x +dTWGYv44CdyuaO/owK1Cbis+MpmHA9XJUyljUVqmKVlI0bcqNH7ewzLmHOSHcAsn +EqYhVUdOo2+8uZEJeFzbNAZtqEKwY5zZtfn2NEAVi4tv3tZPEgcClgpKjGmSJR7v +JeMaB4SnJNKT8yQS5xVOMTWPIS84Zpqb+yRtafkCgYA+b4JSnBUtDsHIaLbP6Y4N +yd7SneucwuJBF9d8n+fzWcDMDLi/JYtSdgSlMPMruv0CeXNTZyEqQUAKzbpd1/eQ +0hXmPqoyq8mV+qwo5avEddPbkKh7oPCaqsM7VfB1jwFXWZCiOoXk0mOQ3+Miv6hJ +t2A0YvyV9XdFvaxGMg0Ajg== +-----END PRIVATE KEY----- diff --git a/test_data/test1/bar/goodbye.txt/text.txt b/test_data/test1/bar/goodbye.txt/text.txt new file mode 100644 index 0000000..befe075 --- /dev/null +++ b/test_data/test1/bar/goodbye.txt/text.txt @@ -0,0 +1 @@ +Fucked up beyond recognotion. diff --git a/test_data/test1/bar/hello.txt b/test_data/test1/bar/hello.txt new file mode 100644 index 0000000..e69de29 diff --git a/test_data/test1/foo/hello_breakfast.txt b/test_data/test1/foo/hello_breakfast.txt new file mode 100644 index 0000000..d033c4d --- /dev/null +++ b/test_data/test1/foo/hello_breakfast.txt @@ -0,0 +1,2 @@ +My dear Tweety, +I'd like to greet you as my breakfast. diff --git a/test_data/test1/foo/mydir/myfile.txt b/test_data/test1/foo/mydir/myfile.txt new file mode 100644 index 0000000..54c45d4 --- /dev/null +++ b/test_data/test1/foo/mydir/myfile.txt @@ -0,0 +1 @@ +This is MY file! \ No newline at end of file diff --git a/test_data/test1/foo/some_text.txt b/test_data/test1/foo/some_text.txt new file mode 100644 index 0000000..e69de29 diff --git a/test_data/test_config.json b/test_data/test_config.json new file mode 100644 index 0000000..693073c --- /dev/null +++ b/test_data/test_config.json @@ -0,0 +1,43 @@ +{ + /* Listen on other addresses is pointless. */ + "address": "127.0.0.1", + + /* By default listen on port 8086, like first intel commercially successful cpu model. */ + "port": 8086, + + /* Path location of certificate file */ + "cert": "../../test_data/example.crt", + + /* Path location of certificate key */ + "key": "../../test_data/example.key", + + /* We really want to use https for backup archive tranferring. */ + "nohttps": false, + + /* Which location we should query by our client curl, for example. */ + "location": "/archive", + + /* User for basic auth. */ + "user": "JohnnyGoode", + + /* Password for basic auth. */ + "password": "SharpShooter", + + /* Where to store logs. */ + "log": "/var/log/backer.log", + + /* Verbosity of logs */ + "loglevel": "info", + + /* Directories to backup */ + "directories": [ + "../../test_data/test1/foo", + "../../test_data/test1/bar" + ], + + /* Backup streaming timeout in minutes. */ + "backup_timeout": 60, + + /* Gzip compression level (1-9, where 9 is best compression). */ + "compression_level": 6 +}