Implemented tar+xz compression + unit tests for it.
This commit is contained in:
@@ -142,8 +142,8 @@ func LoadConfig(path string) error {
|
||||
log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm)
|
||||
}
|
||||
|
||||
if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" {
|
||||
return fmt.Errorf("Compression_algorithm must be gzip, bzip2, zstd or lz4, got %s", C.CompressionAlgorithm)
|
||||
if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" && C.CompressionAlgorithm != "xz" {
|
||||
return fmt.Errorf("Compression_algorithm must be gzip, bzip2, zstd, lz4 or xz, got %s", C.CompressionAlgorithm)
|
||||
}
|
||||
|
||||
// Compile exclude patterns.
|
||||
|
||||
@@ -295,6 +295,27 @@ func TestLoadConfigCompressionAlgorithmLz4(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadConfigCompressionAlgorithmXz tests xz compression algorithm.
|
||||
func TestLoadConfigCompressionAlgorithmXz(t *testing.T) {
|
||||
config := `{
|
||||
"user": "test",
|
||||
"password": "test",
|
||||
"directories": ["../../test_data/test1/foo"],
|
||||
"nohttps": true,
|
||||
"compression_algorithm": "xz"
|
||||
}`
|
||||
tmpFile := writeTempConfig(t, config)
|
||||
|
||||
err := LoadConfig(tmpFile)
|
||||
if err != nil {
|
||||
t.Errorf("Expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if C.CompressionAlgorithm != "xz" {
|
||||
t.Errorf("C.CompressionAlgorithm should be xz, got %s", C.CompressionAlgorithm)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadConfigCompressionAlgorithmInvalid tests invalid compression algorithm.
|
||||
func TestLoadConfigCompressionAlgorithmInvalid(t *testing.T) {
|
||||
config := `{
|
||||
|
||||
@@ -91,6 +91,10 @@ func NewServer(configPath string) (*http.Server, error) {
|
||||
archive = CreateTarLz4Stream(ctx, files)
|
||||
extension = "tar.lz4"
|
||||
|
||||
case "xz":
|
||||
archive = CreateTarXzStream(ctx, files)
|
||||
extension = "tar.xz"
|
||||
|
||||
default:
|
||||
archive = CreateTarGzStream(ctx, files)
|
||||
extension = "tar.gz"
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/dsnet/compress/bzip2"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/pierrec/lz4"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// CreateTarGzStream takes a list of files and returns a reader that reads from a tar archive containing these files.
|
||||
@@ -590,6 +591,145 @@ func CreateTarLz4Stream(ctx context.Context, filepaths []string) io.ReadCloser {
|
||||
return pr
|
||||
}
|
||||
|
||||
// CreateTarXzStream takes a list of files and returns a reader that reads from an xz-compressed tar archive.
|
||||
func CreateTarXzStream(ctx context.Context, filepaths []string) io.ReadCloser {
|
||||
pr, pw := io.Pipe()
|
||||
|
||||
go func() {
|
||||
var pipeErr error
|
||||
|
||||
closePipeWithError := func(err error) {
|
||||
if pipeErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
pipeErr = err
|
||||
pw.CloseWithError(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if pipeErr == nil {
|
||||
pw.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
xw, err := xz.NewWriter(pw)
|
||||
if err != nil {
|
||||
closePipeWithError(fmt.Errorf("xz writer error: %w", err))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if pipeErr == nil {
|
||||
if err := xw.Close(); err != nil {
|
||||
if !isPipeClosedError(err) {
|
||||
log.Errorf("Failed to close xz writer: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
tw := tar.NewWriter(xw)
|
||||
|
||||
defer func() {
|
||||
if pipeErr == nil {
|
||||
if err := tw.Close(); err != nil {
|
||||
if !isPipeClosedError(err) {
|
||||
log.Errorf("Failed to close tar writer: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for _, fpath := range filepaths {
|
||||
var (
|
||||
err error
|
||||
linkTarget string
|
||||
)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
closePipeWithError(ctx.Err())
|
||||
|
||||
return
|
||||
|
||||
default:
|
||||
}
|
||||
|
||||
log.Debugf("Adding file: %s", fpath)
|
||||
|
||||
st, err := os.Lstat(fpath)
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Skipping %s: %v", fpath, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if st.Mode()&os.ModeSymlink != 0 {
|
||||
linkTarget, err = os.Readlink(fpath)
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Skipping %s: readlink failed: %v", fpath, err)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if st.Mode()&os.ModeSocket != 0 {
|
||||
log.Debugf("Skipping socket: %s", fpath)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(st, linkTarget)
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Skipping %s: tar header failed: %v", fpath, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
header.Format = tar.FormatGNU
|
||||
header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/")
|
||||
|
||||
mode := st.Mode()
|
||||
|
||||
if err := writeWithContext(ctx, func() error {
|
||||
return tw.WriteHeader(header)
|
||||
}); err != nil {
|
||||
closePipeWithError(err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if mode.IsRegular() {
|
||||
f, err := os.Open(fpath)
|
||||
if err != nil {
|
||||
log.Warnf("Skipping %s: open failed: %v", fpath, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
err = copyWithContext(ctx, tw, f)
|
||||
|
||||
if e := f.Close(); e != nil {
|
||||
log.Warnf("Failed to close file %s: %v", fpath, e)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Skipping %s: copy failed: %v", fpath, err)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return pr
|
||||
}
|
||||
|
||||
// isPipeClosedError checks if the error is due to a closed pipe.
|
||||
func isPipeClosedError(err error) bool {
|
||||
if err == nil {
|
||||
|
||||
Reference in New Issue
Block a user