aleesa-bot/aleesa-irc-go
0
Fork 0
Code Activity

Update dependencies

Browse Source
This commit is contained in:
eleksir
2025-10-12 13:55:23 +03:00
parent 897a2035d9
commit 3ad60b9c69
437 changed files with 26255 additions and 43246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

311
vendor/github.com/DataDog/zstd/zstd_stream.go generated vendored
View File

@@ -1,6 +1,8 @@
package zstd
/*
#define ZSTD_STATIC_LINKING_ONLY
#include "stdint.h" // for uintptr_t
#include "zstd.h"
typedef struct compressStream2_result_s {
@@ -9,10 +11,9 @@ typedef struct compressStream2_result_s {
size_t bytes_written;
} compressStream2_result;
static void ZSTD_compressStream2_wrapper(compressStream2_result* result, ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
static void ZSTD_compressStream2_wrapper(compressStream2_result* result, ZSTD_CCtx* ctx, uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize) {
ZSTD_outBuffer outBuffer = { (void*)dst, maxDstSize, 0 };
ZSTD_inBuffer inBuffer = { (void*)src, srcSize, 0 };
size_t retCode = ZSTD_compressStream2(ctx, &outBuffer, &inBuffer, ZSTD_e_continue);
result->return_code = retCode;
@@ -20,21 +21,9 @@ static void ZSTD_compressStream2_wrapper(compressStream2_result* result, ZSTD_CC
result->bytes_written = outBuffer.pos;
}
static void ZSTD_compressStream2_flush(compressStream2_result* result, ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
size_t retCode = ZSTD_compressStream2(ctx, &outBuffer, &inBuffer, ZSTD_e_flush);
result->return_code = retCode;
result->bytes_consumed = inBuffer.pos;
result->bytes_written = outBuffer.pos;
}
static void ZSTD_compressStream2_finish(compressStream2_result* result, ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
static void ZSTD_compressStream2_finish(compressStream2_result* result, ZSTD_CCtx* ctx, uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize) {
ZSTD_outBuffer outBuffer = { (void*)dst, maxDstSize, 0 };
ZSTD_inBuffer inBuffer = { (void*)src, srcSize, 0 };
size_t retCode = ZSTD_compressStream2(ctx, &outBuffer, &inBuffer, ZSTD_e_end);
result->return_code = retCode;
@@ -49,10 +38,9 @@ typedef struct decompressStream2_result_s {
size_t bytes_written;
} decompressStream2_result;
static void ZSTD_decompressStream_wrapper(decompressStream2_result* result, ZSTD_DCtx* ctx,
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
static void ZSTD_decompressStream_wrapper(decompressStream2_result* result, ZSTD_DCtx* ctx, uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize) {
ZSTD_outBuffer outBuffer = { (void*)dst, maxDstSize, 0 };
ZSTD_inBuffer inBuffer = { (void*)src, srcSize, 0 };
size_t retCode = ZSTD_decompressStream(ctx, &outBuffer, &inBuffer);
result->return_code = retCode;
@@ -72,7 +60,6 @@ import (
var errShortRead = errors.New("short read")
var errReaderClosed = errors.New("Reader is closed")
var ErrNoParallelSupport = errors.New("No parallel support")
// Writer is an io.WriteCloser that zstd-compresses its input.
type Writer struct {
@@ -80,6 +67,7 @@ type Writer struct {
ctx *C.ZSTD_CCtx
dict []byte
srcBuffer []byte
dstBuffer []byte
firstError error
underlyingWriter io.Writer
@@ -137,6 +125,7 @@ func NewWriterLevelDict(w io.Writer, level int, dict []byte) *Writer {
CompressionLevel: level,
ctx: ctx,
dict: dict,
srcBuffer: make([]byte, 0),
dstBuffer: make([]byte, CompressBound(1024)),
firstError: err,
underlyingWriter: w,
@@ -152,96 +141,66 @@ func (w *Writer) Write(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
total := len(p)
// Check if dstBuffer is enough
w.dstBuffer = w.dstBuffer[0:cap(w.dstBuffer)]
if len(w.dstBuffer) < CompressBound(len(p)) {
w.dstBuffer = make([]byte, CompressBound(len(p)))
}
dstoff := 0
consumed := 0
for len(p) > 0 {
C.ZSTD_compressStream2_wrapper(
w.resultBuffer,
w.ctx,
unsafe.Pointer(&w.dstBuffer[dstoff]),
C.size_t(len(w.dstBuffer[dstoff:])),
unsafe.Pointer(&p[0]),
C.size_t(len(p)),
)
ret := int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
// The stream is dead after this.
w.firstError = err
return 0, err
}
p = p[w.resultBuffer.bytes_consumed:]
dstoff += int(w.resultBuffer.bytes_written)
consumed += int(w.resultBuffer.bytes_consumed)
if len(p) > 0 && dstoff == len(w.dstBuffer) {
// We have bytes remaining to compress and our output buffer
// filled up. This shouldn't happen since we calculated it
// in advance using CompressBound, but we need to handle it
// in case there was some miscalculation, or the internal
// stream buffer contained enough data from previous writes
// to overflow dstBuffer. (it's not clear from the docs
// whether this is possible)
//
// Allocate space for whatever we haven't compressed yet.
newbuf := make([]byte, len(w.dstBuffer)+CompressBound(total-consumed))
copy(newbuf, w.dstBuffer)
w.dstBuffer = newbuf
// Do not do an extra memcopy if zstd ingest all input data
srcData := p
fastPath := len(w.srcBuffer) == 0
if !fastPath {
w.srcBuffer = append(w.srcBuffer, p...)
srcData = w.srcBuffer
}
srcPtr := C.uintptr_t(uintptr(0)) // Do not point anywhere, if src is empty
if len(srcData) > 0 {
srcPtr = C.uintptr_t(uintptr(unsafe.Pointer(&srcData[0])))
}
C.ZSTD_compressStream2_wrapper(
w.resultBuffer,
w.ctx,
C.uintptr_t(uintptr(unsafe.Pointer(&w.dstBuffer[0]))),
C.size_t(len(w.dstBuffer)),
srcPtr,
C.size_t(len(srcData)),
)
runtime.KeepAlive(p) // Ensure p is kept until here so pointer doesn't disappear during C call
ret := int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
return 0, err
}
consumed := int(w.resultBuffer.bytes_consumed)
if !fastPath {
w.srcBuffer = w.srcBuffer[consumed:]
} else {
remaining := len(p) - consumed
if remaining > 0 {
// We still have some non-consumed data, copy remaining data to srcBuffer
// Try to not reallocate w.srcBuffer if we already have enough space
if cap(w.srcBuffer) >= remaining {
w.srcBuffer = w.srcBuffer[0:remaining]
} else {
w.srcBuffer = make([]byte, remaining)
}
copy(w.srcBuffer, p[consumed:])
}
}
written := int(w.resultBuffer.bytes_written)
// Write to underlying buffer
_, err := w.underlyingWriter.Write(w.dstBuffer[:dstoff])
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
// Same behaviour as zlib, we can't know how much data we wrote, only
// if there was an error
if err != nil {
return 0, err
}
return total, err
}
// Flush writes any unwritten data to the underlying io.Writer.
func (w *Writer) Flush() error {
if w.firstError != nil {
return w.firstError
}
ret := 1 // So we loop at least once
for ret > 0 {
C.ZSTD_compressStream2_flush(
w.resultBuffer,
w.ctx,
unsafe.Pointer(&w.dstBuffer[0]),
C.size_t(len(w.dstBuffer)),
unsafe.Pointer(uintptr(0)),
C.size_t(0),
)
ret = int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
return err
}
written := int(w.resultBuffer.bytes_written)
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
if err != nil {
return err
}
if ret > 0 { // We have a hint if we need to resize the dstBuffer
w.dstBuffer = w.dstBuffer[:cap(w.dstBuffer)]
if len(w.dstBuffer) < ret {
w.dstBuffer = make([]byte, ret)
}
}
}
return nil
return len(p), err
}
// Close closes the Writer, flushing any unwritten data to the underlying
@@ -253,24 +212,26 @@ func (w *Writer) Close() error {
ret := 1 // So we loop at least once
for ret > 0 {
srcPtr := C.uintptr_t(uintptr(0)) // Do not point anywhere, if src is empty
if len(w.srcBuffer) > 0 {
srcPtr = C.uintptr_t(uintptr(unsafe.Pointer(&w.srcBuffer[0])))
}
C.ZSTD_compressStream2_finish(
w.resultBuffer,
w.ctx,
unsafe.Pointer(&w.dstBuffer[0]),
C.uintptr_t(uintptr(unsafe.Pointer(&w.dstBuffer[0]))),
C.size_t(len(w.dstBuffer)),
unsafe.Pointer(uintptr(0)),
C.size_t(0),
srcPtr,
C.size_t(len(w.srcBuffer)),
)
ret = int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
return err
}
w.srcBuffer = w.srcBuffer[w.resultBuffer.bytes_consumed:]
written := int(w.resultBuffer.bytes_written)
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
if err != nil {
C.ZSTD_freeCStream(w.ctx)
return err
}
w.underlyingWriter.Write(w.dstBuffer[:written])
if ret > 0 { // We have a hint if we need to resize the dstBuffer
w.dstBuffer = w.dstBuffer[:cap(w.dstBuffer)]
@@ -283,28 +244,6 @@ func (w *Writer) Close() error {
return getError(int(C.ZSTD_freeCStream(w.ctx)))
}
// Set the number of workers to run the compression in parallel using multiple threads
// If > 1, the Write() call will become asynchronous. This means data will be buffered until processed.
// If you call Write() too fast, you might incur a memory buffer up to as large as your input.
// Consider calling Flush() periodically if you need to compress a very large file that would not fit all in memory.
// By default only one worker is used.
func (w *Writer) SetNbWorkers(n int) error {
if w.firstError != nil {
return w.firstError
}
if err := getError(int(C.ZSTD_CCtx_setParameter(w.ctx, C.ZSTD_c_nbWorkers, C.int(n)))); err != nil {
w.firstError = err
// First error case, a shared libary is used, and the library was compiled without parallel support
if err.Error() == "Unsupported parameter" {
return ErrNoParallelSupport
} else {
// This could happen if a very large number is passed in, and possibly zstd refuse to create as many threads, or the OS fails to do so
return err
}
}
return nil
}
// cSize is the recommended size of reader.compressionBuffer. This func and
// invocation allow for a one-time check for validity.
var cSize = func() int {
@@ -424,87 +363,50 @@ func (r *reader) Read(p []byte) (int, error) {
return 0, r.firstError
}
if len(p) == 0 {
return 0, nil
// If we already have enough bytes, return
if r.decompSize-r.decompOff >= len(p) {
copy(p, r.decompressionBuffer[r.decompOff:])
r.decompOff += len(p)
return len(p), nil
}
// If we already have some uncompressed bytes, return without blocking
if r.decompSize > r.decompOff {
if r.decompSize-r.decompOff > len(p) {
copy(p, r.decompressionBuffer[r.decompOff:])
r.decompOff += len(p)
return len(p), nil
}
// From https://golang.org/pkg/io/#Reader
// > Read conventionally returns what is available instead of waiting for more.
copy(p, r.decompressionBuffer[r.decompOff:r.decompSize])
got := r.decompSize - r.decompOff
r.decompOff = r.decompSize
return got, nil
}
copy(p, r.decompressionBuffer[r.decompOff:r.decompSize])
got := r.decompSize - r.decompOff
r.decompSize = 0
r.decompOff = 0
// Repeatedly read from the underlying reader until we get
// at least one zstd block, so that we don't block if the
// other end has flushed a block.
for {
// - If the last decompression didn't entirely fill the decompression buffer,
// zstd flushed all it could, and needs new data. In that case, do 1 Read.
// - If the last decompression did entirely fill the decompression buffer,
// it might have needed more room to decompress the input. In that case,
// don't do any unnecessary Read that might block.
needsData := r.decompSize < len(r.decompressionBuffer)
var src []byte
if !needsData {
src = r.compressionBuffer[:r.compressionLeft]
} else {
src = r.compressionBuffer
var n int
var err error
// Read until data arrives or an error occurs.
for n == 0 && err == nil {
n, err = r.underlyingReader.Read(src[r.compressionLeft:])
}
if err != nil && err != io.EOF { // Handle underlying reader errors first
return 0, fmt.Errorf("failed to read from underlying reader: %w", err)
}
if n == 0 {
// Ideally, we'd return with ErrUnexpectedEOF in all cases where the stream was unexpectedly EOF'd
// during a block or frame, i.e. when there are incomplete, pending compression data.
// However, it's hard to detect those cases with zstd. Namely, there is no way to know the size of
// the current buffered compression data in the zstd stream internal buffers.
// Best effort: throw ErrUnexpectedEOF if we still have some pending buffered compression data that
// zstd doesn't want to accept.
// If we don't have any buffered compression data but zstd still has some in its internal buffers,
// we will return with EOF instead.
if r.compressionLeft > 0 {
return 0, io.ErrUnexpectedEOF
}
return 0, io.EOF
}
src = src[:r.compressionLeft+n]
for got < len(p) {
// Populate src
src := r.compressionBuffer
reader := r.underlyingReader
n, err := TryReadFull(reader, src[r.compressionLeft:])
if err != nil && err != errShortRead { // Handle underlying reader errors first
return 0, fmt.Errorf("failed to read from underlying reader: %s", err)
} else if n == 0 && r.compressionLeft == 0 {
return got, io.EOF
}
src = src[:r.compressionLeft+n]
// C code
var srcPtr *byte // Do not point anywhere, if src is empty
srcPtr := C.uintptr_t(uintptr(0)) // Do not point anywhere, if src is empty
if len(src) > 0 {
srcPtr = &src[0]
srcPtr = C.uintptr_t(uintptr(unsafe.Pointer(&src[0])))
}
C.ZSTD_decompressStream_wrapper(
r.resultBuffer,
r.ctx,
unsafe.Pointer(&r.decompressionBuffer[0]),
C.uintptr_t(uintptr(unsafe.Pointer(&r.decompressionBuffer[0]))),
C.size_t(len(r.decompressionBuffer)),
unsafe.Pointer(srcPtr),
srcPtr,
C.size_t(len(src)),
)
retCode := int(r.resultBuffer.return_code)
// Keep src here even though we reuse later, the code might be deleted at some point
// Keep src here eventhough we reuse later, the code might be deleted at some point
runtime.KeepAlive(src)
if err := getError(retCode); err != nil {
return 0, fmt.Errorf("failed to decompress: %w", err)
if err = getError(retCode); err != nil {
return 0, fmt.Errorf("failed to decompress: %s", err)
}
// Put everything in buffer
@@ -513,9 +415,10 @@ func (r *reader) Read(p []byte) (int, error) {
left := src[bytesConsumed:]
copy(r.compressionBuffer, left)
}
r.compressionLeft = len(src) - bytesConsumed
r.compressionLeft = len(src) - int(bytesConsumed)
r.decompSize = int(r.resultBuffer.bytes_written)
r.decompOff = copy(p, r.decompressionBuffer[:r.decompSize])
r.decompOff = copy(p[got:], r.decompressionBuffer[:r.decompSize])
got += r.decompOff
// Resize buffers
nsize := retCode // Hint for next src buffer size
@@ -527,9 +430,25 @@ func (r *reader) Read(p []byte) (int, error) {
nsize = r.compressionLeft
}
r.compressionBuffer = resize(r.compressionBuffer, nsize)
if r.decompOff > 0 {
return r.decompOff, nil
}
}
return got, nil
}
// TryReadFull reads buffer just as ReadFull does
// Here we expect that buffer may end and we do not return ErrUnexpectedEOF as ReadAtLeast does.
// We return errShortRead instead to distinguish short reads and failures.
// We cannot use ReadFull/ReadAtLeast because it masks Reader errors, such as network failures
// and causes panic instead of error.
func TryReadFull(r io.Reader, buf []byte) (n int, err error) {
for n < len(buf) && err == nil {
var nn int
nn, err = r.Read(buf[n:])
n += nn
}
if n == len(buf) && err == io.EOF {
err = nil // EOF at the end is somewhat expected
} else if err == io.EOF {
err = errShortRead
}
return
}