Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
f622330f41 | ||
|
|
3ad60b9c69 | ||
|
|
897a2035d9 | ||
|
|
351695a7c5 | ||
|
|
d527c4ab8f | ||
|
|
79840796f6 | ||
|
|
24507ed6b4 | ||
|
|
484bfa9ba0 | ||
|
|
8f8073ca3e | ||
|
|
7bcc407857 | ||
|
|
8c24a1d8c6 | ||
|
|
4ca2fffb33 | ||
|
|
d198741b37 | ||
|
|
afe6e5384d | ||
|
|
b6f8cf9155 | ||
|
|
9bac4fb257 | ||
|
|
7958db9c0d | ||
|
|
4869c146fe |
9
.gitignore
vendored
9
.gitignore
vendored
@@ -1,6 +1,3 @@
|
||||
.idea/**
|
||||
.vscode/**
|
||||
.vstags
|
||||
data/config.json
|
||||
data/*_db
|
||||
aleesa-irc-go
|
||||
/data/config.json
|
||||
/data/*_db
|
||||
/aleesa-irc-go
|
||||
|
||||
83
.golangci.yml
Normal file
83
.golangci.yml
Normal file
@@ -0,0 +1,83 @@
|
||||
version: "2"
|
||||
run:
|
||||
go: "1.21"
|
||||
modules-download-mode: readonly
|
||||
tests: false
|
||||
allow-parallel-runners: true
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- asasalint
|
||||
- asciicheck
|
||||
- bodyclose
|
||||
- containedctx
|
||||
- contextcheck
|
||||
- decorder
|
||||
- durationcheck
|
||||
- errcheck
|
||||
- errchkjson
|
||||
- errorlint
|
||||
- exhaustive
|
||||
- ginkgolinter
|
||||
- gocheckcompilerdirectives
|
||||
- gocritic
|
||||
- godot
|
||||
- gosec
|
||||
- gosmopolitan
|
||||
- govet
|
||||
- grouper
|
||||
- importas
|
||||
- ineffassign
|
||||
- ireturn
|
||||
- makezero
|
||||
- mirror
|
||||
- nakedret
|
||||
- nilerr
|
||||
- nilnil
|
||||
- nlreturn
|
||||
- noctx
|
||||
- nolintlint
|
||||
- nonamedreturns
|
||||
- predeclared
|
||||
- promlinter
|
||||
- reassign
|
||||
- revive
|
||||
- rowserrcheck
|
||||
- staticcheck
|
||||
- unconvert
|
||||
- unused
|
||||
- wastedassign
|
||||
- whitespace
|
||||
- wsl
|
||||
settings:
|
||||
gosec:
|
||||
excludes:
|
||||
# Оно не используется для повышения безопасности или каких-то таких секъюрных вещей, оно используется для выборки
|
||||
# произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью.
|
||||
- G404
|
||||
# Не еби мозги. Сертификаты для локалхоста или для локальной сети могут быть невалидными, если ты сможешь
|
||||
# выписать валидный серт для локалхоста, тебе миллионы разработчиков поставят золотой памятник. Просто потому
|
||||
# что эти пидоры-безпасники всрали всё что только можно всрать своими дебильными политиками.
|
||||
- G402
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- comments
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
- goimports
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
- vendor$
|
||||
12
DESIGN
Normal file
12
DESIGN
Normal file
@@ -0,0 +1,12 @@
|
||||
Для irc admin-модуля:
|
||||
|
||||
1. берём pebbledb как бэкэнд сохранения настроек.
|
||||
|
||||
2. имплементировать trusted-участников чятика для настройки бота под конкретный канальчик.
|
||||
Есть два варианта:
|
||||
- ручной список в конфиге, где прописывается owner бота и доверенные лица (опционально доверенных можно прописывать в pebbledb admin_db)
|
||||
- доверенными являются op-ы
|
||||
|
||||
Первый вариант - довольно гибкий, но у нас всегда bot owner является доверенным. Не то чтобы это неправильно. Но с другой стороны можно сделать owner-а суперадмином, а другим админам дать возможность добавлять user-ов для своего канальчика в список доверенных. И запретить удалять себя из списка доверенных.
|
||||
|
||||
Второй вариант проще. И выглядит более привычным в текущих реалиях. Но есть нюанс - у нас нету простой возможности вынуть список участников вместе с флагами (предположительно). Кроме того, необходимо писать обработчк ивентов для всех изменений состояний юзеров (join/part и op/deop) для поддержания списка участников в актуальном состоянии.
|
||||
79
Makefile
79
Makefile
@@ -1,20 +1,87 @@
|
||||
#!/usr/bin/env gmake -f
|
||||
|
||||
GOOPTS=CGO_ENABLED=0
|
||||
BUILDOPTS=-ldflags="-s -w" -a -gcflags=all=-l -trimpath
|
||||
BUILDOPTS=-ldflags="-s -w" -a -gcflags=all=-l -trimpath -buildvcs=false
|
||||
MYNAME=aleesa-irc-go
|
||||
BINARY=${MYNAME}
|
||||
UNIX_BINARY=${MYNAME}
|
||||
WINDOWS_BINARY=${MYNAME}.exe
|
||||
RMCMD=rm -rf
|
||||
|
||||
# На windows имя бинарника может зависеть не только от платформы, но и от выбранной цели, для linux-а суффикс .exe
|
||||
# не нужен
|
||||
ifeq ($(OS),Windows_NT)
|
||||
ifdef GOOS
|
||||
ifeq ($(GOOS),windows)
|
||||
BINARY=${WINDOWS_BINARY}
|
||||
else # not ifeq ($(GOOS),windows)
|
||||
BINARY=${MYNAME}
|
||||
endif # ifeq ($(GOOS),windows)
|
||||
else # not ifdef GOOS
|
||||
BINARY=${WINDOWS_BINARY}
|
||||
endif # ifdef GOOS
|
||||
ifeq ($(SHELL), sh.exe)
|
||||
RMCMD=DEL /Q /F
|
||||
endif
|
||||
endif
|
||||
|
||||
# Явно определяем символ новой строки, чтобы избежать неоднозначности на windows
|
||||
define IFS
|
||||
|
||||
endef
|
||||
|
||||
|
||||
all: clean build
|
||||
|
||||
|
||||
build:
|
||||
${GOOPTS} go build ${BUILDOPTS} -o aleesa-irc-go types.go globals.go collection.go irc-user-mode.go \
|
||||
settings-db-util.go util.go msg-parsers.go aleesa-irc-go-lib.go main.go
|
||||
ifeq ($(OS),Windows_NT)
|
||||
# Looks like on windows gnu make explicitly set SHELL to sh.exe, if it was not set.
|
||||
ifeq ($(SHELL), sh.exe)
|
||||
# # Vanilla cmd.exe / powershell.
|
||||
SET "CGO_ENABLED=0"
|
||||
go build ${BUILDOPTS} -o ${BINARY} ./cmd/${MYNAME}
|
||||
else ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
|
||||
# # git-bash
|
||||
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY} ./cmd/${MYNAME}
|
||||
else # not ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
|
||||
# # Some other shell.
|
||||
# # TODO: handle it.
|
||||
$(info "-- Dunno how to handle this shell: ${SHELL}")
|
||||
endif # ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
|
||||
else # not ($(OS),Windows_NT)
|
||||
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY} ./cmd/${MYNAME}
|
||||
endif # ifeq ($(OS),Windows_NT)
|
||||
|
||||
|
||||
clean:
|
||||
go clean
|
||||
ifeq ($(OS),Windows_NT)
|
||||
ifeq ($(SHELL),sh.exe)
|
||||
# # Vanilla cmd.exe / powershell.
|
||||
if exist ${WINDOWS_BINARY} ${RMCMD} ${WINDOWS_BINARY}
|
||||
if exist ${UNIX_BINARY} ${RMCMD} ${UNIX_BINARY}
|
||||
else # not ifeq ($(SHELL),sh.exe)
|
||||
${RMCMD} ./${WINDOWS_BINARY}
|
||||
${RMCMD} ./${UNIX_BINARY}
|
||||
endif # ifeq ($(SHELL),sh.exe)
|
||||
else # not ifeq ($(OS),Windows_NT)
|
||||
${RMCMD} ./${BINARY}
|
||||
endif
|
||||
|
||||
|
||||
upgrade:
|
||||
rm -rf vendor
|
||||
go get -d -u -t ./...
|
||||
ifeq ($(OS),Windows_NT)
|
||||
ifeq ($(SHELL),sh.exe)
|
||||
# # Vanilla cmd.exe / powershell.
|
||||
if exist vendor DEL /F /S /Q vendor >nul
|
||||
else # not ifeq ($(SHELL),sh.exe)
|
||||
# # git-bash
|
||||
$(RM) -r vendor
|
||||
endif # ifeq ($(SHELL),sh.exe)
|
||||
else # not ifeq ($(OS),Windows_NT)
|
||||
$(RM) -r vendor
|
||||
endif
|
||||
go get -u ./...
|
||||
go mod tidy
|
||||
go mod vendor
|
||||
|
||||
|
||||
82
README
82
README
@@ -1,82 +0,0 @@
|
||||
aleesa-irc-go
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Сервис для aleesa-bot, работающий с протоколом IRC. С основными сервисами бота работает по протоколу Redis pubsub.
|
||||
|
||||
|
||||
Механика работы
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Сервис подписан на pub-sub redis-канал irc. В этот канал пишут остальные сервисы aleesa-bot-а в ответ на запросы этого
|
||||
сервиса.
|
||||
|
||||
Формат протокола общения по redis pub-sub - см README от aleesa-misc-go.
|
||||
|
||||
|
||||
Как это собрать?
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Понадобится golang версии 1.19 или более новая.
|
||||
|
||||
Сам процесс сборки простой, достаточно запустить:
|
||||
make
|
||||
|
||||
и через некоторое время мы получаем бинарник aleesa-irc-go. Это и есть наше приложение.
|
||||
|
||||
|
||||
Как это запустить?
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Приложение тестировалось на alpine linux x86_64 3.16, но должно работать на любом (более свежем) linux-е.
|
||||
|
||||
После создания data/config.json из data/config_sample.json, можно запускать приложение.
|
||||
|
||||
Предполагается, что конфиг написан в формате human json (https://hjson.github.io), который допускает коментарии и
|
||||
некоторые другие вольности в форматировании json-а. Конфиг ищется последовательно по следующим путям по принципу "кто
|
||||
первый того и тапки":
|
||||
~/.aleesa-irc-go.json
|
||||
~/aleesa-irc-go.json
|
||||
/etc/aleesa-irc-go.json
|
||||
data/config.json
|
||||
|
||||
В каталоге contrib находится скрипт для alpine linux, системы инициализации openrc. Этот скрипт достаточно положить в
|
||||
/etc/init.d/aleesa-irc-go. После чего скопировать бинарник в каталог /var/lib/aleesa-irc-go, положить конфиг в одну
|
||||
из дефолтных локаций (тестировалась /var/lib/aleesa-irc-go/data/config.json) и после этого сервис запускается
|
||||
командой:
|
||||
service aleesa-irc-go start
|
||||
|
||||
В автозагрузку его можно прописать через команду:
|
||||
rc-update add aleesa-irc-go default
|
||||
|
||||
|
||||
Nota Bene
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
По механике, в процессе работы мы форкаем горутинки на каждое входящее сообщение, особо не контроллируя этот процесс.
|
||||
Предполагается, что на это приложение не будет большой нагрузки - ограничения irc не должны дать возможности развить
|
||||
достаточно большой message rate, чтобы об это споткнуться.
|
||||
|
||||
Формально, более удачным было бы иметь автоматический пул воркеров-горутинок и раскидывать им через channel-ы
|
||||
сообщения. Внутри каждой горутинки можно было бы повиcнуть в бесконечном цикле на канале входящих сообщений и так и
|
||||
работать. Но на практике оно того не стОит.
|
||||
|
||||
|
||||
Nota Bene #2
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Go не поддерживает системный вызов fork() из-за чего демонизация программ на гошке средствами самой гошки - это в
|
||||
некотором роде проблема. Решается она с помощью утилиты daemonize (https://github.com/bmc/daemonize).
|
||||
|
||||
Соответствено, если для запуска этой программы предполагается использовать возможности демонизации, то можно
|
||||
воспользоваться этой программой.
|
||||
|
||||
В alpine linux есть средства openrc для супервайзинга foreground-сервисов.
|
||||
|
||||
|
||||
Nota Bene #3
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
К сожалению, в боте не подразумевается функциональность invite-ов. Причина банальна - слишком жёсткие ограничения у
|
||||
многих irc-сетей на количество сообщений в единицу времени, а по механике в большинстве случаев ограничение работает не
|
||||
client-client или client-channel, а глобально, client-server (за исключением случаев, когда на канале у бота есть +v
|
||||
или +o), что позволяет абузить клиента, то есть в данном случае, бота.
|
||||
|
||||
Сообщения об invite-ах пишутся в лог с уровнем info.
|
||||
|
||||
|
||||
Special Thanks
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
https://github.com/akyoto/cache за "простой" кэш, из которого я сделал collection.go
|
||||
83
README.md
Normal file
83
README.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# aleesa-irc-go
|
||||
|
||||
Сервис для aleesa-bot, работающий с протоколом IRC. С основными сервисами бота работает по протоколу Redis pubsub.
|
||||
|
||||
## Механика работы
|
||||
|
||||
Сервис подписан на pub-sub redis-канал irc. В этот канал пишут остальные сервисы aleesa-bot-а в ответ на запросы этого
|
||||
сервиса.
|
||||
|
||||
Формат протокола общения по redis pub-sub - см. документацию в репозитории aleesa-doc.
|
||||
|
||||
## Как это собрать?
|
||||
|
||||
Понадобится компилятор go версии 1.25.1 или более новый.
|
||||
|
||||
Сам процесс сборки простой, достаточно запустить make или gmake, в зависимости от платформы:
|
||||
|
||||
```bash
|
||||
make
|
||||
```
|
||||
|
||||
и через некоторое время мы получаем бинарник **aleesa-irc-go**. Это и есть наше приложение.
|
||||
|
||||
|
||||
## Как это запустить?
|
||||
|
||||
Приложение тестировалось на alpine linux x86_64 3.16, но должно работать на любом (более свежем) linux-е, windows-е,
|
||||
freebsd, macos.
|
||||
|
||||
После создания **data/config.json** из **data/config_sample.json**, можно запускать приложение.
|
||||
|
||||
Предполагается, что конфиг написан в формате [human json](https://hjson.github.io), который допускает коментарии и
|
||||
некоторые другие вольности в форматировании json-а. Конфиг ищется последовательно по следующим путям по принципу "кто
|
||||
первый того и тапки":
|
||||
|
||||
```
|
||||
~/.aleesa-irc-go.json
|
||||
~/aleesa-irc-go.json
|
||||
/etc/aleesa-irc-go.json
|
||||
data/config.json
|
||||
```
|
||||
|
||||
В каталоге contrib находится скрипт для alpine linux, системы инициализации openrc. Этот скрипт достаточно положить в
|
||||
**/etc/init.d/aleesa-irc-go**. После чего скопировать бинарник в каталог **/var/lib/aleesa-irc-go**, положить конфиг в
|
||||
одну из дефолтных локаций (тестировалась **/var/lib/aleesa-irc-go/data/config.json**) и после этого сервис запускается
|
||||
командой:
|
||||
|
||||
```bash
|
||||
service aleesa-irc-go start
|
||||
```
|
||||
|
||||
В автозагрузку его можно прописать через команду:
|
||||
|
||||
```bash
|
||||
rc-update add aleesa-irc-go default
|
||||
```
|
||||
|
||||
## Nota Bene
|
||||
|
||||
Go не поддерживает системный вызов fork() из-за чего демонизация программ на гошке средствами самой гошки - это в
|
||||
некотором роде проблема. Решается она с помощью утилиты [daemonize](https://github.com/bmc/daemonize).
|
||||
|
||||
Соответствено, если для запуска этой программы предполагается использовать возможности демонизации, то можно
|
||||
воспользоваться этой программой.
|
||||
|
||||
В alpine linux есть средства openrc для супервайзинга foreground-сервисов.
|
||||
|
||||
Технически, можно было бы решить эту проблему fork()-а с помощью "github.com/sevlyar/go-daemon", но на текущий момент
|
||||
она не стоит остро, поэтому решения не требуется.
|
||||
|
||||
## Nota Bene 2
|
||||
|
||||
К сожалению, в боте не подразумевается функциональность invite-ов. Причина банальна - слишком жёсткие ограничения у
|
||||
многих irc-сетей на количество сообщений в единицу времени, а по механике в большинстве случаев ограничение работает не
|
||||
client-client или client-channel, а глобально, client-server (за исключением случаев, когда на канале у бота есть +v
|
||||
или +o), что позволяет абузить клиента, то есть в данном случае, бота.
|
||||
|
||||
Сообщения об invite-ах пишутся в лог с уровнем info.
|
||||
|
||||
|
||||
## Special Thanks
|
||||
|
||||
https://github.com/akyoto/cache за "простой" кэш, из которого я сделал anycollection и boolcollection.
|
||||
89
cmd/aleesa-irc-go/main.go
Normal file
89
cmd/aleesa-irc-go/main.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"aleesa-irc-go/internal/irc"
|
||||
"aleesa-irc-go/internal/log"
|
||||
"aleesa-irc-go/internal/setting"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
)
|
||||
|
||||
// Производит некоторую инициализацию перед запуском main().
|
||||
func init() {
|
||||
irc.ReadConfig()
|
||||
|
||||
// Проинициализируем мапку MODE-ами пользователей произвольным значением и удалим его
|
||||
var usersModes = map[string]map[string]map[string]bool{
|
||||
"user": {
|
||||
"mode": {
|
||||
"v": true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
irc.UserMode.Set("channel", usersModes)
|
||||
irc.UserMode.Delete("channel")
|
||||
|
||||
setting.DataDir = irc.Config.DataDir
|
||||
}
|
||||
|
||||
// Собственно, какбэ "точка входа" - основная процедура в нашем боте.
|
||||
func main() {
|
||||
// Main context
|
||||
var (
|
||||
ctx = context.Background()
|
||||
logfile *os.File
|
||||
err error
|
||||
)
|
||||
// Откроем лог и скормим его логгеру
|
||||
if irc.Config.Log != "" {
|
||||
logfile, err = os.OpenFile(irc.Config.Log, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("Unable to open log file %s: %s", irc.Config.Log, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
} else {
|
||||
logfile = os.Stderr
|
||||
}
|
||||
|
||||
log.Init(irc.Config.Loglevel, logfile)
|
||||
|
||||
// Иницализируем redis-клиента
|
||||
irc.RedisClient = redis.NewClient(&redis.Options{
|
||||
Addr: fmt.Sprintf("%s:%d", irc.Config.Redis.Server, irc.Config.Redis.Port),
|
||||
})
|
||||
|
||||
log.Debugf("Lazy connect() to redis at %s:%d", irc.Config.Redis.Server, irc.Config.Redis.Port)
|
||||
irc.Subscriber = irc.RedisClient.Subscribe(ctx, irc.Config.Redis.MyChannel)
|
||||
redisMsgChan := irc.Subscriber.Channel()
|
||||
|
||||
go irc.IrcClientRun()
|
||||
go irc.IrcSend()
|
||||
go irc.IrcSendUnrestricted()
|
||||
|
||||
// Самое время поставить траппер сигналов
|
||||
signal.Notify(irc.SigChan,
|
||||
syscall.SIGINT,
|
||||
syscall.SIGTERM,
|
||||
syscall.SIGQUIT)
|
||||
|
||||
go irc.SigHandler()
|
||||
|
||||
// Обработчик событий от редиски
|
||||
for msg := range redisMsgChan {
|
||||
if irc.Shutdown {
|
||||
continue
|
||||
}
|
||||
|
||||
irc.RedisMsgParser(msg.Payload)
|
||||
}
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
39
go.mod
39
go.mod
@@ -1,40 +1,41 @@
|
||||
module aleesa-irc-go
|
||||
|
||||
go 1.19
|
||||
go 1.25.1
|
||||
|
||||
require (
|
||||
github.com/cockroachdb/errors v1.9.0
|
||||
github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861
|
||||
github.com/cockroachdb/errors v1.12.0
|
||||
github.com/cockroachdb/pebble v1.1.5
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/hjson/hjson-go v3.3.0+incompatible
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/thoj/go-ircevent v0.0.0-20210723090443-73e444401d64
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/DataDog/zstd v1.5.2 // indirect
|
||||
github.com/DataDog/zstd v1.4.5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
|
||||
github.com/cockroachdb/redact v1.1.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/getsentry/sentry-go v0.14.0 // indirect
|
||||
github.com/getsentry/sentry-go v0.27.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/klauspost/compress v1.15.12 // indirect
|
||||
github.com/klauspost/compress v1.16.0 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/prometheus/client_golang v1.13.1 // indirect
|
||||
github.com/prometheus/client_golang v1.15.0 // indirect
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.8.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.9.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.9.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect
|
||||
golang.org/x/net v0.1.0 // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/text v0.4.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
|
||||
golang.org/x/net v0.38.0 // indirect
|
||||
golang.org/x/sys v0.31.0 // indirect
|
||||
golang.org/x/text v0.23.0 // indirect
|
||||
google.golang.org/protobuf v1.33.0 // indirect
|
||||
)
|
||||
|
||||
723
go.sum
723
go.sum
@@ -1,760 +1,137 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
|
||||
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
|
||||
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
|
||||
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
|
||||
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
|
||||
github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
|
||||
github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
|
||||
github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
|
||||
github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
|
||||
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
|
||||
github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI=
|
||||
github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
|
||||
github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0=
|
||||
github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88=
|
||||
github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ=
|
||||
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74=
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861 h1:YarvrMA6pv032mYkBakTCkzqKpatcAZrdIknHJv+yVg=
|
||||
github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861/go.mod h1:qf9bLis2yy1XyNYD01wvIHPabuC1STzQsvGibYVsom4=
|
||||
github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
|
||||
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
|
||||
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.12.0 h1:d7oCs6vuIMUQRVbi6jWWWEJZahLCfJpnJSVobd1/sUo=
|
||||
github.com/cockroachdb/errors v1.12.0/go.mod h1:SvzfYNNBshAVbZ8wzNc/UPK3w1vf0dKDUP41ucAIf7g=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw=
|
||||
github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo=
|
||||
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
|
||||
github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
|
||||
github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c=
|
||||
github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70=
|
||||
github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I=
|
||||
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
|
||||
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
|
||||
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI=
|
||||
github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
|
||||
github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
|
||||
github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
|
||||
github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
|
||||
github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
|
||||
github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
|
||||
github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
|
||||
github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
|
||||
github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
|
||||
github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk=
|
||||
github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=
|
||||
github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
|
||||
github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=
|
||||
github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
|
||||
github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
|
||||
github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0=
|
||||
github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
|
||||
github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
|
||||
github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
|
||||
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
|
||||
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
|
||||
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
|
||||
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
|
||||
github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
|
||||
github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM=
|
||||
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
|
||||
github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
|
||||
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
|
||||
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
|
||||
github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c=
|
||||
github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
|
||||
github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
|
||||
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
|
||||
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
|
||||
github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
|
||||
github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
|
||||
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
|
||||
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
|
||||
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
|
||||
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
|
||||
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
|
||||
github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/thoj/go-ircevent v0.0.0-20210723090443-73e444401d64 h1:l/T7dYuJEQZOwVOpjIXr1180aM9PZL/d1MnMVIxefX4=
|
||||
github.com/thoj/go-ircevent v0.0.0-20210723090443-73e444401d64/go.mod h1:Q1NAJOuRdQCqN/VIWdnaaEhV8LpeO2rtlBP7/iDJNII=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
|
||||
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
|
||||
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
|
||||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
|
||||
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
|
||||
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
|
||||
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
|
||||
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
|
||||
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
|
||||
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
|
||||
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
|
||||
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
|
||||
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
package main
|
||||
package anycollection
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Collection это sync.Map-ка, хранящая в себе произвольные итемы
|
||||
// Collection это sync.Map-ка, хранящая в себе произвольные итемы.
|
||||
type Collection struct {
|
||||
items sync.Map
|
||||
close chan struct{}
|
||||
}
|
||||
|
||||
// Item представляет собой интерфейс для хранения произвольных данных
|
||||
// Item представляет собой интерфейс для хранения произвольных данных.
|
||||
type item struct {
|
||||
data interface{}
|
||||
data any
|
||||
}
|
||||
|
||||
// NewCollection создаёт инстанс коллекции
|
||||
// NewCollection создаёт инстанс коллекции.
|
||||
func NewCollection() *Collection {
|
||||
c := &Collection{
|
||||
close: make(chan struct{}),
|
||||
@@ -24,8 +24,8 @@ func NewCollection() *Collection {
|
||||
return c
|
||||
}
|
||||
|
||||
// Get достаёт данные по заданному ключу из коллекции
|
||||
func (collection *Collection) Get(key interface{}) (interface{}, bool) {
|
||||
// Get достаёт данные по заданному ключу из коллекции.
|
||||
func (collection *Collection) Get(key any) (any, bool) {
|
||||
obj, exists := collection.items.Load(key)
|
||||
|
||||
if !exists {
|
||||
@@ -37,16 +37,16 @@ func (collection *Collection) Get(key interface{}) (interface{}, bool) {
|
||||
return item.data, true
|
||||
}
|
||||
|
||||
// Set сохраняет данные с заданным ключом в коллекцию
|
||||
func (collection *Collection) Set(key interface{}, value interface{}) {
|
||||
// Set сохраняет данные с заданным ключом в коллекцию.
|
||||
func (collection *Collection) Set(key any, value any) {
|
||||
collection.items.Store(key, item{
|
||||
data: value,
|
||||
})
|
||||
}
|
||||
|
||||
// Range применяет функцию f ко всем ключам в коллекции
|
||||
func (collection *Collection) Range(f func(key, value interface{}) bool) {
|
||||
fn := func(key, value interface{}) bool {
|
||||
// Range применяет функцию f ко всем ключам в коллекции.
|
||||
func (collection *Collection) Range(f func(key, value any) bool) {
|
||||
fn := func(key, value any) bool {
|
||||
item := value.(item)
|
||||
|
||||
return f(key, item.data)
|
||||
@@ -55,12 +55,12 @@ func (collection *Collection) Range(f func(key, value interface{}) bool) {
|
||||
collection.items.Range(fn)
|
||||
}
|
||||
|
||||
// Delete удаляет ключ и значение из коллекции данных
|
||||
func (collection *Collection) Delete(key interface{}) {
|
||||
// Delete удаляет ключ и значение из коллекции данных.
|
||||
func (collection *Collection) Delete(key any) {
|
||||
collection.items.Delete(key)
|
||||
}
|
||||
|
||||
// Close очищает и высвобождает ресурсы, занятые коллекцией
|
||||
// Close очищает и высвобождает ресурсы, занятые коллекцией.
|
||||
func (collection *Collection) Close() {
|
||||
collection.close <- struct{}{}
|
||||
collection.items = sync.Map{}
|
||||
58
internal/boolcollection/boolcollection.go
Normal file
58
internal/boolcollection/boolcollection.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package boolcollection
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Collection это sync.Map-ка, хранящая в себе итемы типа bool.
|
||||
type Collection struct {
|
||||
items sync.Map
|
||||
close chan struct{}
|
||||
}
|
||||
|
||||
// Item представляет собой интерфейс для хранения данных типа bool.
|
||||
type item struct {
|
||||
data bool
|
||||
}
|
||||
|
||||
// NewCollection создаёт инстанс коллекции.
|
||||
func NewCollection() *Collection {
|
||||
c := &Collection{
|
||||
close: make(chan struct{}),
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// Get достаёт данные по заданному ключу из коллекции.
|
||||
func (collection *Collection) Get(key string) (bool, bool) {
|
||||
obj, exists := collection.items.Load(key)
|
||||
|
||||
if !exists {
|
||||
return false, false
|
||||
}
|
||||
|
||||
item := obj.(item)
|
||||
|
||||
return item.data, true
|
||||
}
|
||||
|
||||
// Set сохраняет данные с заданным ключом в коллекцию.
|
||||
func (collection *Collection) Set(key string, value bool) {
|
||||
collection.items.Store(key, item{
|
||||
data: value,
|
||||
})
|
||||
}
|
||||
|
||||
// Delete удаляет ключ и значение из коллекции данных.
|
||||
func (collection *Collection) Delete(key string) {
|
||||
collection.items.Delete(key)
|
||||
}
|
||||
|
||||
// Close очищает и высвобождает ресурсы, занятые коллекцией.
|
||||
func (collection *Collection) Close() {
|
||||
collection.close <- struct{}{}
|
||||
collection.items = sync.Map{}
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
@@ -1,38 +1,40 @@
|
||||
package main
|
||||
package irc
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"aleesa-irc-go/internal/boolcollection"
|
||||
"aleesa-irc-go/internal/log"
|
||||
|
||||
irc "github.com/thoj/go-ircevent"
|
||||
)
|
||||
|
||||
// ircClientInit горутинка для работы с протоколом irc
|
||||
func ircClientRun() {
|
||||
for {
|
||||
if shutdown {
|
||||
// Если мы завершаем работу программы, то нам ничего обрабатывать не надо
|
||||
break
|
||||
}
|
||||
|
||||
// Иницализируем irc-клиента
|
||||
serverString := fmt.Sprintf("%s:%d", config.Irc.Server, config.Irc.Port)
|
||||
// ircClientInit горутинка для работы с протоколом irc.
|
||||
func IrcClientRun() {
|
||||
for !Shutdown {
|
||||
// Иницализируем irc-клиента.
|
||||
// TODO: make use of capabilities: https://defs.ircdocs.horse/defs/clientcaps .
|
||||
// TODO: make use of tags caps https://defs.ircdocs.horse/defs/tags .
|
||||
serverString := fmt.Sprintf("%s:%d", Config.Irc.Server, Config.Irc.Port)
|
||||
log.Debugf("Preparing to connect to %s", serverString)
|
||||
log.Debugf("Using nick %s and username %s", config.Irc.Nick, config.Irc.User)
|
||||
ircClient = irc.IRC(config.Irc.Nick, config.Irc.User)
|
||||
ircClient.RealName = config.Irc.User
|
||||
log.Debugf("Using nick %s and username %s", Config.Irc.Nick, Config.Irc.User)
|
||||
ircClient = irc.IRC(Config.Irc.Nick, Config.Irc.User)
|
||||
ircClient.RealName = Config.Irc.User
|
||||
ircClient.Version = "Aleesa Bot v4.something"
|
||||
|
||||
if config.Irc.Ssl {
|
||||
if Config.Irc.Ssl {
|
||||
log.Debug("Force use ssl for connection")
|
||||
|
||||
ircClient.UseTLS = true
|
||||
|
||||
if !config.Irc.SslVerify {
|
||||
if !Config.Irc.SslVerify {
|
||||
log.Debug("Skip server certificate validation")
|
||||
|
||||
ircClient.TLSConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
} else {
|
||||
log.Debug("Force server certificate validation")
|
||||
@@ -41,58 +43,97 @@ func ircClientRun() {
|
||||
log.Debug("Skip ssl for connection")
|
||||
}
|
||||
|
||||
if config.Irc.Sasl && config.Irc.Password != "" {
|
||||
ircClient.SASLLogin = config.Irc.User
|
||||
ircClient.SASLPassword = config.Irc.Password
|
||||
if Config.Irc.Sasl && Config.Irc.Password != "" {
|
||||
ircClient.SASLLogin = Config.Irc.User
|
||||
ircClient.SASLPassword = Config.Irc.Password
|
||||
ircClient.UseSASL = true
|
||||
}
|
||||
|
||||
// Навесим коллбэков на некоторые ответы сервера на наши запросы
|
||||
// Навесим коллбэков на некоторые ответы сервера на наши запросы.
|
||||
|
||||
// 001 RPL_WELCOME уже есть в github.com/thoj/go-ircevent/irc_callback.go.
|
||||
|
||||
// Сделаем уже что-то полезное! Motd нам уже прислали и теперь можно авторизоваться и джойниться
|
||||
ircClient.AddCallback("004", func(e *irc.Event) {
|
||||
// Из это строки мы можем узнать, какие флаги для user MODE и channem MODE можно навешивать.
|
||||
// Как минимум, эта строка нам нужна, чтобы выяснить, можем ли мы взять себе +B, мы же бот :).
|
||||
e.Connection.Lock()
|
||||
|
||||
// Формат строки с MODE-ами https://datatracker.ietf.org/doc/html/rfc2812#section-5.1 .
|
||||
availableUserModes = boolcollection.NewCollection()
|
||||
availableChanModes = boolcollection.NewCollection()
|
||||
|
||||
log.Debugf("Add %s to list of available user modes", e.Arguments[3])
|
||||
|
||||
for _, mode := range strings.Split(e.Arguments[3], "") {
|
||||
availableUserModes.Set(mode, true)
|
||||
}
|
||||
|
||||
availableUserModes.Set("announced", true)
|
||||
|
||||
log.Debugf("Add %s to list available channel modes", e.Arguments[4])
|
||||
|
||||
for _, mode := range strings.Split(e.Arguments[4], "") {
|
||||
availableChanModes.Set(mode, true)
|
||||
}
|
||||
|
||||
availableChanModes.Set("announced", true)
|
||||
|
||||
e.Connection.Unlock()
|
||||
})
|
||||
|
||||
ircClient.AddCallback("005", func(e *irc.Event) {
|
||||
// TODO: Make use of it. https://defs.ircdocs.horse/defs/isupport
|
||||
})
|
||||
|
||||
ircClient.AddCallback("319", func(e *irc.Event) {
|
||||
/* Это одна из строк с данными, прилетающая в ответ на запрос whois на определённого юзера
|
||||
* Из этой строки нас интересует, на каких каналах пользователь op (то есть с префиксом @) или имеет voice
|
||||
* (то есть с префиксом +) чтобы внести его в свою базу mode-ов.
|
||||
*/
|
||||
log.Debugf("319 RPL_WHOISCHANNELS, %s", e.Raw)
|
||||
channelsWithModes := strings.Split(e.Arguments[2], " ")
|
||||
dstNick := e.Arguments[1]
|
||||
|
||||
for _, channelWithMode := range channelsWithModes {
|
||||
for _, mychannel := range config.Irc.Channels {
|
||||
for _, mychannel := range Config.Irc.Channels {
|
||||
tmp := regexp.MustCompile("#").Split(channelWithMode, 2)
|
||||
channel := tmp[1]
|
||||
modes := tmp[0]
|
||||
|
||||
if channel == mychannel {
|
||||
// Для каждого канала формат строго 1 из 4-х: #channel | @#channel | +#channel | @+#channel
|
||||
switch {
|
||||
case modes == "@+":
|
||||
switch modes {
|
||||
case "@+":
|
||||
userModeUpdateUser(channel, dstNick, "+o")
|
||||
userModeUpdateUser(channel, dstNick, "+v")
|
||||
case modes == "@":
|
||||
case "@":
|
||||
userModeUpdateUser(channel, dstNick, "+o")
|
||||
case modes == "+":
|
||||
case "+":
|
||||
userModeUpdateUser(channel, dstNick, "+v")
|
||||
default:
|
||||
userModeUpdateUser(channel, dstNick, "-o")
|
||||
userModeUpdateUser(channel, dstNick, "-v")
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 433 ERR_NICKNAMEINUSE уже есть в github.com/thoj/go-ircevent/irc_callback.go.
|
||||
|
||||
ircClient.AddCallback("353", func(e *irc.Event) {
|
||||
// Это одна из строк данных, прилетающая в ответ на запрос names, на канале
|
||||
log.Debugf("353 RPL_NAMREPLY, %s", e.Raw)
|
||||
e.Connection.Lock()
|
||||
namesString := e.Arguments[3]
|
||||
channel := e.Arguments[2]
|
||||
|
||||
for _, mychannel := range config.Irc.Channels {
|
||||
for _, mychannel := range Config.Irc.Channels {
|
||||
if channel == mychannel {
|
||||
for _, name := range strings.Split(namesString, " ") {
|
||||
mode := name[:1]
|
||||
|
||||
var nick string
|
||||
|
||||
if mode == "@" || mode == "+" {
|
||||
@@ -101,25 +142,58 @@ func ircClientRun() {
|
||||
nick = name
|
||||
}
|
||||
|
||||
switch {
|
||||
case mode == "@":
|
||||
switch mode {
|
||||
case "@":
|
||||
userModeUpdateUser(channel, nick, "+o")
|
||||
case mode == "+":
|
||||
case "+":
|
||||
userModeUpdateUser(channel, nick, "+v")
|
||||
default:
|
||||
userModeUpdateUser(channel, nick, "-o")
|
||||
userModeUpdateUser(channel, nick, "-v")
|
||||
}
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
e.Connection.Unlock()
|
||||
})
|
||||
|
||||
ircClient.AddCallback("352", func(e *irc.Event) {
|
||||
// Относительно бесполезная для нас команда. Не возвращает канала (в случае с ngircd), так что узнать mode
|
||||
// юзера на конкретном канале из этого невозможно. А остальные данные пока непонятно куда применять.
|
||||
log.Warnf("352 RPL_WHOREPLY, %s", e.Raw)
|
||||
// Если сервер не может прочитать MOTD, то он может вернуть 422 ERR_NOMOTD, тоде самое навесим и туда тоже.
|
||||
ircClient.AddCallback("376", func(e *irc.Event) {
|
||||
// TODO: make use of modes https://defs.ircdocs.horse/defs/usermodes
|
||||
// Если у нас есть доступный +B возьмём его себе, мы же бот.
|
||||
announced, _ := availableUserModes.Get("announced")
|
||||
botFlag, _ := availableUserModes.Get("B")
|
||||
|
||||
if announced {
|
||||
if botFlag {
|
||||
log.Info("Grabbing +B flag to mark me as bot")
|
||||
ircClient.Mode(ircClient.GetNick(), "+B")
|
||||
// TODO: Wait for it.
|
||||
} else {
|
||||
log.Info("Skip +B flag, server does not support it, noone will know that I am bot")
|
||||
}
|
||||
} else {
|
||||
// Этого по идее не должно быть.
|
||||
log.Info("Skip +B flag, server did not announce modes (yet?), noone will know that I am bot")
|
||||
}
|
||||
|
||||
if !Config.Irc.Sasl && Config.Irc.Password != "" && !nickIsUsed {
|
||||
log.Info("Identifying via NickServ")
|
||||
|
||||
message := fmt.Sprintf("identify %s %s", Config.Irc.Nick, Config.Irc.Password)
|
||||
imChan <- iMsg{ChatID: "NickServ", Text: message}
|
||||
}
|
||||
// TODO: wait until +R flag been set? could be implemented with waiting for channel message.
|
||||
|
||||
log.Debug("Trying to join to preconfigured channels")
|
||||
|
||||
for _, channel := range Config.Irc.Channels {
|
||||
log.Infof("Joining to %s channel", channel)
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
// TODO: wait for join or join error.
|
||||
})
|
||||
|
||||
// Навесим коллбэков на все возможные и невозможные error status code, которые мы можем получить и сдампим это
|
||||
@@ -157,6 +231,39 @@ func ircClientRun() {
|
||||
log.Errorf("421 ERR_UNKNOWNCOMMAND, %s", e.Raw)
|
||||
})
|
||||
|
||||
// Аналогичный коллбэк висит на 376 RPL_ENDOFMOTD.
|
||||
ircClient.AddCallback("422", func(e *irc.Event) {
|
||||
// TODO: make use of modes https://defs.ircdocs.horse/defs/usermodes
|
||||
// Если у нас есть доступный +B возьмём его себе, мы же бот.
|
||||
botFlag, ok := availableUserModes.Get("B")
|
||||
|
||||
if ok && botFlag {
|
||||
log.Info("Grabbing +B flag to mark me as bot")
|
||||
ircClient.Mode(ircClient.GetNick(), "+B")
|
||||
} else {
|
||||
if ok {
|
||||
log.Info("Skip +B flag, server does not support it, noone will know that I am bot")
|
||||
} else {
|
||||
log.Info("Skip +B flag, server did not announce modes (yet?), noone will know that I am bot")
|
||||
}
|
||||
}
|
||||
|
||||
if !Config.Irc.Sasl && Config.Irc.Password != "" && !nickIsUsed {
|
||||
log.Info("Identifying via NickServ")
|
||||
|
||||
message := fmt.Sprintf("identify %s %s", Config.Irc.Nick, Config.Irc.Password)
|
||||
imChan <- iMsg{ChatID: "NickServ", Text: message}
|
||||
}
|
||||
// TODO: wait until +R flag been set? could be implemented with waiting for channel message.
|
||||
|
||||
log.Debug("Trying to join to preconfigured channels")
|
||||
|
||||
for _, channel := range Config.Irc.Channels {
|
||||
log.Infof("Joining to %s channel", channel)
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
ircClient.AddCallback("431", func(e *irc.Event) {
|
||||
log.Errorf("431 ERR_NONICKNAMEGIVEN, %s", e.Raw)
|
||||
})
|
||||
@@ -167,9 +274,12 @@ func ircClientRun() {
|
||||
|
||||
ircClient.AddCallback("433", func(e *irc.Event) {
|
||||
log.Errorf("433 ERR_NICKNAMEINUSE, %s", e.Raw)
|
||||
|
||||
nickIsUsed = true
|
||||
time.Sleep(60 * time.Second)
|
||||
ircClient.Nick(config.Irc.Nick)
|
||||
|
||||
<-time.NewTimer(60 * time.Second).C // 1 minute sleep.
|
||||
|
||||
ircClient.Nick(Config.Irc.Nick)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("436", func(e *irc.Event) {
|
||||
@@ -179,6 +289,7 @@ func ircClientRun() {
|
||||
log.Errorf("436 ERR_NICKCOLLISION, %s", e.Raw)
|
||||
})
|
||||
|
||||
// 437: ERR_UNAVAILRESOURCE уже есть в github.com/thoj/go-ircevent/irc_callback.go.
|
||||
ircClient.AddCallback("437", func(e *irc.Event) {
|
||||
log.Errorf("437 ERR_UNAVAILRESOURCE, %s", e.Raw)
|
||||
})
|
||||
@@ -191,6 +302,16 @@ func ircClientRun() {
|
||||
log.Warnf("442 ERR_NOTONCHANNEL, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("443", func(e *irc.Event) {
|
||||
// Returned when a client tries to invite a user to a channel they're already on.
|
||||
log.Errorf("443 ERR_USERONCHANNEL, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("446", func(e *irc.Event) {
|
||||
// Returned by USERS when it has been disabled or not implemented.
|
||||
log.Errorf("446 ERR_USERSDISABLED, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("451", func(e *irc.Event) {
|
||||
// Предполагается, что надо авторизоваться, перед тем как что-то делать на сервере
|
||||
log.Errorf("451 ERR_NOTREGISTERED, %s", e.Raw)
|
||||
@@ -213,18 +334,19 @@ func ircClientRun() {
|
||||
log.Errorf("465 ERR_YOUREBANNEDCREEP, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("467", func(e *irc.Event) {
|
||||
log.Errorf("467 ERR_KEYSET, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("471", func(e *irc.Event) {
|
||||
// Это значит, что народу на канале максимальное количество. Будем пробовать присунуться раз в 30 сек.
|
||||
channel := e.Arguments[1]
|
||||
log.Warnf("471 ERR_CHANNELISFULL, %s", e.Raw)
|
||||
time.Sleep(30 * time.Second)
|
||||
<-time.NewTimer(30 * time.Second).C
|
||||
|
||||
// Проверяем, а должны ли мы быть заджоенныеми к указанному, каналу, а то вдруг нет?
|
||||
for _, ircChan := range config.Irc.Channels {
|
||||
if ircChan == channel {
|
||||
ircClient.Join(channel)
|
||||
break
|
||||
}
|
||||
if slices.Contains(Config.Irc.Channels, channel) {
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -237,14 +359,11 @@ func ircClientRun() {
|
||||
// По завершении работ +i снимают.
|
||||
channel := e.Arguments[1]
|
||||
log.Errorf("473 ERR_INVITEONLYCHAN, %s", e.Raw)
|
||||
time.Sleep(30 * time.Second)
|
||||
<-time.NewTimer(30 * time.Second).C
|
||||
|
||||
// Проверяем, а должны ли мы быть заджоенныеми к указанному, каналу, а то вдруг нет?
|
||||
for _, ircChan := range config.Irc.Channels {
|
||||
if ircChan == channel {
|
||||
ircClient.Join(channel)
|
||||
break
|
||||
}
|
||||
// Проверяем, а должны ли мы быть заджоенными к указанному, каналу, а то вдруг нет?
|
||||
if slices.Contains(Config.Irc.Channels, channel) {
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -254,18 +373,21 @@ func ircClientRun() {
|
||||
|
||||
log.Errorf("474 ERR_BANNEDFROMCHAN, %s", e.Raw)
|
||||
// Если нас забанили, то информацию о MODE-ах пользователей мы теряем
|
||||
time.Sleep(30 * time.Second)
|
||||
<-time.NewTimer(30 * time.Second).C
|
||||
|
||||
// Вдруг, нас забанили, но какбэ не навсегда?
|
||||
// TODO: вынести в настройки?
|
||||
|
||||
// Проверяем, а должны ли мы быть заджоенныеми к указанному, каналу, а то вдруг нет?
|
||||
for _, ircChan := range config.Irc.Channels {
|
||||
if ircChan == channel {
|
||||
ircClient.Join(channel)
|
||||
break
|
||||
}
|
||||
if slices.Contains(Config.Irc.Channels, channel) {
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
ircClient.AddCallback("475", func(e *irc.Event) {
|
||||
log.Errorf("475 ERR_BADCHANNELKEY, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("477", func(e *irc.Event) {
|
||||
log.Errorf("477 ERR_NOCHANMODES, %s", e.Raw)
|
||||
})
|
||||
@@ -274,6 +396,10 @@ func ircClientRun() {
|
||||
log.Errorf("478 ERR_BANLISTFULL, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("481", func(e *irc.Event) {
|
||||
log.Errorf("481 ERR_NOPRIVILEGES, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("482", func(e *irc.Event) {
|
||||
log.Errorf("482 ERR_CHANOPRIVSNEEDED, %s", e.Raw)
|
||||
})
|
||||
@@ -286,6 +412,10 @@ func ircClientRun() {
|
||||
log.Errorf("485 ERR_UNIQOPPRIVSNEEDED, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("491", func(e *irc.Event) {
|
||||
log.Errorf("491 ERR_NOOPERHOST, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("501", func(e *irc.Event) {
|
||||
log.Errorf("501 ERR_UMODEUNKNOWNFLAG, %s", e.Raw)
|
||||
})
|
||||
@@ -294,22 +424,6 @@ func ircClientRun() {
|
||||
log.Errorf("502 ERR_USERSDONTMATCH, %s", e.Raw)
|
||||
})
|
||||
|
||||
// Сделаем уже что-то полезное! Motd нам уже прислали и теперь можно авторизоваться и джойниться
|
||||
ircClient.AddCallback("001", func(e *irc.Event) {
|
||||
if !config.Irc.Sasl && config.Irc.Password != "" && !nickIsUsed {
|
||||
log.Info("Identifying via NickServ")
|
||||
message := fmt.Sprintf("identify %s %s", config.Irc.Nick, config.Irc.Password)
|
||||
imChan <- iMsg{ChatId: "NickServ", Text: message}
|
||||
}
|
||||
|
||||
log.Debug("Trying to join to preconfigured channels")
|
||||
|
||||
for _, channel := range config.Irc.Channels {
|
||||
log.Infof("Joining to %s channel", channel)
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
// Навесим коллбэков на другие, интересные нам события
|
||||
ircClient.AddCallback("KICK", func(e *irc.Event) {
|
||||
dstNick := e.Arguments[1]
|
||||
@@ -318,10 +432,11 @@ func ircClientRun() {
|
||||
|
||||
if e.Arguments[1] == ircClient.GetNick() {
|
||||
// Нас кикнули с канала, и мы теряем информацию о MODE-ах пользователей
|
||||
userMode.Delete(channel)
|
||||
UserMode.Delete(channel)
|
||||
// TODO: reason?
|
||||
log.Warnf("%s kicks us from %s", srcFullNick, channel)
|
||||
time.Sleep(10 * time.Second)
|
||||
<-time.NewTimer(10 * time.Second).C
|
||||
|
||||
ircClient.Join(channel)
|
||||
} else {
|
||||
// Кого-то другого кикнули с канала
|
||||
@@ -334,21 +449,22 @@ func ircClientRun() {
|
||||
srcNick := e.Nick
|
||||
dstNick := e.Arguments[0]
|
||||
|
||||
if e.Nick == config.Irc.Nick {
|
||||
if e.Nick == Config.Irc.Nick {
|
||||
nickIsUsed = false
|
||||
|
||||
if config.Irc.Password != "" {
|
||||
if Config.Irc.Password != "" {
|
||||
log.Warn("I regain my nick")
|
||||
} else {
|
||||
log.Warn("I regain my nick, trying to identify myself via NickServ")
|
||||
message := fmt.Sprintf("identify %s %s", config.Irc.Nick, config.Irc.Password)
|
||||
imChan <- iMsg{ChatId: "NickServ", Text: message}
|
||||
|
||||
message := fmt.Sprintf("identify %s %s", Config.Irc.Nick, Config.Irc.Password)
|
||||
imChan <- iMsg{ChatID: "NickServ", Text: message}
|
||||
}
|
||||
} else {
|
||||
log.Infof("%s renames themself to %s", srcNick, dstNick)
|
||||
}
|
||||
|
||||
// Неважно чей ник сменился, надо забыть, что было и снова узнать mode-ы сменишего nick джентельмена
|
||||
// Неважно чей ник сменился, надо забыть, что было и снова узнать mode-ы сменишего nick джентельмена.
|
||||
// TODO: реализовать userModeRenameUser()
|
||||
userModePurgeUser(srcNick)
|
||||
ircClient.Whois(dstNick)
|
||||
@@ -360,12 +476,12 @@ func ircClientRun() {
|
||||
channel := e.Arguments[0]
|
||||
|
||||
if nick == ircClient.GetNick() {
|
||||
// Команда names отправляется автоматом.
|
||||
log.Infof("I joined to %s", channel)
|
||||
// Команда names отправляется автоматом
|
||||
} else {
|
||||
log.Infof("%s joined to %s", fullNick, channel)
|
||||
// Технически, тут не надо спрашивать whois на пользователя, но мы спрашиваем, чтобы уточнить mode,
|
||||
// вдруг сервер проставляет mode заранее (хотя не должен)
|
||||
// вдруг сервер проставляет mode заранее (хотя не должен).
|
||||
ircClient.Whois(nick)
|
||||
}
|
||||
})
|
||||
@@ -377,7 +493,7 @@ func ircClientRun() {
|
||||
|
||||
if nick == ircClient.GetNick() {
|
||||
log.Infof("I parted from %s", channel)
|
||||
userMode.Delete(channel)
|
||||
UserMode.Delete(channel)
|
||||
} else {
|
||||
log.Infof("%s parted from %s", fullNick, channel)
|
||||
userModeDeleteUser(channel, nick)
|
||||
@@ -403,6 +519,7 @@ func ircClientRun() {
|
||||
channel := e.Arguments[0]
|
||||
fullSrcNick := e.Source
|
||||
srcNick := e.Nick
|
||||
|
||||
var dstNick string
|
||||
|
||||
if len(e.Arguments) >= 3 { // кого-то по-MODE-или на канале
|
||||
@@ -415,6 +532,7 @@ func ircClientRun() {
|
||||
}
|
||||
} else { // Установка mode-а при заходе на сервер
|
||||
dstNick = e.Arguments[0]
|
||||
|
||||
log.Infof("Server set my mode to %s", mode)
|
||||
}
|
||||
|
||||
@@ -454,17 +572,16 @@ func ircClientRun() {
|
||||
ircMsgParser(e.Arguments[0], e.Nick, e.User, e.Source, e.Arguments[1])
|
||||
})
|
||||
|
||||
/*
|
||||
ircClient.AddCallback("*", func(e *irc.Event) {
|
||||
log.Debugf("Incoming EVENT (Raw): %s", e.Raw)
|
||||
})
|
||||
*/
|
||||
ircClient.AddCallback("*", func(e *irc.Event) {
|
||||
log.Debugf("Incoming EVENT (Raw): %s", e.Raw)
|
||||
})
|
||||
|
||||
log.Debugf("Make an actual connection to irc")
|
||||
|
||||
if err := ircClient.Connect(serverString); err != nil {
|
||||
log.Fatalf("Unable to prepare irc connection: %s", err)
|
||||
time.Sleep(3 * time.Second)
|
||||
log.Errorf("Unable to prepare irc connection: %s", err)
|
||||
<-time.NewTimer(3 * time.Second).C // Sleep for 3 seconds.
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -474,26 +591,27 @@ func ircClientRun() {
|
||||
|
||||
// ircSend отправляет сообщение в irc. Якобы с применением ratelimit-ов, но на практике длинные сообщения разделяются
|
||||
// на более короткие и тут ratelimit не срабатывает. Если не повезёт, то сервер может за такое дело и кикнуть.
|
||||
func ircSend() {
|
||||
func IrcSend() {
|
||||
for {
|
||||
m := <-imChan
|
||||
|
||||
switch {
|
||||
case config.Irc.RateLimit.Type == "simple_delay":
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
switch Config.Irc.RateLimit.Type {
|
||||
case "simple_delay":
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text)
|
||||
ircClient.Action(m.ChatID, m.Text)
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
|
||||
sleepDelay := time.Duration(config.Irc.RateLimit.SimpleDelay) * time.Millisecond
|
||||
sleepDelay := time.Duration(Config.Irc.RateLimit.SimpleDelay) * time.Millisecond
|
||||
log.Debugf("Due to delay type simple_delay waiting for %d milliseconds", int(sleepDelay))
|
||||
time.Sleep(sleepDelay)
|
||||
case config.Irc.RateLimit.Type == "token_bucket":
|
||||
<-time.NewTimer(sleepDelay).C
|
||||
case "token_bucket":
|
||||
var currentTimeMs = time.Now().UnixMilli()
|
||||
var expirationTimeMs = config.Irc.RateLimit.TokenBucket.ExpirationTime * 1000
|
||||
|
||||
var expirationTimeMs = Config.Irc.RateLimit.TokenBucket.ExpirationTime * 1000
|
||||
|
||||
if len(msgBucket.Timestamps) > 0 {
|
||||
var newBucket bucket
|
||||
@@ -505,10 +623,11 @@ func ircSend() {
|
||||
}
|
||||
|
||||
bucketFill := len(newBucket.Timestamps)
|
||||
log.Debugf("Message bucket filled with %d/%d messages", bucketFill, config.Irc.RateLimit.TokenBucket.Size)
|
||||
log.Debugf("Message bucket filled with %d/%d messages", bucketFill, Config.Irc.RateLimit.TokenBucket.Size)
|
||||
|
||||
newBucket.Timestamps = append(newBucket.Timestamps, currentTimeMs)
|
||||
|
||||
if bucketFill >= config.Irc.RateLimit.TokenBucket.Size {
|
||||
if bucketFill >= Config.Irc.RateLimit.TokenBucket.Size {
|
||||
newBucket.IsFull = true
|
||||
}
|
||||
|
||||
@@ -516,52 +635,55 @@ func ircSend() {
|
||||
} else {
|
||||
// Проставим время отправки сообщения
|
||||
log.Debug("Message bucket is empty")
|
||||
|
||||
msgBucket.Timestamps = append(msgBucket.Timestamps, currentTimeMs)
|
||||
}
|
||||
|
||||
if msgBucket.IsFull {
|
||||
log.Debug("Message bucket overflowed, hitting ratelimit")
|
||||
|
||||
sleepPeriod := expirationTimeMs - (currentTimeMs - msgBucket.Timestamps[len(msgBucket.Timestamps)-1])
|
||||
|
||||
if config.Irc.RateLimit.TokenBucket.Limit > 0 {
|
||||
sleepPeriod = sleepPeriod / int64(config.Irc.RateLimit.TokenBucket.Limit)
|
||||
if Config.Irc.RateLimit.TokenBucket.Limit > 0 {
|
||||
sleepPeriod /= int64(Config.Irc.RateLimit.TokenBucket.Limit)
|
||||
}
|
||||
|
||||
log.Debugf("Sleeping for %d milliseconds", sleepPeriod)
|
||||
time.Sleep(time.Duration(sleepPeriod) * time.Millisecond)
|
||||
<-time.NewTimer(time.Duration(sleepPeriod) * time.Millisecond).C
|
||||
|
||||
currentTimeMs = time.Now().UnixMilli()
|
||||
// Обновим время отправки сообщения
|
||||
msgBucket.Timestamps[len(msgBucket.Timestamps)-1] = currentTimeMs
|
||||
}
|
||||
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if len(m.Text) > 4 && m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text[4:])
|
||||
ircClient.Action(m.ChatID, m.Text[4:])
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
default:
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if len(m.Text) > 4 && m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text[4:])
|
||||
ircClient.Action(m.ChatID, m.Text[4:])
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ircSendUnrestricted() {
|
||||
func IrcSendUnrestricted() {
|
||||
for {
|
||||
m := <-imChanUnrestricted
|
||||
log.Debugf("Skipping ratelimit and sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
log.Debugf("Skipping ratelimit and sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if len(m.Text) > 4 && m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text[4:])
|
||||
ircClient.Action(m.ChatID, m.Text[4:])
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,50 +1,55 @@
|
||||
package main
|
||||
package irc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/cockroachdb/pebble"
|
||||
"aleesa-irc-go/internal/anycollection"
|
||||
"aleesa-irc-go/internal/boolcollection"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
irc "github.com/thoj/go-ircevent"
|
||||
)
|
||||
|
||||
// Config - это у нас глобальная штука :)
|
||||
var config myConfig
|
||||
// Config - это у нас глобальная штука :).
|
||||
var Config MyConfig
|
||||
|
||||
// To break circular message forwarding we must set some sane default, it can be overridden via config
|
||||
// To break circular message forwarding we must set some sane default, it can be overridden via config.
|
||||
var forwardMax int64 = 5
|
||||
|
||||
// Объектики irc-клиента
|
||||
// Объектики irc-клиента.
|
||||
var ircClient *irc.Connection
|
||||
var nickIsUsed = false
|
||||
|
||||
// Объектики клиента-редиски
|
||||
var redisClient *redis.Client
|
||||
var subscriber *redis.PubSub
|
||||
// Объектики клиента-редиски.
|
||||
var RedisClient *redis.Client
|
||||
var Subscriber *redis.PubSub
|
||||
|
||||
// Main context
|
||||
// Main context.
|
||||
var ctx = context.Background()
|
||||
|
||||
// Ставится в true, если мы получили сигнал на выключение
|
||||
var shutdown = false
|
||||
// Ставится в true, если мы получили сигнал на выключение.
|
||||
var Shutdown = false
|
||||
|
||||
// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов
|
||||
var sigChan = make(chan os.Signal, 1)
|
||||
// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов.
|
||||
var SigChan = make(chan os.Signal, 1)
|
||||
|
||||
// Канал, в который пишутся сообщения для отправки в IRC в обычном порядке
|
||||
// Канал, в который пишутся сообщения для отправки в IRC в обычном порядке.
|
||||
var imChan = make(chan iMsg, 10000)
|
||||
|
||||
// Канал, в который пишутся сообщения для отправки в IRC, если ограничений нет (у бота +o или +v на канале)
|
||||
// Канал, в который пишутся сообщения для отправки в IRC, если ограничений нет (у бота +o или +v на канале).
|
||||
var imChanUnrestricted = make(chan iMsg, 100)
|
||||
|
||||
// "Ведёрко" с timestamp-ами последних отправленных сообщений
|
||||
// "Ведёрко" с timestamp-ами последних отправленных сообщений.
|
||||
var msgBucket bucket
|
||||
|
||||
// Мапка с открытыми дескрипторами баз с настройками
|
||||
var settingsDB = make(map[string]*pebble.DB)
|
||||
// "Базюлька" с MODE-ами пользователей на каналах.
|
||||
var UserMode = anycollection.NewCollection()
|
||||
|
||||
// "Базюлька" с MODE-ами пользователей на каналах
|
||||
var userMode = NewCollection()
|
||||
// "Базюлька" с доступными MODE-ами пользователя.
|
||||
var availableUserModes = boolcollection.NewCollection()
|
||||
|
||||
// "Базюлька" c с доступными MODE-ами каналов.
|
||||
var availableChanModes = boolcollection.NewCollection()
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package irc
|
||||
|
||||
import "strings"
|
||||
|
||||
@@ -8,33 +8,34 @@ import "strings"
|
||||
активируют мягкий ratelimit на стороне сервера, без них, сервер просто рвёт соединение по превышению лимитов
|
||||
*/
|
||||
|
||||
// Удаляет сведения о MODE-ах пользователя для заданного канала
|
||||
// Удаляет сведения о MODE-ах пользователя для заданного канала.
|
||||
func userModeDeleteUser(ircChan string, nick string) {
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
channel, ok := UserMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
channelData := channel.(map[string]map[string]bool)
|
||||
delete(channelData, nick)
|
||||
|
||||
if len(channelData) == 0 {
|
||||
userMode.Delete(ircChan)
|
||||
UserMode.Delete(ircChan)
|
||||
} else {
|
||||
userMode.Set(ircChan, channelData)
|
||||
UserMode.Set(ircChan, channelData)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Удаляет сведения о MODE-ах пользователя для всех каналов, на которых есть бот
|
||||
// Удаляет сведения о MODE-ах пользователя для всех каналов, на которых есть бот.
|
||||
func userModePurgeUser(nick string) {
|
||||
for _, channel := range config.Irc.Channels {
|
||||
for _, channel := range Config.Irc.Channels {
|
||||
userModeDeleteUser(channel, nick)
|
||||
}
|
||||
}
|
||||
|
||||
// Обновляет или создаёт запись о mode-ах пользователей
|
||||
// Обновляет или создаёт запись о mode-ах пользователей.
|
||||
func userModeUpdateUser(ircChan string, nick string, modes string) {
|
||||
var channelData map[string]map[string]bool
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
channel, ok := UserMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
channelData = channel.(map[string]map[string]bool)
|
||||
@@ -48,24 +49,25 @@ func userModeUpdateUser(ircChan string, nick string, modes string) {
|
||||
channelData[nick] = make(map[string]bool)
|
||||
}
|
||||
|
||||
switch {
|
||||
case modes[:1] == "+":
|
||||
switch modes[:1] {
|
||||
case "+":
|
||||
for _, mode := range strings.Split(modes[1:], "") {
|
||||
channelData[nick][mode] = true
|
||||
}
|
||||
case modes[:1] == "-":
|
||||
case "-":
|
||||
for _, mode := range strings.Split(modes[1:], "") {
|
||||
channelData[nick][mode] = false
|
||||
}
|
||||
}
|
||||
|
||||
userMode.Set(ircChan, channelData)
|
||||
UserMode.Set(ircChan, channelData)
|
||||
}
|
||||
|
||||
// Возвращает значение MODE-а v для запрошенного ника
|
||||
// Возвращает значение MODE-а v для запрошенного ника.
|
||||
func userModeIsVoiced(ircChan string, nick string) bool {
|
||||
var channelData map[string]map[string]bool
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
channel, ok := UserMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
channelData = channel.(map[string]map[string]bool)
|
||||
@@ -88,10 +90,11 @@ func userModeIsVoiced(ircChan string, nick string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Возвращает значение MODE-а o для запрошенного ника
|
||||
// Возвращает значение MODE-а o для запрошенного ника.
|
||||
func userModeIsOped(ircChan string, nick string) bool {
|
||||
var channelData map[string]map[string]bool
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
channel, ok := UserMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
channelData = channel.(map[string]map[string]bool)
|
||||
@@ -116,7 +119,8 @@ func userModeIsOped(ircChan string, nick string) bool {
|
||||
|
||||
func userModeIsHere(ircChan string, nick string) bool {
|
||||
var channelData map[string]map[string]bool
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
channel, ok := UserMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
channelData = channel.(map[string]map[string]bool)
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package irc
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@@ -6,15 +6,16 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"aleesa-irc-go/internal/log"
|
||||
"aleesa-irc-go/internal/setting"
|
||||
)
|
||||
|
||||
// ircMsgParser парсит сообщения, прилетевшие из IRC-ки
|
||||
func ircMsgParser(channel string, nick string, user string, source string, msg string) {
|
||||
// ircMsgParser парсит сообщения, прилетевшие из IRC-ки.
|
||||
func ircMsgParser(channel string, nick string, user string, source string, msg string) { //nolint: revive
|
||||
// nick - это выбранный пользователем nick (если он занят, то его "нарисует" сервер)
|
||||
// user - это короткое имя пользователя, под которым его видит сервер
|
||||
// source - это длинное имя пользователя, оно содержит в себе помимо user, ещё и ip с которого пришёл пользователь
|
||||
if shutdown {
|
||||
if Shutdown {
|
||||
// Если мы завершаем работу программы, то нам ничего обрабатывать не надо
|
||||
return
|
||||
}
|
||||
@@ -26,90 +27,90 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
}
|
||||
|
||||
// Ловим команды и обрабатываем их
|
||||
if (len(msg) > len(config.Csign)) && (msg[:len(config.Csign)] == config.Csign) {
|
||||
if (len(msg) > len(Config.Csign)) && (msg[:len(Config.Csign)] == Config.Csign) {
|
||||
var outgoingMessage string
|
||||
|
||||
var message sMsg
|
||||
message.From = config.Redis.MyChannel
|
||||
message.From = Config.Redis.MyChannel
|
||||
message.Userid = user // как его видит сервер
|
||||
message.Chatid = channel // чятик, в который написал user
|
||||
message.Threadid = "" // тредиков в irc нету, поэтому это поле отправляем пустым
|
||||
message.Plugin = config.Redis.MyChannel
|
||||
message.Plugin = Config.Redis.MyChannel
|
||||
message.Mode = "public"
|
||||
// Предполагается, что на команды бот автоматом отвечает
|
||||
message.Misc.Answer = 1
|
||||
message.Misc.Fwdcnt = 0
|
||||
message.Misc.Csign = config.Csign
|
||||
message.Misc.Csign = Config.Csign
|
||||
message.Misc.Username = nick
|
||||
message.Misc.Botnick = config.Irc.Nick
|
||||
message.Misc.Botnick = Config.Irc.Nick
|
||||
message.Misc.Msgformat = 0
|
||||
|
||||
var cmd = msg[len(config.Csign):]
|
||||
var cmd = msg[len(Config.Csign):]
|
||||
|
||||
switch {
|
||||
case cmd == "help" || msg == "помощь":
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%shelp | %sпомощь - это сообщение", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sanek | %sанек | %sанекдот - рандомный анекдот с anekdot.ru", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sbuni - комикс-стрип hapi buni", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sbunny - кролик", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%srabbit | %sкролик - кролик", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%scat | %sкис - кошечка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sdice | %sroll | %sкости - бросить кости", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sdig | %sкопать - заняться археологией", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sdrink | %sпраздник - какой сегодня праздник?", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfish | %sfisher - порыбачить", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sрыба | %sрыбка | %sрыбалка - порыбачить", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sf | %sф - рандомная фраза из сборника цитат fortune_mod", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfortune | %sфортунка - рандомная фраза из сборника цитат fortune_mod", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfox | %sлис - лисичка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfriday | %sпятница - а не пятница ли сегодня?", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfrog | %sлягушка - лягушка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%shorse | %sлошадь | %sлошадка - лошадка", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%skarma фраза - посмотреть карму фразы", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sкарма фраза - посмотреть карму фразы", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintln("фраза++ | фраза-- - повысить или понизить карму фразы")}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%slat | %sлат - сгенерировать фразу из крылатого латинского выражения", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%smonkeyuser - комикс-стрип MonkeyUser", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sowl | %sсова - сова", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sping | %sпинг - попинговать бота", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sproverb | %sпословица - рандомная русская пословица", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%ssnail | %sулитка - улитка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%ssome_brew - выдать соответствующий напиток, бармен может налить rum, ром, vodka, водку, tequila, текила, whisky, виски, absinthe, абсент", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sver | %sversion - написать что-то про версию ПО", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sверсия - написать что-то про версию ПО", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sw город | %sп город - погода в городе", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sxkcd - комикс-стрип с xkcb.ru", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%shelp | %sпомощь - это сообщение", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sanek | %sанек | %sанекдот - рандомный анекдот с anekdot.ru", Config.Csign, Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sbuni - комикс-стрип hapi buni", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sbunny - кролик", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%srabbit | %sкролик - кролик", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%scat | %sкис - кошечка", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sdice | %sroll | %sкости - бросить кости", Config.Csign, Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sdig | %sкопать - заняться археологией", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sdrink | %sпраздник - какой сегодня праздник?", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfish | %sfisher - порыбачить", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sрыба | %sрыбка | %sрыбалка - порыбачить", Config.Csign, Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sf | %sф - рандомная фраза из сборника цитат fortune_mod", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfortune | %sфортунка - рандомная фраза из сборника цитат fortune_mod", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfox | %sлис - лисичка", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfriday | %sпятница - а не пятница ли сегодня?", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfrog | %sлягушка - лягушка", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%shorse | %sлошадь | %sлошадка - лошадка", Config.Csign, Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%skarma фраза - посмотреть карму фразы", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sкарма фраза - посмотреть карму фразы", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintln("фраза++ | фраза-- - повысить или понизить карму фразы")}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%slat | %sлат - сгенерировать фразу из крылатого латинского выражения", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%smonkeyuser - комикс-стрип MonkeyUser", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sowl | %sсова - сова", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sping | %sпинг - попинговать бота", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sproverb | %sпословица - рандомная русская пословица", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%ssnail | %sулитка - улитка", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%ssome_brew - выдать соответствующий напиток, бармен может налить rum, ром, vodka, водку, tequila, текила, whisky, виски, absinthe, абсент", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sver | %sversion - написать что-то про версию ПО", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sверсия - написать что-то про версию ПО", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sw <город> | %sп <город> - погода в городе", Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sxkcd - комикс-стрип с xkcb.ru", Config.Csign)}
|
||||
|
||||
if userModeIsOped(channel, nick) {
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin - настройки некоторых плагинов бота для канала", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin - настройки некоторых плагинов бота для канала", Config.Csign)}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
case cmd == "admin":
|
||||
if userModeIsOped(channel, nick) {
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin oboobs # - где 1 - вкл, 0 - выкл плагина oboobs", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin oboobs показываем ли сисечки по просьбе участников чата (команды %stits, %stities, %sboobs, %sboobies, %sсиси, %sсисечки)", config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin obutts # - где 1 - вкл, 0 - выкл плагина obutts", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin obutts показываем ли попки по просьбе участников чата (команды %sass, %sbutt, %sbooty, %sпопа, %sпопка)", config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin oboobs # - где 1 - вкл, 0 - выкл плагина oboobs", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin oboobs показываем ли сисечки по просьбе участников чата (команды %stits, %stities, %sboobs, %sboobies, %sсиси, %sсисечки)", Config.Csign, Config.Csign, Config.Csign, Config.Csign, Config.Csign, Config.Csign, Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin obutts # - где 1 - вкл, 0 - выкл плагина obutts", Config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin obutts показываем ли попки по просьбе участников чата (команды %sass, %sbutt, %sbooty, %sпопа, %sпопка)", Config.Csign, Config.Csign, Config.Csign, Config.Csign, Config.Csign, Config.Csign)}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
case cmd == "admin oboobs":
|
||||
if userModeIsOped(channel, nick) {
|
||||
value := getSetting(channel, "oboobs")
|
||||
value := setting.Get(channel, "oboobs")
|
||||
|
||||
switch {
|
||||
case value == "":
|
||||
_ = saveSetting(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
case value == "0":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
case value == "1":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs включен"}
|
||||
switch value {
|
||||
case "":
|
||||
_ = setting.Set(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
case "0":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
case "1":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs включен"}
|
||||
default:
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,12 +118,12 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin oboobs 1":
|
||||
if userModeIsOped(channel, nick) {
|
||||
err := saveSetting(channel, "oboobs", "1")
|
||||
err := setting.Set(channel, "oboobs", "1")
|
||||
|
||||
if err != nil {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs всё ещё выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs всё ещё выключен"}
|
||||
} else {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs включен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs включен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,26 +131,26 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin oboobs 0":
|
||||
if userModeIsOped(channel, nick) {
|
||||
_ = saveSetting(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
_ = setting.Set(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
case cmd == "admin obutts":
|
||||
if userModeIsOped(channel, nick) {
|
||||
value := getSetting(channel, "obutts")
|
||||
value := setting.Get(channel, "obutts")
|
||||
|
||||
switch {
|
||||
case value == "":
|
||||
_ = saveSetting(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
case value == "0":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
case value == "1":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts включен"}
|
||||
switch value {
|
||||
case "":
|
||||
_ = setting.Set(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
case "0":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
case "1":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts включен"}
|
||||
default:
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,12 +158,12 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin obutts 1":
|
||||
if userModeIsOped(channel, nick) {
|
||||
err := saveSetting(channel, "obutts", "1")
|
||||
err := setting.Set(channel, "obutts", "1")
|
||||
|
||||
if err != nil {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts всё ещё выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts всё ещё выключен"}
|
||||
} else {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts включен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts включен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,8 +171,8 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin obutts 0":
|
||||
if userModeIsOped(channel, nick) {
|
||||
_ = saveSetting(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
_ = setting.Set(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
}
|
||||
|
||||
return
|
||||
@@ -182,15 +183,17 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
// Общий список простых команд
|
||||
cmds := []string{"ping", "пинг", "пинх", "pong", "понг", "понх", "coin", "монетка", "roll", "dice", "кости",
|
||||
"ver", "version", "версия", "хэлп", "halp", "kde", "кде", "lat", "лат", "friday", "пятница", "proverb",
|
||||
"пословица", "fortune", "фортунка", "f", "ф", "anek", "анек", "анекдот", "buni", "cat", "кис", "drink",
|
||||
"праздник", "fox", "лис", "frog", "лягушка", "horse", "лошадь", "лошадка", "monkeyuser", "owl", "сова",
|
||||
"сыч", "rabbit", "bunny", "кролик", "snail", "улитка", "xkcd", "dig", "копать", "fish", "fishing", "рыба",
|
||||
"рыбка", "рыбалка", "karma", "карма"}
|
||||
"пословица", "пословиться", "fortune", "фортунка", "f", "ф", "anek", "анек", "анекдот", "buni", "cat",
|
||||
"кис", "drink", "праздник", "fox", "лис", "frog", "лягушка", "horse", "лошадь", "лошадка", "monkeyuser",
|
||||
"owl", "сова", "сыч", "rabbit", "bunny", "кролик", "snail", "улитка", "xkcd", "dig", "копать", "fish",
|
||||
"fishing", "рыба", "рыбка", "рыбалка", "karma", "карма", "w", "п", "weather", "погода", "погодка",
|
||||
"погадка"}
|
||||
|
||||
for _, command := range cmds {
|
||||
if cmd == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -205,6 +208,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmd == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -225,7 +229,8 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
message.Misc.Username = strings.TrimSpace(pile[1])
|
||||
} else {
|
||||
msg = fmt.Sprintf("Я тут не вижу участника с ником %s", userNick)
|
||||
imChan <- iMsg{ChatId: channel, Text: msg}
|
||||
imChan <- iMsg{ChatID: channel, Text: msg}
|
||||
|
||||
return
|
||||
}
|
||||
} // в противном случае юзер просто поставил пробел на конце команды, интерпретируем это как
|
||||
@@ -233,6 +238,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
outgoingMessage = msg
|
||||
done = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -249,6 +255,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmdLen > len(command) && cmd[0:len(command)] == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -256,7 +263,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
// Отключаемые команды
|
||||
if !done {
|
||||
value := getSetting(channel, "obutts")
|
||||
value := setting.Get(channel, "obutts")
|
||||
|
||||
if value == "1" {
|
||||
cmds := []string{"butt", "booty", "ass", "попа", "попка"}
|
||||
@@ -266,13 +273,14 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmd[:cmdLen] == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !done {
|
||||
value = getSetting(channel, "oboobs")
|
||||
value = setting.Get(channel, "oboobs")
|
||||
|
||||
if value == "1" {
|
||||
cmds := []string{"tits", "boobs", "tities", "boobies", "сиси", "сисечки"}
|
||||
@@ -282,6 +290,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmd[:cmdLen] == command {
|
||||
// done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -296,25 +305,26 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Unable to to serialize message for redis: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Заталкиваем наш json в редиску
|
||||
if err := redisClient.Publish(ctx, config.Redis.Channel, data).Err(); err != nil {
|
||||
log.Warnf("Unable to send data to redis channel %s: %s", config.Redis.Channel, err)
|
||||
if err := RedisClient.Publish(ctx, Config.Redis.Channel, data).Err(); err != nil {
|
||||
log.Warnf("Unable to send data to redis channel %s: %s", Config.Redis.Channel, err)
|
||||
} else {
|
||||
log.Debugf("Sent msg to redis channel %s: %s", config.Redis.Channel, string(data))
|
||||
log.Debugf("Sent msg to redis channel %s: %s", Config.Redis.Channel, string(data))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Это уже просто трёп в чятике
|
||||
var message sMsg
|
||||
message.From = config.Redis.MyChannel
|
||||
message.From = Config.Redis.MyChannel
|
||||
message.Userid = user // как его видит сервер
|
||||
message.Chatid = channel // чятик, в который написал user
|
||||
message.Threadid = "" // тредиков в irc нету, поэтому это поле отправляем пустым
|
||||
message.Message = msg
|
||||
message.Plugin = config.Redis.MyChannel
|
||||
message.Plugin = Config.Redis.MyChannel
|
||||
message.Mode = "public"
|
||||
|
||||
// Если во фразе содержится ключевые слова, под это есть костыль в craniac-е - (предполагаем, что) он
|
||||
@@ -341,32 +351,31 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
}
|
||||
|
||||
message.Misc.Fwdcnt = 0
|
||||
message.Misc.Csign = config.Csign
|
||||
message.Misc.Csign = Config.Csign
|
||||
message.Misc.Username = nick
|
||||
message.Misc.Botnick = config.Irc.Nick
|
||||
message.Misc.Botnick = Config.Irc.Nick
|
||||
message.Misc.Msgformat = 0
|
||||
|
||||
data, err := json.Marshal(message)
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Unable to to serialize message for redis: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Заталкиваем наш json в редиску
|
||||
if err := redisClient.Publish(ctx, config.Redis.Channel, data).Err(); err != nil {
|
||||
log.Warnf("Unable to send data to redis channel %s: %s", config.Redis.Channel, err)
|
||||
if err := RedisClient.Publish(ctx, Config.Redis.Channel, data).Err(); err != nil {
|
||||
log.Warnf("Unable to send data to redis channel %s: %s", Config.Redis.Channel, err)
|
||||
} else {
|
||||
log.Debugf("Sent msg to redis channel %s: %s", config.Redis.Channel, string(data))
|
||||
log.Debugf("Sent msg to redis channel %s: %s", Config.Redis.Channel, string(data))
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// redisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными
|
||||
func redisMsgParser(msg string) {
|
||||
if shutdown {
|
||||
// RedisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными.
|
||||
func RedisMsgParser(msg string) {
|
||||
if Shutdown {
|
||||
// Если мы завершаем работу программы, то нам ничего обрабатывать не надо
|
||||
return
|
||||
}
|
||||
@@ -377,50 +386,58 @@ func redisMsgParser(msg string) {
|
||||
|
||||
if err := json.Unmarshal([]byte(msg), &j); err != nil {
|
||||
log.Warnf("Unable to to parse message from redis channel: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Validate our j
|
||||
if exist := j.From; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no from field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Chatid; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no chatid field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Userid; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no userid field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Message; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no message field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Plugin; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no plugin field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Mode; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no mode field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// j.Misc.Answer может и не быть, тогда ответа на такое сообщение не будет
|
||||
if j.Misc.Answer == 0 {
|
||||
log.Debug("Field Misc->Answer = 0, skipping message")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// j.Misc.BotNick тоже можно не передавать, тогда будет записана пустая строка
|
||||
// j.Misc.CSign если нам его не передали, возьмём значение из конфига
|
||||
if exist := j.Misc.Csign; exist == "" {
|
||||
j.Misc.Csign = config.Csign
|
||||
j.Misc.Csign = Config.Csign
|
||||
}
|
||||
|
||||
// j.Misc.FwdCnt если нам его не передали, то будет 0
|
||||
@@ -437,13 +454,11 @@ func redisMsgParser(msg string) {
|
||||
|
||||
for _, message := range lines {
|
||||
if userModeIsOped(j.Chatid, ircClient.GetNick()) || userModeIsVoiced(j.Chatid, ircClient.GetNick()) {
|
||||
imChanUnrestricted <- iMsg{ChatId: j.Chatid, Text: message}
|
||||
imChanUnrestricted <- iMsg{ChatID: j.Chatid, Text: message}
|
||||
} else {
|
||||
imChan <- iMsg{ChatId: j.Chatid, Text: message}
|
||||
imChan <- iMsg{ChatID: j.Chatid, Text: message}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
@@ -1,7 +1,7 @@
|
||||
package main
|
||||
package irc
|
||||
|
||||
// Конфиг
|
||||
type myConfig struct {
|
||||
// Конфиг.
|
||||
type MyConfig struct {
|
||||
Redis struct {
|
||||
Server string `json:"server,omitempty"`
|
||||
Port int `json:"port,omitempty"`
|
||||
@@ -35,7 +35,7 @@ type myConfig struct {
|
||||
DataDir string `json:"data_dir,omitempty"`
|
||||
}
|
||||
|
||||
// Входящее сообщение из pubsub-канала redis-ки
|
||||
// Входящее сообщение из pubsub-канала redis-ки.
|
||||
type rMsg struct {
|
||||
From string `json:"from,omitempty"`
|
||||
Chatid string `json:"chatid,omitempty"`
|
||||
@@ -55,7 +55,7 @@ type rMsg struct {
|
||||
} `json:"Misc"`
|
||||
}
|
||||
|
||||
// Исходящее сообщение в pubsub-канал redis-ки
|
||||
// Исходящее сообщение в pubsub-канал redis-ки.
|
||||
type sMsg struct {
|
||||
From string `json:"from"`
|
||||
Chatid string `json:"chatid"`
|
||||
@@ -75,14 +75,13 @@ type sMsg struct {
|
||||
} `json:"misc"`
|
||||
}
|
||||
|
||||
// Сообщение в irc
|
||||
// TODO: Имплементировать тип сообщения - privmsg или action
|
||||
// TODO: Имплементировать тип сообщения - privmsg или action.
|
||||
type iMsg struct {
|
||||
ChatId string
|
||||
ChatID string
|
||||
Text string
|
||||
}
|
||||
|
||||
// "Ведёрко" для реализации ограничителя скорости отправки исходящих сообщений - Token Bucket
|
||||
// "Ведёрко" для реализации ограничителя скорости отправки исходящих сообщений - Token Bucket.
|
||||
type bucket struct {
|
||||
Timestamps []int64
|
||||
IsFull bool
|
||||
@@ -1,24 +1,25 @@
|
||||
package main
|
||||
package irc
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"aleesa-irc-go/internal/log"
|
||||
"aleesa-irc-go/internal/setting"
|
||||
|
||||
"github.com/hjson/hjson-go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Читает и валидирует конфиг, а также выставляет некоторые default-ы, если значений для параметров в конфиге нет
|
||||
func readConfig() {
|
||||
// ReadConfig читает и валидирует конфиг, а также выставляет некоторые default-ы, если значений для параметров в конфиге нет.
|
||||
func ReadConfig() {
|
||||
configLoaded := false
|
||||
executablePath, err := os.Executable()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("Unable to get current executable path: %s", err)
|
||||
log.Error(fmt.Sprintf("Unable to get current executable path: %s", err))
|
||||
}
|
||||
|
||||
configJSONPath := fmt.Sprintf("%s/data/config.json", filepath.Dir(executablePath))
|
||||
@@ -41,27 +42,31 @@ func readConfig() {
|
||||
// Конфиг-файл длинноват для конфига, попробуем следующего кандидата
|
||||
if fileInfo.Size() > 65535 {
|
||||
log.Warnf("Config file %s is too long for config, skipping", location)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadFile(location)
|
||||
buf, err := os.ReadFile(location)
|
||||
|
||||
// Не удалось прочитать, попробуем следующего кандидата
|
||||
if err != nil {
|
||||
log.Warnf("Skip reading config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Исходя из документации, hjson какбы умеет парсить "кривой" json, но парсит его в map-ку.
|
||||
// Интереснее на выходе получить структурку: то есть мы вначале конфиг преобразуем в map-ку, затем эту map-ку
|
||||
// сериализуем в json, а потом json преврщааем в стркутурку. Не очень эффективно, но он и не часто требуется.
|
||||
var sampleConfig myConfig
|
||||
var sampleConfig MyConfig
|
||||
|
||||
var tmp map[string]interface{}
|
||||
err = hjson.Unmarshal(buf, &tmp)
|
||||
|
||||
// Не удалось распарсить - попробуем следующего кандидата
|
||||
if err != nil {
|
||||
log.Warnf("Skip parsing config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -70,17 +75,20 @@ func readConfig() {
|
||||
// Не удалось преобразовать map-ку в json
|
||||
if err != nil {
|
||||
log.Warnf("Skip parsing config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(tmpjson, &sampleConfig); err != nil {
|
||||
log.Warnf("Skip parsing config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Валидируем значения из конфига
|
||||
if sampleConfig.Redis.Server == "" {
|
||||
sampleConfig.Redis.Server = "localhost"
|
||||
|
||||
log.Infof("Redis server is not defined in config, using localhost")
|
||||
}
|
||||
|
||||
@@ -101,15 +109,17 @@ func readConfig() {
|
||||
// Значения для IRC-клиента
|
||||
if sampleConfig.Irc.Server == "" {
|
||||
sampleConfig.Irc.Server = "localhost"
|
||||
|
||||
log.Errorf("Irc server is not defined in config, using localhost")
|
||||
}
|
||||
|
||||
if sampleConfig.Irc.Port == 0 {
|
||||
sampleConfig.Irc.Port = 6667
|
||||
|
||||
log.Infof("Irc port is not defined in config, using 6667")
|
||||
}
|
||||
|
||||
if sampleConfig.Irc.Ssl != true {
|
||||
if !sampleConfig.Irc.Ssl {
|
||||
sampleConfig.Irc.SslVerify = false
|
||||
}
|
||||
|
||||
@@ -180,9 +190,11 @@ func readConfig() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
config = sampleConfig
|
||||
Config = sampleConfig
|
||||
configLoaded = true
|
||||
|
||||
log.Infof("Using %s as config file", location)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
@@ -192,19 +204,19 @@ func readConfig() {
|
||||
}
|
||||
}
|
||||
|
||||
// Хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения
|
||||
func sigHandler() {
|
||||
// SigHandler хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения.
|
||||
func SigHandler() {
|
||||
var err error
|
||||
|
||||
for {
|
||||
var s = <-sigChan
|
||||
var s = <-SigChan
|
||||
switch s {
|
||||
case syscall.SIGINT:
|
||||
log.Infoln("Got SIGINT, quitting")
|
||||
log.Info("Got SIGINT, quitting")
|
||||
case syscall.SIGTERM:
|
||||
log.Infoln("Got SIGTERM, quitting")
|
||||
log.Info("Got SIGTERM, quitting")
|
||||
case syscall.SIGQUIT:
|
||||
log.Infoln("Got SIGQUIT, quitting")
|
||||
log.Info("Got SIGQUIT, quitting")
|
||||
|
||||
// Заходим на новую итерацию, если у нас "неинтересный" сигнал
|
||||
default:
|
||||
@@ -212,16 +224,16 @@ func sigHandler() {
|
||||
}
|
||||
|
||||
// Чтобы не срать в логи ошибками от редиски, проставим shutdown state приложения в true
|
||||
shutdown = true
|
||||
Shutdown = true
|
||||
|
||||
// Отпишемся от всех каналов и закроем коннект к редиске
|
||||
if err = subscriber.Unsubscribe(ctx); err != nil {
|
||||
if err = Subscriber.Unsubscribe(ctx); err != nil {
|
||||
log.Errorf("Unable to unsubscribe from redis channels cleanly: %s", err)
|
||||
} else {
|
||||
log.Debug("Unsubscribe from all redis channels")
|
||||
}
|
||||
|
||||
if err = subscriber.Close(); err != nil {
|
||||
if err = Subscriber.Close(); err != nil {
|
||||
log.Errorf("Unable to close redis connection cleanly: %s", err)
|
||||
} else {
|
||||
log.Debug("Close redis connection")
|
||||
@@ -229,16 +241,12 @@ func sigHandler() {
|
||||
|
||||
log.Debug("Close irc connection")
|
||||
ircClient.Quit()
|
||||
|
||||
log.Debug("Close userModeDB")
|
||||
userMode.Close()
|
||||
UserMode.Close()
|
||||
|
||||
if len(settingsDB) > 0 {
|
||||
log.Debug("Closing runtime irc channel settings db")
|
||||
|
||||
for _, db := range settingsDB {
|
||||
_ = db.Close()
|
||||
}
|
||||
}
|
||||
log.Debug("Close settingsDB")
|
||||
setting.CloseAll()
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
106
internal/log/log.go
Normal file
106
internal/log/log.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Init setup logger stuff.
|
||||
// level can be error, warn, info, debug if something other supplied info level selected.
|
||||
// fileDescriptor should be opened before supplying it to Init().
|
||||
func Init(level string, fileDescriptor *os.File) {
|
||||
var loglevel slog.Level
|
||||
|
||||
// no panic, no trace.
|
||||
switch level {
|
||||
case "error":
|
||||
loglevel = slog.LevelError
|
||||
|
||||
case "warn":
|
||||
loglevel = slog.LevelWarn
|
||||
|
||||
case "info":
|
||||
loglevel = slog.LevelInfo
|
||||
|
||||
case "debug":
|
||||
loglevel = slog.LevelDebug
|
||||
|
||||
default:
|
||||
loglevel = slog.LevelInfo
|
||||
|
||||
}
|
||||
|
||||
opts := &slog.HandlerOptions{
|
||||
// Use the ReplaceAttr function on the handler options
|
||||
// to be able to replace any single attribute in the log output
|
||||
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
|
||||
// check that we are handling the time key
|
||||
if a.Key != slog.TimeKey {
|
||||
return a
|
||||
}
|
||||
|
||||
t := a.Value.Time()
|
||||
|
||||
// change the value from a time.Time to a String
|
||||
// where the string has the correct time format.
|
||||
a.Value = slog.StringValue(t.Format(time.DateTime))
|
||||
|
||||
return a
|
||||
},
|
||||
|
||||
Level: loglevel,
|
||||
}
|
||||
|
||||
slog.SetDefault(
|
||||
slog.New(
|
||||
slog.NewTextHandler(
|
||||
fileDescriptor,
|
||||
opts,
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
// Error passes message directly to slog.Error().
|
||||
func Error(message string) {
|
||||
slog.Error(message)
|
||||
}
|
||||
|
||||
// Errorf logs message on error log level and allows to supply arguments in printf() style.
|
||||
func Errorf(format string, a ...any) {
|
||||
slog.Error(fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
// Warn passes message directly to slog.Warn().
|
||||
func Warn(message string) {
|
||||
slog.Warn(message)
|
||||
}
|
||||
|
||||
// Warnf logs message on warn log level and allows to supply arguments in printf() style.
|
||||
func Warnf(format string, a ...any) {
|
||||
slog.Warn(fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
// Info passes message directly to slog.Info().
|
||||
func Info(message string) {
|
||||
slog.Info(message)
|
||||
}
|
||||
|
||||
// Infof logs message on info log level and allows to supply arguments in printf() style.
|
||||
func Infof(format string, a ...any) {
|
||||
slog.Info(fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
// Debug passes message directly to slog.Debug().
|
||||
func Debug(message string) {
|
||||
slog.Debug(message)
|
||||
}
|
||||
|
||||
// Debugf logs message on debug log level and allows to supply arguments in printf() style.
|
||||
func Debugf(format string, a ...any) {
|
||||
slog.Debug(fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package setting
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
@@ -6,14 +6,20 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
|
||||
"aleesa-irc-go/internal/log"
|
||||
|
||||
"github.com/cockroachdb/errors/oserror"
|
||||
"github.com/cockroachdb/pebble"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// StoreKV сохраняет в указанной бд ключ и значение
|
||||
// Мапка с открытыми дескрипторами баз с настройками.
|
||||
var settingsDB = make(map[string]*pebble.DB)
|
||||
var DataDir string
|
||||
|
||||
// StoreKV сохраняет в указанной бд ключ и значение.
|
||||
func StoreKV(db *pebble.DB, key string, value string) error {
|
||||
var kArray = []byte(key)
|
||||
|
||||
var vArray = []byte(value)
|
||||
|
||||
err := db.Set(kArray, vArray, pebble.Sync)
|
||||
@@ -21,11 +27,12 @@ func StoreKV(db *pebble.DB, key string, value string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// FetchV достаёт значение по ключу
|
||||
// FetchV достаёт значение по ключу.
|
||||
func FetchV(db *pebble.DB, key string) (string, error) {
|
||||
var kArray = []byte(key)
|
||||
|
||||
var vArray []byte
|
||||
|
||||
var valueString = ""
|
||||
|
||||
vArray, closer, err := db.Get(kArray)
|
||||
@@ -40,8 +47,8 @@ func FetchV(db *pebble.DB, key string) (string, error) {
|
||||
return valueString, err
|
||||
}
|
||||
|
||||
// Достанем настройку из БД с настройками
|
||||
func getSetting(chatID string, setting string) string {
|
||||
// Достанем настройку из БД с настройками.
|
||||
func Get(chatID string, setting string) string {
|
||||
var err error
|
||||
|
||||
chatHash := sha256.Sum256([]byte(chatID))
|
||||
@@ -54,10 +61,11 @@ func getSetting(chatID string, setting string) string {
|
||||
// настройками порождается огромное количество файлов. Умолчальное ограничение на количество файлов - 500 штук,
|
||||
// что нас не устраивает, поэтому немного снизим эту цифру до более приемлемых значений
|
||||
options.L0CompactionFileThreshold = 8
|
||||
settingsDB[database], err = pebble.Open(config.DataDir+"/"+database, &options)
|
||||
settingsDB[database], err = pebble.Open(DataDir+"/"+database, &options)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("Unable to open settings db, %s: %s\n", database, err)
|
||||
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -66,13 +74,14 @@ func getSetting(chatID string, setting string) string {
|
||||
|
||||
// Если из базы ничего не вынулось, по каким-то причинам, то просто вернём пустую строку
|
||||
if err != nil {
|
||||
if errors.Is(err, pebble.ErrNotFound) {
|
||||
switch {
|
||||
case errors.Is(err, pebble.ErrNotFound):
|
||||
log.Debugf("Unable to get value for %s: no record found in db %s", setting, database)
|
||||
} else if errors.Is(err, fs.ErrNotExist) {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
log.Debugf("Unable to get value for %s: db dir %s does not exist", setting, database)
|
||||
} else if errors.Is(err, oserror.ErrNotExist) {
|
||||
case errors.Is(err, oserror.ErrNotExist):
|
||||
log.Debugf("Unable to get value for %s: db dir %s does not exist", setting, database)
|
||||
} else {
|
||||
default:
|
||||
log.Errorf("Unable to get value for %s in db dir %s: %s", setting, database, err)
|
||||
}
|
||||
|
||||
@@ -82,10 +91,12 @@ func getSetting(chatID string, setting string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
// Сохраним настройку в БД с настройками
|
||||
func saveSetting(chatID string, setting string, value string) error {
|
||||
// Сохраним настройку в БД с настройками.
|
||||
func Set(chatID string, setting string, value string) error {
|
||||
var chatHash = sha256.Sum256([]byte(chatID))
|
||||
|
||||
var database = fmt.Sprintf("settings_db/%x", chatHash)
|
||||
|
||||
var err error
|
||||
|
||||
// Если БД не открыта, откроем её
|
||||
@@ -96,10 +107,11 @@ func saveSetting(chatID string, setting string, value string) error {
|
||||
// что нас не устраивает, поэтому немного снизим эту цифру до более приемлемых значений
|
||||
options.L0CompactionFileThreshold = 8
|
||||
|
||||
settingsDB[database], err = pebble.Open(config.DataDir+"/"+database, &options)
|
||||
settingsDB[database], err = pebble.Open(DataDir+"/"+database, &options)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("Unable to open settings db, %s: %s\n", database, err)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -111,4 +123,15 @@ func saveSetting(chatID string, setting string, value string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Закрываем все инстансы баз с настройками.
|
||||
func CloseAll() {
|
||||
if len(settingsDB) > 0 {
|
||||
log.Debug("Closing runtime irc channel settings db")
|
||||
|
||||
for _, db := range settingsDB {
|
||||
_ = db.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
102
main.go
102
main.go
@@ -1,102 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Производит некоторую инициализацию перед запуском main()
|
||||
func init() {
|
||||
log.SetFormatter(&log.TextFormatter{
|
||||
DisableQuote: true,
|
||||
DisableLevelTruncation: false,
|
||||
DisableColors: true,
|
||||
FullTimestamp: true,
|
||||
TimestampFormat: "2006-01-02 15:04:05",
|
||||
})
|
||||
|
||||
readConfig()
|
||||
|
||||
// no panic, no trace
|
||||
switch config.Loglevel {
|
||||
case "fatal":
|
||||
log.SetLevel(log.FatalLevel)
|
||||
case "error":
|
||||
log.SetLevel(log.ErrorLevel)
|
||||
case "warn":
|
||||
log.SetLevel(log.WarnLevel)
|
||||
case "info":
|
||||
log.SetLevel(log.InfoLevel)
|
||||
case "debug":
|
||||
log.SetLevel(log.DebugLevel)
|
||||
default:
|
||||
log.SetLevel(log.InfoLevel)
|
||||
}
|
||||
|
||||
// Проинициализируем мапку MODE-ами пользователей произвольным значением и удалим его
|
||||
var usersModes = map[string]map[string]map[string]bool{
|
||||
"user": {
|
||||
"mode": {
|
||||
"v": true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
userMode.Set("channel", usersModes)
|
||||
userMode.Delete("channel")
|
||||
}
|
||||
|
||||
// Собственно, какбэ "точка входа" - основная процедура в нашем боте
|
||||
func main() {
|
||||
// Main context
|
||||
var ctx = context.Background()
|
||||
|
||||
// Откроем лог и скормим его логгеру
|
||||
if config.Log != "" {
|
||||
logfile, err := os.OpenFile(config.Log, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to open log file %s: %s", config.Log, err)
|
||||
}
|
||||
|
||||
log.SetOutput(logfile)
|
||||
}
|
||||
|
||||
// Иницализируем redis-клиента
|
||||
redisClient = redis.NewClient(&redis.Options{
|
||||
Addr: fmt.Sprintf("%s:%d", config.Redis.Server, config.Redis.Port),
|
||||
})
|
||||
|
||||
log.Debugf("Lazy connect() to redis at %s:%d", config.Redis.Server, config.Redis.Port)
|
||||
subscriber = redisClient.Subscribe(ctx, config.Redis.MyChannel)
|
||||
redisMsgChan := subscriber.Channel()
|
||||
|
||||
go ircClientRun()
|
||||
go ircSend()
|
||||
go ircSendUnrestricted()
|
||||
|
||||
// Самое время поставить траппер сигналов
|
||||
signal.Notify(sigChan,
|
||||
syscall.SIGINT,
|
||||
syscall.SIGTERM,
|
||||
syscall.SIGQUIT)
|
||||
|
||||
go sigHandler()
|
||||
|
||||
// Обработчик событий от редиски
|
||||
for msg := range redisMsgChan {
|
||||
if shutdown {
|
||||
continue
|
||||
}
|
||||
|
||||
redisMsgParser(msg.Payload)
|
||||
}
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
22
vendor/github.com/DataDog/zstd/README.md
generated
vendored
22
vendor/github.com/DataDog/zstd/README.md
generated
vendored
@@ -6,8 +6,8 @@
|
||||
|
||||
[C Zstd Homepage](https://github.com/facebook/zstd)
|
||||
|
||||
The current headers and C files are from *v1.5.0* (Commit
|
||||
[10f0e699](https://github.com/facebook/zstd/releases/tag/v1.5.0)).
|
||||
The current headers and C files are from *v1.4.4* (Commit
|
||||
[10f0e699](https://github.com/facebook/zstd/releases/tag/v1.4.4)).
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -19,21 +19,6 @@ There are two main APIs:
|
||||
The compress/decompress APIs mirror that of lz4, while the streaming API was
|
||||
designed to be a drop-in replacement for zlib.
|
||||
|
||||
### Building against an external libzstd
|
||||
|
||||
By default, zstd source code is vendored in this repository and the binding will be built with
|
||||
the vendored source code bundled.
|
||||
|
||||
If you want to build this binding against an external static or shared libzstd library, you can
|
||||
use the `external_libzstd` build tag. This will look for the libzstd pkg-config file and extract
|
||||
build and linking parameters from that pkg-config file.
|
||||
|
||||
Note that it requires at least libzstd 1.4.0.
|
||||
|
||||
```bash
|
||||
go build -tags external_libzstd
|
||||
```
|
||||
|
||||
### Simple `Compress/Decompress`
|
||||
|
||||
|
||||
@@ -75,9 +60,6 @@ NewWriterLevelDict(w io.Writer, level int, dict []byte) *Writer
|
||||
// Write compresses the input data and write it to the underlying writer
|
||||
(w *Writer) Write(p []byte) (int, error)
|
||||
|
||||
// Flush writes any unwritten data to the underlying writer
|
||||
(w *Writer) Flush() error
|
||||
|
||||
// Close flushes the buffer and frees C zstd objects
|
||||
(w *Writer) Close() error
|
||||
```
|
||||
|
||||
135
vendor/github.com/DataDog/zstd/bitstream.h
generated
vendored
135
vendor/github.com/DataDog/zstd/bitstream.h
generated
vendored
@@ -1,16 +1,35 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* bitstream
|
||||
* Part of FSE library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
bitstream
|
||||
Part of FSE library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef BITSTREAM_H_MODULE
|
||||
#define BITSTREAM_H_MODULE
|
||||
@@ -18,6 +37,7 @@
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* This API consists of small unitary functions, which must be inlined for best performance.
|
||||
* Since link-time-optimization is not available for all compilers,
|
||||
@@ -28,7 +48,6 @@ extern "C" {
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include "mem.h" /* unaligned access routines */
|
||||
#include "compiler.h" /* UNLIKELY() */
|
||||
#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
|
||||
#include "error_private.h" /* error codes and messages */
|
||||
|
||||
@@ -36,12 +55,10 @@ extern "C" {
|
||||
/*=========================================
|
||||
* Target specific
|
||||
=========================================*/
|
||||
#ifndef ZSTD_NO_INTRINSICS
|
||||
# if defined(__BMI__) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
# elif defined(__ICCARM__)
|
||||
# include <intrinsics.h>
|
||||
# endif
|
||||
#if defined(__BMI__) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
#elif defined(__ICCARM__)
|
||||
# include <intrinsics.h>
|
||||
#endif
|
||||
|
||||
#define STREAM_ACCUMULATOR_MIN_32 25
|
||||
@@ -143,18 +160,9 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
# if STATIC_BMI2 == 1
|
||||
return _lzcnt_u32(val) ^ 31;
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
unsigned long r=0;
|
||||
_BitScanReverse ( &r, val );
|
||||
return (unsigned) r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
||||
return __builtin_clz (val) ^ 31;
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
@@ -210,7 +218,7 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
|
||||
size_t value, unsigned nbBits)
|
||||
{
|
||||
DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
|
||||
MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
||||
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
|
||||
@@ -283,7 +291,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
|
||||
*/
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
|
||||
@@ -300,22 +308,22 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
switch(srcSize)
|
||||
{
|
||||
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
|
||||
ZSTD_FALLTHROUGH;
|
||||
/* fall-through */
|
||||
|
||||
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
|
||||
ZSTD_FALLTHROUGH;
|
||||
/* fall-through */
|
||||
|
||||
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
|
||||
ZSTD_FALLTHROUGH;
|
||||
/* fall-through */
|
||||
|
||||
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
|
||||
ZSTD_FALLTHROUGH;
|
||||
/* fall-through */
|
||||
|
||||
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
|
||||
ZSTD_FALLTHROUGH;
|
||||
/* fall-through */
|
||||
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
|
||||
ZSTD_FALLTHROUGH;
|
||||
/* fall-through */
|
||||
|
||||
default: break;
|
||||
}
|
||||
@@ -329,36 +337,23 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
|
||||
MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
|
||||
{
|
||||
return bitContainer >> start;
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
{
|
||||
U32 const regMask = sizeof(bitContainer)*8 - 1;
|
||||
/* if start > regMask, bitstream is corrupted, and result is undefined */
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
/* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
|
||||
* than accessing memory. When bmi2 instruction is not present, we consider
|
||||
* such cpus old (pre-Haswell, 2013) and their performance is not of that
|
||||
* importance.
|
||||
*/
|
||||
#if defined(__x86_64__) || defined(_M_X86)
|
||||
return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
|
||||
#else
|
||||
return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
{
|
||||
#if defined(STATIC_BMI2) && STATIC_BMI2 == 1
|
||||
return _bzhi_u64(bitContainer, nbBits);
|
||||
#else
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
return bitContainer & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
/*! BIT_lookBits() :
|
||||
@@ -367,7 +362,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 co
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted */
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
/* arbitrate between double-shift and shift+mask */
|
||||
#if 1
|
||||
@@ -390,7 +385,7 @@ MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
@@ -399,7 +394,7 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value. */
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBits(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
@@ -416,23 +411,6 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStreamFast() :
|
||||
* Similar to BIT_reloadDStream(), but with two differences:
|
||||
* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
|
||||
* 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
|
||||
* point you must use BIT_reloadDStream() to reload.
|
||||
*/
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (UNLIKELY(bitD->ptr < bitD->limitPtr))
|
||||
return BIT_DStream_overflow;
|
||||
assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BIT_DStream_unfinished;
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStream() :
|
||||
* Refill `bitD` from buffer previously set in BIT_initDStream() .
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
@@ -444,7 +422,10 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
return BIT_DStream_overflow;
|
||||
|
||||
if (bitD->ptr >= bitD->limitPtr) {
|
||||
return BIT_reloadDStreamFast(bitD);
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BIT_DStream_unfinished;
|
||||
}
|
||||
if (bitD->ptr == bitD->start) {
|
||||
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
|
||||
@@ -477,5 +458,3 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
||||
#endif
|
||||
|
||||
#endif /* BITSTREAM_H_MODULE */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
137
vendor/github.com/DataDog/zstd/clevels.h
generated
vendored
137
vendor/github.com/DataDog/zstd/clevels.h
generated
vendored
@@ -1,137 +0,0 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_CLEVELS_H
|
||||
#define ZSTD_CLEVELS_H
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||
#include "zstd.h"
|
||||
|
||||
/*-===== Pre-defined compression levels =====-*/
|
||||
|
||||
#define ZSTD_MAX_CLEVEL 22
|
||||
|
||||
#ifdef __GNUC__
|
||||
__attribute__((__unused__))
|
||||
#endif
|
||||
|
||||
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
|
||||
{ /* "default" - for any srcSize > 256 KB */
|
||||
/* W, C, H, S, L, TL, strat */
|
||||
{ 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
|
||||
{ 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
|
||||
{ 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
|
||||
{ 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */
|
||||
{ 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */
|
||||
{ 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */
|
||||
{ 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */
|
||||
{ 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */
|
||||
{ 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */
|
||||
{ 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */
|
||||
{ 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */
|
||||
{ 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */
|
||||
{ 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */
|
||||
{ 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */
|
||||
{ 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */
|
||||
{ 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */
|
||||
{ 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */
|
||||
{ 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */
|
||||
{ 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */
|
||||
{ 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */
|
||||
{ 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */
|
||||
},
|
||||
{ /* for srcSize <= 256 KB */
|
||||
/* W, C, H, S, L, T, strat */
|
||||
{ 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
|
||||
{ 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */
|
||||
{ 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */
|
||||
{ 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */
|
||||
{ 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/
|
||||
{ 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/
|
||||
{ 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/
|
||||
{ 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */
|
||||
{ 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
|
||||
{ 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/
|
||||
{ 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/
|
||||
{ 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */
|
||||
{ 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
|
||||
{ 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/
|
||||
{ 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/
|
||||
{ 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/
|
||||
{ 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/
|
||||
{ 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
|
||||
{ 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/
|
||||
{ 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/
|
||||
{ 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/
|
||||
},
|
||||
{ /* for srcSize <= 128 KB */
|
||||
/* W, C, H, S, L, T, strat */
|
||||
{ 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
|
||||
{ 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */
|
||||
{ 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */
|
||||
{ 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */
|
||||
{ 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */
|
||||
{ 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */
|
||||
{ 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */
|
||||
{ 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */
|
||||
{ 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
|
||||
{ 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */
|
||||
{ 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */
|
||||
{ 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/
|
||||
{ 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
|
||||
{ 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/
|
||||
{ 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/
|
||||
{ 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/
|
||||
{ 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/
|
||||
{ 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/
|
||||
{ 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/
|
||||
{ 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
|
||||
{ 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/
|
||||
},
|
||||
{ /* for srcSize <= 16 KB */
|
||||
/* W, C, H, S, L, T, strat */
|
||||
{ 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
|
||||
{ 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
|
||||
{ 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
|
||||
{ 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */
|
||||
{ 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */
|
||||
{ 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/
|
||||
{ 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */
|
||||
{ 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */
|
||||
{ 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/
|
||||
{ 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/
|
||||
{ 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/
|
||||
{ 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/
|
||||
{ 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/
|
||||
{ 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/
|
||||
{ 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/
|
||||
{ 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/
|
||||
{ 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/
|
||||
{ 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/
|
||||
{ 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/
|
||||
{ 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
|
||||
{ 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/
|
||||
{ 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
|
||||
{ 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
|
||||
#endif /* ZSTD_CLEVELS_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
219
vendor/github.com/DataDog/zstd/compiler.h
generated
vendored
219
vendor/github.com/DataDog/zstd/compiler.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,15 +11,13 @@
|
||||
#ifndef ZSTD_COMPILER_H
|
||||
#define ZSTD_COMPILER_H
|
||||
|
||||
#include "portability_macros.h"
|
||||
|
||||
/*-*******************************************************
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
/* force inlining */
|
||||
|
||||
#if !defined(ZSTD_NO_INLINE)
|
||||
#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# define INLINE_KEYWORD inline
|
||||
#else
|
||||
# define INLINE_KEYWORD
|
||||
@@ -41,17 +38,6 @@
|
||||
|
||||
#endif
|
||||
|
||||
/**
|
||||
On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).
|
||||
This explicitly marks such functions as __cdecl so that the code will still compile
|
||||
if a CC other than __cdecl has been made the default.
|
||||
*/
|
||||
#if defined(_MSC_VER)
|
||||
# define WIN_CDECL __cdecl
|
||||
#else
|
||||
# define WIN_CDECL
|
||||
#endif
|
||||
|
||||
/**
|
||||
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
|
||||
* parameters. They must be inlined for the compiler to eliminate the constant
|
||||
@@ -93,19 +79,30 @@
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* target attribute */
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */
|
||||
#endif
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
|
||||
#else
|
||||
# define TARGET_ATTRIBUTE(target)
|
||||
#endif
|
||||
|
||||
/* Target attribute for BMI2 dynamic dispatch.
|
||||
* Enable lzcnt, bmi, and bmi2.
|
||||
* We test for bmi1 & bmi2. lzcnt is included in bmi1.
|
||||
/* Enable runtime BMI2 dispatch based on the CPU.
|
||||
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
||||
*/
|
||||
#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
|
||||
#ifndef DYNAMIC_BMI2
|
||||
#if ((defined(__clang__) && __has_attribute(__target__)) \
|
||||
|| (defined(__GNUC__) \
|
||||
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
|
||||
&& (defined(__x86_64__) || defined(_M_X86)) \
|
||||
&& !defined(__BMI2__)
|
||||
# define DYNAMIC_BMI2 1
|
||||
#else
|
||||
# define DYNAMIC_BMI2 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* prefetch
|
||||
* can be disabled, by declaring NO_PREFETCH build macro */
|
||||
@@ -120,9 +117,6 @@
|
||||
# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
|
||||
# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
|
||||
# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
|
||||
# elif defined(__aarch64__)
|
||||
# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr)))
|
||||
# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr)))
|
||||
# else
|
||||
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
|
||||
@@ -141,9 +135,8 @@
|
||||
}
|
||||
|
||||
/* vectorization
|
||||
* older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
|
||||
* and some compilers, like Intel ICC and MCST LCC, do not support it at all. */
|
||||
#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__)
|
||||
* older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */
|
||||
#if !defined(__clang__) && defined(__GNUC__)
|
||||
# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
|
||||
# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
|
||||
# else
|
||||
@@ -153,19 +146,6 @@
|
||||
# define DONT_VECTORIZE
|
||||
#endif
|
||||
|
||||
/* Tell the compiler that a branch is likely or unlikely.
|
||||
* Only use these macros if it causes the compiler to generate better code.
|
||||
* If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc
|
||||
* and clang, please do.
|
||||
*/
|
||||
#if defined(__GNUC__)
|
||||
#define LIKELY(x) (__builtin_expect((x), 1))
|
||||
#define UNLIKELY(x) (__builtin_expect((x), 0))
|
||||
#else
|
||||
#define LIKELY(x) (x)
|
||||
#define UNLIKELY(x) (x)
|
||||
#endif
|
||||
|
||||
/* disable warnings */
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
@@ -176,163 +156,4 @@
|
||||
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
||||
#endif
|
||||
|
||||
/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
|
||||
#ifndef STATIC_BMI2
|
||||
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
|
||||
# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
|
||||
# define STATIC_BMI2 1
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifndef STATIC_BMI2
|
||||
#define STATIC_BMI2 0
|
||||
#endif
|
||||
|
||||
/* compile time determination of SIMD support */
|
||||
#if !defined(ZSTD_NO_INTRINSICS)
|
||||
# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
|
||||
# define ZSTD_ARCH_X86_SSE2
|
||||
# endif
|
||||
# if defined(__ARM_NEON) || defined(_M_ARM64)
|
||||
# define ZSTD_ARCH_ARM_NEON
|
||||
# endif
|
||||
#
|
||||
# if defined(ZSTD_ARCH_X86_SSE2)
|
||||
# include <emmintrin.h>
|
||||
# elif defined(ZSTD_ARCH_ARM_NEON)
|
||||
# include <arm_neon.h>
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* C-language Attributes are added in C23. */
|
||||
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
|
||||
# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
|
||||
#else
|
||||
# define ZSTD_HAS_C_ATTRIBUTE(x) 0
|
||||
#endif
|
||||
|
||||
/* Only use C++ attributes in C++. Some compilers report support for C++
|
||||
* attributes when compiling with C.
|
||||
*/
|
||||
#if defined(__cplusplus) && defined(__has_cpp_attribute)
|
||||
# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
|
||||
#else
|
||||
# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0
|
||||
#endif
|
||||
|
||||
/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute.
|
||||
* - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough
|
||||
* - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough
|
||||
* - Else: __attribute__((__fallthrough__))
|
||||
*/
|
||||
#ifndef ZSTD_FALLTHROUGH
|
||||
# if ZSTD_HAS_C_ATTRIBUTE(fallthrough)
|
||||
# define ZSTD_FALLTHROUGH [[fallthrough]]
|
||||
# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough)
|
||||
# define ZSTD_FALLTHROUGH [[fallthrough]]
|
||||
# elif __has_attribute(__fallthrough__)
|
||||
/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon
|
||||
* gcc complains about: a label can only be part of a statement and a declaration is not a statement.
|
||||
*/
|
||||
# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__))
|
||||
# else
|
||||
# define ZSTD_FALLTHROUGH
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/*-**************************************************************
|
||||
* Alignment check
|
||||
*****************************************************************/
|
||||
|
||||
/* this test was initially positioned in mem.h,
|
||||
* but this file is removed (or replaced) for linux kernel
|
||||
* so it's now hosted in compiler.h,
|
||||
* which remains valid for both user & kernel spaces.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_ALIGNOF
|
||||
# if defined(__GNUC__) || defined(_MSC_VER)
|
||||
/* covers gcc, clang & MSVC */
|
||||
/* note : this section must come first, before C11,
|
||||
* due to a limitation in the kernel source generator */
|
||||
# define ZSTD_ALIGNOF(T) __alignof(T)
|
||||
|
||||
# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
|
||||
/* C11 support */
|
||||
# include <stdalign.h>
|
||||
# define ZSTD_ALIGNOF(T) alignof(T)
|
||||
|
||||
# else
|
||||
/* No known support for alignof() - imperfect backup */
|
||||
# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T))
|
||||
|
||||
# endif
|
||||
#endif /* ZSTD_ALIGNOF */
|
||||
|
||||
/*-**************************************************************
|
||||
* Sanitizer
|
||||
*****************************************************************/
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER
|
||||
/* Not all platforms that support msan provide sanitizers/msan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
* include the header file... */
|
||||
#include <stddef.h> /* size_t */
|
||||
#define ZSTD_DEPS_NEED_STDINT
|
||||
#include "zstd_deps.h" /* intptr_t */
|
||||
|
||||
/* Make memory region fully initialized (without changing its contents). */
|
||||
void __msan_unpoison(const volatile void *a, size_t size);
|
||||
|
||||
/* Make memory region fully uninitialized (without changing its contents).
|
||||
This is a legacy interface that does not update origin information. Use
|
||||
__msan_allocated_memory() instead. */
|
||||
void __msan_poison(const volatile void *a, size_t size);
|
||||
|
||||
/* Returns the offset of the first (at least partially) poisoned byte in the
|
||||
memory range, or -1 if the whole range is good. */
|
||||
intptr_t __msan_test_shadow(const volatile void *x, size_t size);
|
||||
#endif
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER
|
||||
/* Not all platforms that support asan provide sanitizers/asan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
* include the header file... */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
/**
|
||||
* Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
|
||||
*
|
||||
* This memory must be previously allocated by your program. Instrumented
|
||||
* code is forbidden from accessing addresses in this region until it is
|
||||
* unpoisoned. This function is not guaranteed to poison the entire region -
|
||||
* it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
|
||||
* alignment restrictions.
|
||||
*
|
||||
* \note This function is not thread-safe because no two threads can poison or
|
||||
* unpoison memory in the same memory region simultaneously.
|
||||
*
|
||||
* \param addr Start of memory region.
|
||||
* \param size Size of memory region. */
|
||||
void __asan_poison_memory_region(void const volatile *addr, size_t size);
|
||||
|
||||
/**
|
||||
* Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
|
||||
*
|
||||
* This memory must be previously allocated by your program. Accessing
|
||||
* addresses in this region is allowed until this region is poisoned again.
|
||||
* This function could unpoison a super-region of <c>[addr, addr+size)</c> due
|
||||
* to ASan alignment restrictions.
|
||||
*
|
||||
* \note This function is not thread-safe because no two threads can
|
||||
* poison or unpoison memory in the same memory region simultaneously.
|
||||
*
|
||||
* \param addr Start of memory region.
|
||||
* \param size Size of memory region. */
|
||||
void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_COMPILER_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
96
vendor/github.com/DataDog/zstd/cover.c
generated
vendored
96
vendor/github.com/DataDog/zstd/cover.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -27,64 +26,47 @@
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* read */
|
||||
#include "pool.h"
|
||||
#include "threading.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "zdict.h"
|
||||
#include "cover.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
/**
|
||||
* There are 32bit indexes used to ref samples, so limit samples size to 4GB
|
||||
* on 64bit builds.
|
||||
* For 32bit builds we choose 1 GB.
|
||||
* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large
|
||||
* contiguous buffer, so 1GB is already a high limit.
|
||||
*/
|
||||
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
||||
#define COVER_DEFAULT_SPLITPOINT 1.0
|
||||
#define DEFAULT_SPLITPOINT 1.0
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#ifndef LOCALDISPLAYLEVEL
|
||||
static int g_displayLevel = 0;
|
||||
#endif
|
||||
#undef DISPLAY
|
||||
static int g_displayLevel = 2;
|
||||
#define DISPLAY(...) \
|
||||
{ \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
fflush(stderr); \
|
||||
}
|
||||
#undef LOCALDISPLAYLEVEL
|
||||
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||
#undef DISPLAYLEVEL
|
||||
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
||||
|
||||
#ifndef LOCALDISPLAYUPDATE
|
||||
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
#endif
|
||||
#undef LOCALDISPLAYUPDATE
|
||||
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
||||
if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
|
||||
g_time = clock(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} \
|
||||
}
|
||||
#undef DISPLAYUPDATE
|
||||
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
/*-*************************************
|
||||
* Hash table
|
||||
@@ -138,9 +120,9 @@ static int COVER_map_init(COVER_map_t *map, U32 size) {
|
||||
/**
|
||||
* Internal hash function
|
||||
*/
|
||||
static const U32 COVER_prime4bytes = 2654435761U;
|
||||
static const U32 prime4bytes = 2654435761U;
|
||||
static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
|
||||
return (key * COVER_prime4bytes) >> (32 - map->sizeLog);
|
||||
return (key * prime4bytes) >> (32 - map->sizeLog);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -233,7 +215,7 @@ typedef struct {
|
||||
} COVER_ctx_t;
|
||||
|
||||
/* We need a global context for qsort... */
|
||||
static COVER_ctx_t *g_coverCtx = NULL;
|
||||
static COVER_ctx_t *g_ctx = NULL;
|
||||
|
||||
/*-*************************************
|
||||
* Helper functions
|
||||
@@ -276,11 +258,11 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
|
||||
|
||||
/**
|
||||
* Same as COVER_cmp() except ties are broken by pointer value
|
||||
* NOTE: g_coverCtx must be set to call this function. A global is required because
|
||||
* NOTE: g_ctx must be set to call this function. A global is required because
|
||||
* qsort doesn't take an opaque pointer.
|
||||
*/
|
||||
static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
int result = COVER_cmp(g_coverCtx, lp, rp);
|
||||
static int COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
int result = COVER_cmp(g_ctx, lp, rp);
|
||||
if (result == 0) {
|
||||
result = lp < rp ? -1 : 1;
|
||||
}
|
||||
@@ -289,8 +271,8 @@ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
/**
|
||||
* Faster version for d <= 8.
|
||||
*/
|
||||
static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
|
||||
int result = COVER_cmp8(g_coverCtx, lp, rp);
|
||||
static int COVER_strict_cmp8(const void *lp, const void *rp) {
|
||||
int result = COVER_cmp8(g_ctx, lp, rp);
|
||||
if (result == 0) {
|
||||
result = lp < rp ? -1 : 1;
|
||||
}
|
||||
@@ -621,7 +603,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
/* qsort doesn't take an opaque pointer, so pass as a global.
|
||||
* On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
|
||||
*/
|
||||
g_coverCtx = ctx;
|
||||
g_ctx = ctx;
|
||||
#if defined(__OpenBSD__)
|
||||
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
@@ -743,7 +725,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
||||
COVER_map_t activeDmers;
|
||||
parameters.splitPoint = 1.0;
|
||||
/* Initialize global data */
|
||||
g_displayLevel = (int)parameters.zParams.notificationLevel;
|
||||
g_displayLevel = parameters.zParams.notificationLevel;
|
||||
/* Checks */
|
||||
if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
|
||||
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
|
||||
@@ -964,7 +946,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection){
|
||||
free(selection.dictContent);
|
||||
}
|
||||
|
||||
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
|
||||
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
||||
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
|
||||
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
|
||||
|
||||
@@ -972,8 +954,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
|
||||
size_t largestCompressed = 0;
|
||||
BYTE* customDictContentEnd = customDictContent + dictContentSize;
|
||||
|
||||
BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
|
||||
BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
|
||||
BYTE * largestDictbuffer = (BYTE *)malloc(dictContentSize);
|
||||
BYTE * candidateDictBuffer = (BYTE *)malloc(dictContentSize);
|
||||
double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
|
||||
|
||||
if (!largestDictbuffer || !candidateDictBuffer) {
|
||||
@@ -985,7 +967,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
|
||||
/* Initial dictionary size and compressed size */
|
||||
memcpy(largestDictbuffer, customDictContent, dictContentSize);
|
||||
dictContentSize = ZDICT_finalizeDictionary(
|
||||
largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
|
||||
largestDictbuffer, dictContentSize, customDictContent, dictContentSize,
|
||||
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
|
||||
|
||||
if (ZDICT_isError(dictContentSize)) {
|
||||
@@ -1019,7 +1001,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
|
||||
while (dictContentSize < largestDict) {
|
||||
memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
|
||||
dictContentSize = ZDICT_finalizeDictionary(
|
||||
candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
|
||||
candidateDictBuffer, dictContentSize, customDictContentEnd - dictContentSize, dictContentSize,
|
||||
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
|
||||
|
||||
if (ZDICT_isError(dictContentSize)) {
|
||||
@@ -1071,19 +1053,18 @@ typedef struct COVER_tryParameters_data_s {
|
||||
* This function is thread safe if zstd is compiled with multithreaded support.
|
||||
* It takes its parameters as an *OWNING* opaque pointer to support threading.
|
||||
*/
|
||||
static void COVER_tryParameters(void *opaque)
|
||||
{
|
||||
static void COVER_tryParameters(void *opaque) {
|
||||
/* Save parameters as local variables */
|
||||
COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t*)opaque;
|
||||
COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque;
|
||||
const COVER_ctx_t *const ctx = data->ctx;
|
||||
const ZDICT_cover_params_t parameters = data->parameters;
|
||||
size_t dictBufferCapacity = data->dictBufferCapacity;
|
||||
size_t totalCompressedSize = ERROR(GENERIC);
|
||||
/* Allocate space for hash table, dict, and freqs */
|
||||
COVER_map_t activeDmers;
|
||||
BYTE* const dict = (BYTE*)malloc(dictBufferCapacity);
|
||||
BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
|
||||
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
|
||||
U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32));
|
||||
U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
|
||||
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
|
||||
goto _cleanup;
|
||||
@@ -1098,7 +1079,7 @@ static void COVER_tryParameters(void *opaque)
|
||||
{
|
||||
const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
|
||||
dictBufferCapacity, parameters);
|
||||
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
|
||||
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
|
||||
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
|
||||
totalCompressedSize);
|
||||
|
||||
@@ -1113,18 +1094,19 @@ _cleanup:
|
||||
free(data);
|
||||
COVER_map_destroy(&activeDmers);
|
||||
COVER_dictSelectionFree(selection);
|
||||
free(freqs);
|
||||
if (freqs) {
|
||||
free(freqs);
|
||||
}
|
||||
}
|
||||
|
||||
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer,
|
||||
const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_cover_params_t* parameters)
|
||||
{
|
||||
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_cover_params_t *parameters) {
|
||||
/* constants */
|
||||
const unsigned nbThreads = parameters->nbThreads;
|
||||
const double splitPoint =
|
||||
parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
|
||||
parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
|
||||
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
||||
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
||||
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
||||
@@ -1252,5 +1234,3 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
return dictSize;
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
22
vendor/github.com/DataDog/zstd/cover.h
generated
vendored
22
vendor/github.com/DataDog/zstd/cover.h
generated
vendored
@@ -1,18 +1,3 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memset */
|
||||
@@ -21,6 +6,9 @@
|
||||
#include "pool.h"
|
||||
#include "threading.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
/**
|
||||
@@ -154,8 +142,6 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
|
||||
* smallest dictionary within a specified regression of the compressed size
|
||||
* from the largest dictionary.
|
||||
*/
|
||||
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
|
||||
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
||||
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
|
||||
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
7
vendor/github.com/DataDog/zstd/cpu.h
generated
vendored
7
vendor/github.com/DataDog/zstd/cpu.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) 2018-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -17,6 +16,8 @@
|
||||
* https://github.com/facebook/folly/blob/master/folly/CpuId.h
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "mem.h"
|
||||
|
||||
#ifdef _MSC_VER
|
||||
@@ -212,5 +213,3 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
|
||||
#undef X
|
||||
|
||||
#endif /* ZSTD_COMMON_CPU_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
45
vendor/github.com/DataDog/zstd/debug.c
generated
vendored
45
vendor/github.com/DataDog/zstd/debug.c
generated
vendored
@@ -1,16 +1,35 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* debug
|
||||
* Part of FSE library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
debug
|
||||
Part of FSE library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
@@ -23,5 +42,3 @@
|
||||
#include "debug.h"
|
||||
|
||||
int g_debuglevel = DEBUGLEVEL;
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
74
vendor/github.com/DataDog/zstd/debug.h
generated
vendored
74
vendor/github.com/DataDog/zstd/debug.h
generated
vendored
@@ -1,16 +1,35 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* debug
|
||||
* Part of FSE library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
debug
|
||||
Part of FSE library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
@@ -52,6 +71,15 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* DEBUGFILE can be defined externally,
|
||||
* typically through compiler command line.
|
||||
* note : currently useless.
|
||||
* Value must be stderr or stdout */
|
||||
#ifndef DEBUGFILE
|
||||
# define DEBUGFILE stderr
|
||||
#endif
|
||||
|
||||
|
||||
/* recommended values for DEBUGLEVEL :
|
||||
* 0 : release mode, no debug, all run-time checks disabled
|
||||
* 1 : enables assert() only, no display
|
||||
@@ -68,8 +96,7 @@ extern "C" {
|
||||
*/
|
||||
|
||||
#if (DEBUGLEVEL>=1)
|
||||
# define ZSTD_DEPS_NEED_ASSERT
|
||||
# include "zstd_deps.h"
|
||||
# include <assert.h>
|
||||
#else
|
||||
# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
|
||||
# define assert(condition) ((void)0) /* disable assert (default) */
|
||||
@@ -77,8 +104,7 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
#if (DEBUGLEVEL>=2)
|
||||
# define ZSTD_DEPS_NEED_IO
|
||||
# include "zstd_deps.h"
|
||||
# include <stdio.h>
|
||||
extern int g_debuglevel; /* the variable is only declared,
|
||||
it actually lives in debug.c,
|
||||
and is shared by the whole process.
|
||||
@@ -86,14 +112,14 @@ extern int g_debuglevel; /* the variable is only declared,
|
||||
It's useful when enabling very verbose levels
|
||||
on selective conditions (such as position in src) */
|
||||
|
||||
# define RAWLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
ZSTD_DEBUG_PRINT(__VA_ARGS__); \
|
||||
# define RAWLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
} }
|
||||
# define DEBUGLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \
|
||||
ZSTD_DEBUG_PRINT(" \n"); \
|
||||
# define DEBUGLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
|
||||
fprintf(stderr, " \n"); \
|
||||
} }
|
||||
#else
|
||||
# define RAWLOG(l, ...) {} /* disabled */
|
||||
@@ -106,5 +132,3 @@ extern int g_debuglevel; /* the variable is only declared,
|
||||
#endif
|
||||
|
||||
#endif /* DEBUG_H_12987983217 */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
5
vendor/github.com/DataDog/zstd/divsufsort.c
generated
vendored
5
vendor/github.com/DataDog/zstd/divsufsort.c
generated
vendored
@@ -1,4 +1,3 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* divsufsort.c for libdivsufsort-lite
|
||||
* Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
|
||||
@@ -1577,7 +1576,7 @@ note:
|
||||
/* Construct the inverse suffix array of type B* suffixes using trsort. */
|
||||
trsort(ISAb, SA, m, 1);
|
||||
|
||||
/* Set the sorted order of type B* suffixes. */
|
||||
/* Set the sorted order of tyoe B* suffixes. */
|
||||
for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) {
|
||||
for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { }
|
||||
if(0 <= i) {
|
||||
@@ -1912,5 +1911,3 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
|
||||
|
||||
return pidx;
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
3
vendor/github.com/DataDog/zstd/divsufsort.h
generated
vendored
3
vendor/github.com/DataDog/zstd/divsufsort.h
generated
vendored
@@ -1,4 +1,3 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* divsufsort.h for libdivsufsort-lite
|
||||
* Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
|
||||
@@ -66,5 +65,3 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
|
||||
#endif /* __cplusplus */
|
||||
|
||||
#endif /* _DIVSUFSORT_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
287
vendor/github.com/DataDog/zstd/entropy_common.c
generated
vendored
287
vendor/github.com/DataDog/zstd/entropy_common.c
generated
vendored
@@ -1,17 +1,36 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* Common functions of New Generation Entropy library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
****************************************************************** */
|
||||
/*
|
||||
Common functions of New Generation Entropy library
|
||||
Copyright (C) 2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*************************************************************************** */
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
@@ -39,37 +58,8 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
****************************************************************/
|
||||
static U32 FSE_ctz(U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return __builtin_ctz(val);
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
return __CTZ(val);
|
||||
# else /* Software version */
|
||||
U32 count = 0;
|
||||
while ((val & 1) == 0) {
|
||||
val >>= 1;
|
||||
++count;
|
||||
}
|
||||
return count;
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) headerBuffer;
|
||||
const BYTE* const iend = istart + hbSize;
|
||||
@@ -80,23 +70,23 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
||||
U32 bitStream;
|
||||
int bitCount;
|
||||
unsigned charnum = 0;
|
||||
unsigned const maxSV1 = *maxSVPtr + 1;
|
||||
int previous0 = 0;
|
||||
|
||||
if (hbSize < 8) {
|
||||
/* This function only works when hbSize >= 8 */
|
||||
char buffer[8] = {0};
|
||||
ZSTD_memcpy(buffer, headerBuffer, hbSize);
|
||||
if (hbSize < 4) {
|
||||
/* This function only works when hbSize >= 4 */
|
||||
char buffer[4];
|
||||
memset(buffer, 0, sizeof(buffer));
|
||||
memcpy(buffer, headerBuffer, hbSize);
|
||||
{ size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
|
||||
buffer, sizeof(buffer));
|
||||
if (FSE_isError(countSize)) return countSize;
|
||||
if (countSize > hbSize) return ERROR(corruption_detected);
|
||||
return countSize;
|
||||
} }
|
||||
assert(hbSize >= 8);
|
||||
assert(hbSize >= 4);
|
||||
|
||||
/* init */
|
||||
ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
|
||||
memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
|
||||
bitStream = MEM_readLE32(ip);
|
||||
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
|
||||
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
||||
@@ -107,58 +97,36 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
||||
threshold = 1<<nbBits;
|
||||
nbBits++;
|
||||
|
||||
for (;;) {
|
||||
while ((remaining>1) & (charnum<=*maxSVPtr)) {
|
||||
if (previous0) {
|
||||
/* Count the number of repeats. Each time the
|
||||
* 2-bit repeat code is 0b11 there is another
|
||||
* repeat.
|
||||
* Avoid UB by setting the high bit to 1.
|
||||
*/
|
||||
int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
|
||||
while (repeats >= 12) {
|
||||
charnum += 3 * 12;
|
||||
if (LIKELY(ip <= iend-7)) {
|
||||
ip += 3;
|
||||
unsigned n0 = charnum;
|
||||
while ((bitStream & 0xFFFF) == 0xFFFF) {
|
||||
n0 += 24;
|
||||
if (ip < iend-5) {
|
||||
ip += 2;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitCount -= (int)(8 * (iend - 7 - ip));
|
||||
bitCount &= 31;
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
|
||||
bitStream >>= 16;
|
||||
bitCount += 16;
|
||||
} }
|
||||
while ((bitStream & 3) == 3) {
|
||||
n0 += 3;
|
||||
bitStream >>= 2;
|
||||
bitCount += 2;
|
||||
}
|
||||
charnum += 3 * repeats;
|
||||
bitStream >>= 2 * repeats;
|
||||
bitCount += 2 * repeats;
|
||||
|
||||
/* Add the final repeat which isn't 0b11. */
|
||||
assert((bitStream & 3) < 3);
|
||||
charnum += bitStream & 3;
|
||||
n0 += bitStream & 3;
|
||||
bitCount += 2;
|
||||
|
||||
/* This is an error, but break and return an error
|
||||
* at the end, because returning out of a loop makes
|
||||
* it harder for the compiler to optimize.
|
||||
*/
|
||||
if (charnum >= maxSV1) break;
|
||||
|
||||
/* We don't need to set the normalized count to 0
|
||||
* because we already memset the whole buffer to 0.
|
||||
*/
|
||||
|
||||
if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
|
||||
while (charnum < n0) normalizedCounter[charnum++] = 0;
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
assert((bitCount >> 3) <= 3); /* For first condition to work */
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitCount -= (int)(8 * (iend - 4 - ip));
|
||||
bitCount &= 31;
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
}
|
||||
{
|
||||
int const max = (2*threshold-1) - remaining;
|
||||
bitStream >>= 2;
|
||||
} }
|
||||
{ int const max = (2*threshold-1) - remaining;
|
||||
int count;
|
||||
|
||||
if ((bitStream & (threshold-1)) < (U32)max) {
|
||||
@@ -171,43 +139,24 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
||||
}
|
||||
|
||||
count--; /* extra accuracy */
|
||||
/* When it matters (small blocks), this is a
|
||||
* predictable branch, because we don't use -1.
|
||||
*/
|
||||
if (count >= 0) {
|
||||
remaining -= count;
|
||||
} else {
|
||||
assert(count == -1);
|
||||
remaining += count;
|
||||
}
|
||||
remaining -= count < 0 ? -count : count; /* -1 means +1 */
|
||||
normalizedCounter[charnum++] = (short)count;
|
||||
previous0 = !count;
|
||||
|
||||
assert(threshold > 1);
|
||||
if (remaining < threshold) {
|
||||
/* This branch can be folded into the
|
||||
* threshold update condition because we
|
||||
* know that threshold > 1.
|
||||
*/
|
||||
if (remaining <= 1) break;
|
||||
nbBits = BIT_highbit32(remaining) + 1;
|
||||
threshold = 1 << (nbBits - 1);
|
||||
while (remaining < threshold) {
|
||||
nbBits--;
|
||||
threshold >>= 1;
|
||||
}
|
||||
if (charnum >= maxSV1) break;
|
||||
|
||||
if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
} else {
|
||||
bitCount -= (int)(8 * (iend - 4 - ip));
|
||||
bitCount &= 31;
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} }
|
||||
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
||||
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
|
||||
if (remaining != 1) return ERROR(corruption_detected);
|
||||
/* Only possible when there are too many zeros. */
|
||||
if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall);
|
||||
if (bitCount > 32) return ERROR(corruption_detected);
|
||||
*maxSVPtr = charnum-1;
|
||||
|
||||
@@ -215,43 +164,6 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
/* Avoids the FORCE_INLINE of the _body() function. */
|
||||
static size_t FSE_readNCount_body_default(
|
||||
short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
|
||||
}
|
||||
|
||||
#if DYNAMIC_BMI2
|
||||
BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
|
||||
short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
|
||||
}
|
||||
#endif
|
||||
|
||||
size_t FSE_readNCount_bmi2(
|
||||
short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize, int bmi2)
|
||||
{
|
||||
#if DYNAMIC_BMI2
|
||||
if (bmi2) {
|
||||
return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
|
||||
}
|
||||
#endif
|
||||
(void)bmi2;
|
||||
return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
|
||||
}
|
||||
|
||||
size_t FSE_readNCount(
|
||||
short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
|
||||
/*! HUF_readStats() :
|
||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
@@ -263,17 +175,6 @@ size_t FSE_readNCount(
|
||||
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
|
||||
return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t
|
||||
HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize,
|
||||
void* workSpace, size_t wkspSize,
|
||||
int bmi2)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
@@ -282,7 +183,7 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
|
||||
if (!srcSize) return ERROR(srcSize_wrong);
|
||||
iSize = ip[0];
|
||||
/* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
|
||||
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
oSize = iSize - 127;
|
||||
@@ -296,17 +197,17 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
/* max (hwSize-1) values decoded, as last one is implied */
|
||||
oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2);
|
||||
oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSE_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
|
||||
memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
|
||||
if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
@@ -333,39 +234,3 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
|
||||
/* Avoids the FORCE_INLINE of the _body() function. */
|
||||
static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
|
||||
}
|
||||
|
||||
#if DYNAMIC_BMI2
|
||||
static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize,
|
||||
void* workSpace, size_t wkspSize,
|
||||
int bmi2)
|
||||
{
|
||||
#if DYNAMIC_BMI2
|
||||
if (bmi2) {
|
||||
return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
|
||||
}
|
||||
#endif
|
||||
(void)bmi2;
|
||||
return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
7
vendor/github.com/DataDog/zstd/error_private.c
generated
vendored
7
vendor/github.com/DataDog/zstd/error_private.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -48,12 +47,8 @@ const char* ERR_getErrorString(ERR_enum code)
|
||||
/* following error codes are not stable and may be removed or changed in a future version */
|
||||
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
|
||||
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
||||
case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
|
||||
case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
92
vendor/github.com/DataDog/zstd/error_private.h
generated
vendored
92
vendor/github.com/DataDog/zstd/error_private.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -22,10 +21,8 @@ extern "C" {
|
||||
/* ****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd_errors.h" /* enum list */
|
||||
#include "compiler.h"
|
||||
#include "debug.h"
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
@@ -52,7 +49,7 @@ typedef ZSTD_ErrorCode ERR_enum;
|
||||
/*-****************************************
|
||||
* Error codes handling
|
||||
******************************************/
|
||||
#undef ERROR /* already defined on Visual Studio */
|
||||
#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
|
||||
#define ERROR(name) ZSTD_ERROR(name)
|
||||
#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
|
||||
|
||||
@@ -60,10 +57,6 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
|
||||
|
||||
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
|
||||
|
||||
/* check and forward error code */
|
||||
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
|
||||
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Error Strings
|
||||
@@ -76,87 +69,8 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
return ERR_getErrorString(ERR_getErrorCode(code));
|
||||
}
|
||||
|
||||
/**
|
||||
* Ignore: this is an internal helper.
|
||||
*
|
||||
* This is a helper function to help force C99-correctness during compilation.
|
||||
* Under strict compilation modes, variadic macro arguments can't be empty.
|
||||
* However, variadic function arguments can be. Using a function therefore lets
|
||||
* us statically check that at least one (string) argument was passed,
|
||||
* independent of the compilation flags.
|
||||
*/
|
||||
static INLINE_KEYWORD UNUSED_ATTR
|
||||
void _force_has_format_string(const char *format, ...) {
|
||||
(void)format;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ignore: this is an internal helper.
|
||||
*
|
||||
* We want to force this function invocation to be syntactically correct, but
|
||||
* we don't want to force runtime evaluation of its arguments.
|
||||
*/
|
||||
#define _FORCE_HAS_FORMAT_STRING(...) \
|
||||
if (0) { \
|
||||
_force_has_format_string(__VA_ARGS__); \
|
||||
}
|
||||
|
||||
#define ERR_QUOTE(str) #str
|
||||
|
||||
/**
|
||||
* Return the specified error if the condition evaluates to true.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
* In order to do that (particularly, printing the conditional that failed),
|
||||
* this can't just wrap RETURN_ERROR().
|
||||
*/
|
||||
#define RETURN_ERROR_IF(cond, err, ...) \
|
||||
if (cond) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
}
|
||||
|
||||
/**
|
||||
* Unconditionally return the specified error.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
*/
|
||||
#define RETURN_ERROR(err, ...) \
|
||||
do { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
} while(0);
|
||||
|
||||
/**
|
||||
* If the provided expression evaluates to an error code, returns that error code.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
*/
|
||||
#define FORWARD_IF_ERROR(err, ...) \
|
||||
do { \
|
||||
size_t const err_code = (err); \
|
||||
if (ERR_isError(err_code)) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return err_code; \
|
||||
} \
|
||||
} while(0);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ERROR_H_MODULE */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
1
vendor/github.com/DataDog/zstd/errors.go
generated
vendored
1
vendor/github.com/DataDog/zstd/errors.go
generated
vendored
@@ -1,6 +1,7 @@
|
||||
package zstd
|
||||
|
||||
/*
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
*/
|
||||
import "C"
|
||||
|
||||
14
vendor/github.com/DataDog/zstd/external_zstd.go
generated
vendored
14
vendor/github.com/DataDog/zstd/external_zstd.go
generated
vendored
@@ -1,14 +0,0 @@
|
||||
//go:build external_libzstd
|
||||
// +build external_libzstd
|
||||
|
||||
package zstd
|
||||
|
||||
// #cgo CFLAGS: -DUSE_EXTERNAL_ZSTD
|
||||
// #cgo pkg-config: libzstd
|
||||
/*
|
||||
#include<zstd.h>
|
||||
#if ZSTD_VERSION_NUMBER < 10400
|
||||
#error "ZSTD version >= 1.4 is required"
|
||||
#endif
|
||||
*/
|
||||
import "C"
|
||||
88
vendor/github.com/DataDog/zstd/fastcover.c
generated
vendored
88
vendor/github.com/DataDog/zstd/fastcover.c
generated
vendored
@@ -1,14 +1,3 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
@@ -17,33 +6,24 @@
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* read */
|
||||
#include "pool.h"
|
||||
#include "threading.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "zstd_compress_internal.h" /* ZSTD_hash*() */
|
||||
#include "zdict.h"
|
||||
#include "cover.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
/**
|
||||
* There are 32bit indexes used to ref samples, so limit samples size to 4GB
|
||||
* on 64bit builds.
|
||||
* For 32bit builds we choose 1 GB.
|
||||
* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large
|
||||
* contiguous buffer, so 1GB is already a high limit.
|
||||
*/
|
||||
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
||||
#define FASTCOVER_MAX_F 31
|
||||
#define FASTCOVER_MAX_ACCEL 10
|
||||
#define FASTCOVER_DEFAULT_SPLITPOINT 0.75
|
||||
#define DEFAULT_SPLITPOINT 0.75
|
||||
#define DEFAULT_F 20
|
||||
#define DEFAULT_ACCEL 1
|
||||
|
||||
@@ -51,50 +31,50 @@
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#ifndef LOCALDISPLAYLEVEL
|
||||
static int g_displayLevel = 0;
|
||||
#endif
|
||||
#undef DISPLAY
|
||||
static int g_displayLevel = 2;
|
||||
#define DISPLAY(...) \
|
||||
{ \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
fflush(stderr); \
|
||||
}
|
||||
#undef LOCALDISPLAYLEVEL
|
||||
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||
#undef DISPLAYLEVEL
|
||||
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
||||
|
||||
#ifndef LOCALDISPLAYUPDATE
|
||||
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
#endif
|
||||
#undef LOCALDISPLAYUPDATE
|
||||
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
||||
if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
|
||||
g_time = clock(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} \
|
||||
}
|
||||
#undef DISPLAYUPDATE
|
||||
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Hash Functions
|
||||
***************************************/
|
||||
static const U64 prime6bytes = 227718039650203ULL;
|
||||
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
|
||||
|
||||
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
|
||||
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
|
||||
|
||||
|
||||
/**
|
||||
* Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
|
||||
* Hash the d-byte value pointed to by p and mod 2^f
|
||||
*/
|
||||
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 f, unsigned d) {
|
||||
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
|
||||
if (d == 6) {
|
||||
return ZSTD_hash6Ptr(p, f);
|
||||
return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
|
||||
}
|
||||
return ZSTD_hash8Ptr(p, f);
|
||||
return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
|
||||
}
|
||||
|
||||
|
||||
@@ -471,20 +451,20 @@ typedef struct FASTCOVER_tryParameters_data_s {
|
||||
* This function is thread safe if zstd is compiled with multithreaded support.
|
||||
* It takes its parameters as an *OWNING* opaque pointer to support threading.
|
||||
*/
|
||||
static void FASTCOVER_tryParameters(void* opaque)
|
||||
static void FASTCOVER_tryParameters(void *opaque)
|
||||
{
|
||||
/* Save parameters as local variables */
|
||||
FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t*)opaque;
|
||||
FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
|
||||
const FASTCOVER_ctx_t *const ctx = data->ctx;
|
||||
const ZDICT_cover_params_t parameters = data->parameters;
|
||||
size_t dictBufferCapacity = data->dictBufferCapacity;
|
||||
size_t totalCompressedSize = ERROR(GENERIC);
|
||||
/* Initialize array to keep track of frequency of dmer within activeSegment */
|
||||
U16* segmentFreqs = (U16*)calloc(((U64)1 << ctx->f), sizeof(U16));
|
||||
U16* segmentFreqs = (U16 *)calloc(((U64)1 << ctx->f), sizeof(U16));
|
||||
/* Allocate space for hash table, dict, and freqs */
|
||||
BYTE *const dict = (BYTE*)malloc(dictBufferCapacity);
|
||||
BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
|
||||
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
|
||||
U32* freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
|
||||
U32 *freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
|
||||
if (!segmentFreqs || !dict || !freqs) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
|
||||
goto _cleanup;
|
||||
@@ -496,7 +476,7 @@ static void FASTCOVER_tryParameters(void* opaque)
|
||||
parameters, segmentFreqs);
|
||||
|
||||
const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
|
||||
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
|
||||
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
|
||||
ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
|
||||
totalCompressedSize);
|
||||
|
||||
@@ -557,7 +537,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
|
||||
ZDICT_cover_params_t coverParams;
|
||||
FASTCOVER_accel_t accelParams;
|
||||
/* Initialize global data */
|
||||
g_displayLevel = (int)parameters.zParams.notificationLevel;
|
||||
g_displayLevel = parameters.zParams.notificationLevel;
|
||||
/* Assign splitPoint and f if not provided */
|
||||
parameters.splitPoint = 1.0;
|
||||
parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
|
||||
@@ -627,7 +607,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
/* constants */
|
||||
const unsigned nbThreads = parameters->nbThreads;
|
||||
const double splitPoint =
|
||||
parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
|
||||
parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
|
||||
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
||||
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
||||
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
||||
@@ -640,7 +620,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel;
|
||||
const unsigned shrinkDict = 0;
|
||||
/* Local variables */
|
||||
const int displayLevel = (int)parameters->zParams.notificationLevel;
|
||||
const int displayLevel = parameters->zParams.notificationLevel;
|
||||
unsigned iteration = 1;
|
||||
unsigned d;
|
||||
unsigned k;
|
||||
@@ -724,7 +704,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
data->parameters.splitPoint = splitPoint;
|
||||
data->parameters.steps = kSteps;
|
||||
data->parameters.shrinkDict = shrinkDict;
|
||||
data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel;
|
||||
data->parameters.zParams.notificationLevel = g_displayLevel;
|
||||
/* Check the parameters */
|
||||
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
|
||||
data->ctx->f, accel)) {
|
||||
@@ -765,5 +745,3 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
96
vendor/github.com/DataDog/zstd/fse.h
generated
vendored
96
vendor/github.com/DataDog/zstd/fse.h
generated
vendored
@@ -1,16 +1,35 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* FSE : Finite State Entropy codec
|
||||
* Public Prototypes declaration
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
FSE : Finite State Entropy codec
|
||||
Public Prototypes declaration
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
@@ -24,7 +43,7 @@ extern "C" {
|
||||
/*-*****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include "zstd_deps.h" /* size_t, ptrdiff_t */
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
@@ -138,16 +157,10 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
|
||||
/*! FSE_normalizeCount():
|
||||
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
|
||||
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
|
||||
useLowProbCount is a boolean parameter which trades off compressed size for
|
||||
faster header decoding. When it is set to 1, the compressed data will be slightly
|
||||
smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
|
||||
faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
|
||||
is a good default, since header deserialization makes a big speed difference.
|
||||
Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
|
||||
@return : tableLog,
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
|
||||
const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount);
|
||||
const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
|
||||
|
||||
/*! FSE_NCountWriteBound():
|
||||
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
|
||||
@@ -235,13 +248,6 @@ FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
|
||||
unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
|
||||
const void* rBuffer, size_t rBuffSize);
|
||||
|
||||
/*! FSE_readNCount_bmi2():
|
||||
* Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
|
||||
*/
|
||||
FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
|
||||
unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
|
||||
const void* rBuffer, size_t rBuffSize, int bmi2);
|
||||
|
||||
/*! Constructor and Destructor of FSE_DTable.
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
@@ -302,12 +308,12 @@ If there is an error, the function will return an error code, which can be teste
|
||||
*******************************************/
|
||||
/* FSE buffer bounds */
|
||||
#define FSE_NCOUNTBOUND 512
|
||||
#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)
|
||||
#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)
|
||||
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog)))
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
|
||||
#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
|
||||
@@ -323,9 +329,9 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
|
||||
|
||||
/* FSE_compress_wksp() :
|
||||
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
|
||||
* FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
|
||||
*/
|
||||
#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
|
||||
#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
|
||||
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||
|
||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
||||
@@ -336,31 +342,18 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
||||
|
||||
/* FSE_buildCTable_wksp() :
|
||||
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
|
||||
* See FSE_buildCTable_wksp() for breakdown of workspace usage.
|
||||
* `wkspSize` must be >= `(1<<tableLog)`.
|
||||
*/
|
||||
#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */)
|
||||
#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
|
||||
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||
|
||||
#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
|
||||
#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned))
|
||||
FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||
/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
||||
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
|
||||
#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
|
||||
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
|
||||
|
||||
size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
|
||||
/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
|
||||
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
|
||||
|
||||
typedef enum {
|
||||
FSE_repeat_none, /**< Cannot use the previous table */
|
||||
@@ -671,9 +664,6 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
#ifndef FSE_DEFAULT_MEMORY_USAGE
|
||||
# define FSE_DEFAULT_MEMORY_USAGE 13
|
||||
#endif
|
||||
#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE)
|
||||
# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE"
|
||||
#endif
|
||||
|
||||
/*!FSE_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
@@ -707,7 +697,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
|
||||
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
|
||||
|
||||
#endif /* FSE_STATIC_LINKING_ONLY */
|
||||
@@ -716,5 +706,3 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
181
vendor/github.com/DataDog/zstd/fse_compress.c
generated
vendored
181
vendor/github.com/DataDog/zstd/fse_compress.c
generated
vendored
@@ -1,21 +1,42 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* FSE : Finite State Entropy encoder
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
FSE : Finite State Entropy encoder
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include "compiler.h"
|
||||
#include "mem.h" /* U32, U16, etc. */
|
||||
#include "debug.h" /* assert, DEBUGLOG */
|
||||
@@ -24,9 +45,6 @@
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#include "error_private.h"
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#define ZSTD_DEPS_NEED_MATH64
|
||||
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -76,16 +94,13 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
|
||||
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
||||
U32 const step = FSE_TABLESTEP(tableSize);
|
||||
U32 const maxSV1 = maxSymbolValue+1;
|
||||
|
||||
U16* cumul = (U16*)workSpace; /* size = maxSV1 */
|
||||
FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */
|
||||
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
|
||||
|
||||
FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace;
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */
|
||||
if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
|
||||
/* CTable header */
|
||||
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
||||
tableU16[-2] = (U16) tableLog;
|
||||
tableU16[-1] = (U16) maxSymbolValue;
|
||||
assert(tableLog < 16); /* required for threshold strategy to work */
|
||||
@@ -94,67 +109,26 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
||||
|
||||
#ifdef __clang_analyzer__
|
||||
ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
|
||||
memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
|
||||
#endif
|
||||
|
||||
/* symbol start positions */
|
||||
{ U32 u;
|
||||
cumul[0] = 0;
|
||||
for (u=1; u <= maxSV1; u++) {
|
||||
for (u=1; u <= maxSymbolValue+1; u++) {
|
||||
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
|
||||
cumul[u] = cumul[u-1] + 1;
|
||||
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
|
||||
} else {
|
||||
assert(normalizedCounter[u-1] >= 0);
|
||||
cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1];
|
||||
assert(cumul[u] >= cumul[u-1]); /* no overflow */
|
||||
cumul[u] = cumul[u-1] + normalizedCounter[u-1];
|
||||
} }
|
||||
cumul[maxSV1] = (U16)(tableSize+1);
|
||||
cumul[maxSymbolValue+1] = tableSize+1;
|
||||
}
|
||||
|
||||
/* Spread symbols */
|
||||
if (highThreshold == tableSize - 1) {
|
||||
/* Case for no low prob count symbols. Lay down 8 bytes at a time
|
||||
* to reduce branch misses since we are operating on a small block
|
||||
*/
|
||||
BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */
|
||||
{ U64 const add = 0x0101010101010101ull;
|
||||
size_t pos = 0;
|
||||
U64 sv = 0;
|
||||
U32 s;
|
||||
for (s=0; s<maxSV1; ++s, sv += add) {
|
||||
int i;
|
||||
int const n = normalizedCounter[s];
|
||||
MEM_write64(spread + pos, sv);
|
||||
for (i = 8; i < n; i += 8) {
|
||||
MEM_write64(spread + pos + i, sv);
|
||||
}
|
||||
assert(n>=0);
|
||||
pos += (size_t)n;
|
||||
}
|
||||
}
|
||||
/* Spread symbols across the table. Lack of lowprob symbols means that
|
||||
* we don't need variable sized inner loop, so we can unroll the loop and
|
||||
* reduce branch misses.
|
||||
*/
|
||||
{ size_t position = 0;
|
||||
size_t s;
|
||||
size_t const unroll = 2; /* Experimentally determined optimal unroll */
|
||||
assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
|
||||
for (s = 0; s < (size_t)tableSize; s += unroll) {
|
||||
size_t u;
|
||||
for (u = 0; u < unroll; ++u) {
|
||||
size_t const uPosition = (position + (u * step)) & tableMask;
|
||||
tableSymbol[uPosition] = spread[s + u];
|
||||
}
|
||||
position = (position + (unroll * step)) & tableMask;
|
||||
}
|
||||
assert(position == 0); /* Must have initialized all positions */
|
||||
}
|
||||
} else {
|
||||
U32 position = 0;
|
||||
{ U32 position = 0;
|
||||
U32 symbol;
|
||||
for (symbol=0; symbol<maxSV1; symbol++) {
|
||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||
int nbOccurrences;
|
||||
int const freq = normalizedCounter[symbol];
|
||||
for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) {
|
||||
@@ -163,6 +137,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
while (position > highThreshold)
|
||||
position = (position + step) & tableMask; /* Low proba area */
|
||||
} }
|
||||
|
||||
assert(position==0); /* Must have initialized all positions */
|
||||
}
|
||||
|
||||
@@ -186,17 +161,16 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
case -1:
|
||||
case 1:
|
||||
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
|
||||
assert(total <= INT_MAX);
|
||||
symbolTT[s].deltaFindState = (int)(total - 1);
|
||||
symbolTT[s].deltaFindState = total - 1;
|
||||
total ++;
|
||||
break;
|
||||
default :
|
||||
assert(normalizedCounter[s] > 1);
|
||||
{ U32 const maxBitsOut = tableLog - BIT_highbit32 ((U32)normalizedCounter[s]-1);
|
||||
U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut;
|
||||
{
|
||||
U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
|
||||
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
|
||||
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
|
||||
symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]);
|
||||
total += (unsigned)normalizedCounter[s];
|
||||
symbolTT[s].deltaFindState = total - normalizedCounter[s];
|
||||
total += normalizedCounter[s];
|
||||
} } } }
|
||||
|
||||
#if 0 /* debug : symbol costs */
|
||||
@@ -207,26 +181,31 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
symbol, normalizedCounter[symbol],
|
||||
FSE_getMaxNbBits(symbolTT, symbol),
|
||||
(double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
|
||||
} }
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
|
||||
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding
|
||||
****************************************************************/
|
||||
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog
|
||||
+ 4 /* bitCount initialized at 4 */
|
||||
+ 2 /* first two symbols may use one additional bit each */) / 8)
|
||||
+ 1 /* round up to whole nb bytes */
|
||||
+ 2 /* additional two bytes for bitstream flush */;
|
||||
size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
|
||||
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
||||
}
|
||||
|
||||
@@ -348,10 +327,10 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
|
||||
size_t size;
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
|
||||
return (FSE_CTable*)ZSTD_malloc(size);
|
||||
return (FSE_CTable*)malloc(size);
|
||||
}
|
||||
|
||||
void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
|
||||
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
|
||||
|
||||
/* provides the minimum logSize to safely represent a distribution */
|
||||
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||
@@ -382,10 +361,11 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
|
||||
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
|
||||
}
|
||||
|
||||
|
||||
/* Secondary normalization method.
|
||||
To be used when primary method fails. */
|
||||
|
||||
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount)
|
||||
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
|
||||
{
|
||||
short const NOT_YET_ASSIGNED = -2;
|
||||
U32 s;
|
||||
@@ -402,7 +382,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
||||
continue;
|
||||
}
|
||||
if (count[s] <= lowThreshold) {
|
||||
norm[s] = lowProbCount;
|
||||
norm[s] = -1;
|
||||
distributed++;
|
||||
total -= count[s];
|
||||
continue;
|
||||
@@ -454,7 +434,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
||||
|
||||
{ U64 const vStepLog = 62 - tableLog;
|
||||
U64 const mid = (1ULL << (vStepLog-1)) - 1;
|
||||
U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
|
||||
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
|
||||
U64 tmpTotal = mid;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
if (norm[s]==NOT_YET_ASSIGNED) {
|
||||
@@ -471,9 +451,10 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
const unsigned* count, size_t total,
|
||||
unsigned maxSymbolValue, unsigned useLowProbCount)
|
||||
unsigned maxSymbolValue)
|
||||
{
|
||||
/* Sanity checks */
|
||||
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
@@ -482,9 +463,8 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
|
||||
|
||||
{ static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
|
||||
short const lowProbCount = useLowProbCount ? -1 : 1;
|
||||
U64 const scale = 62 - tableLog;
|
||||
U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */
|
||||
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
|
||||
U64 const vStep = 1ULL<<(scale-20);
|
||||
int stillToDistribute = 1<<tableLog;
|
||||
unsigned s;
|
||||
@@ -496,7 +476,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
if (count[s] == total) return 0; /* rle special case */
|
||||
if (count[s] == 0) { normalizedCounter[s]=0; continue; }
|
||||
if (count[s] <= lowThreshold) {
|
||||
normalizedCounter[s] = lowProbCount;
|
||||
normalizedCounter[s] = -1;
|
||||
stillToDistribute--;
|
||||
} else {
|
||||
short proba = (short)((count[s]*step) >> scale);
|
||||
@@ -510,7 +490,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
} }
|
||||
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
|
||||
/* corner case, need another normalization method */
|
||||
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
|
||||
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
|
||||
if (FSE_isError(errorCode)) return errorCode;
|
||||
}
|
||||
else normalizedCounter[largest] += (short)stillToDistribute;
|
||||
@@ -665,7 +645,9 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
|
||||
|
||||
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
|
||||
|
||||
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
||||
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
|
||||
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
||||
|
||||
/* FSE_compress_wksp() :
|
||||
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* `wkspSize` size must be `(1<<tableLog)`.
|
||||
@@ -684,7 +666,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
||||
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
|
||||
|
||||
/* init conditions */
|
||||
if (wkspSize < FSE_COMPRESS_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
|
||||
if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
|
||||
if (srcSize <= 1) return 0; /* Not compressible */
|
||||
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
@@ -697,7 +679,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
||||
}
|
||||
|
||||
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
|
||||
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) );
|
||||
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
|
||||
|
||||
/* Write table description header */
|
||||
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
||||
@@ -719,16 +701,13 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
||||
|
||||
typedef struct {
|
||||
FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
||||
union {
|
||||
U32 hist_wksp[HIST_WKSP_SIZE_U32];
|
||||
BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
|
||||
} workspace;
|
||||
BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
|
||||
} fseWkspMax_t;
|
||||
|
||||
size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
fseWkspMax_t scratchBuffer;
|
||||
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
|
||||
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
|
||||
}
|
||||
@@ -737,8 +716,6 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS
|
||||
{
|
||||
return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
209
vendor/github.com/DataDog/zstd/fse_decompress.c
generated
vendored
209
vendor/github.com/DataDog/zstd/fse_decompress.c
generated
vendored
@@ -1,30 +1,48 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* FSE : Finite State Entropy decoder
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
FSE : Finite State Entropy decoder
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include "debug.h" /* assert */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include "bitstream.h"
|
||||
#include "compiler.h"
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#include "error_private.h"
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#include "zstd_deps.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -33,6 +51,11 @@
|
||||
#define FSE_isError ERR_isError
|
||||
#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
|
||||
|
||||
/* check and forward error code */
|
||||
#ifndef CHECK_F
|
||||
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Templates
|
||||
@@ -61,27 +84,25 @@
|
||||
FSE_DTable* FSE_createDTable (unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
}
|
||||
|
||||
void FSE_freeDTable (FSE_DTable* dt)
|
||||
{
|
||||
ZSTD_free(dt);
|
||||
free(dt);
|
||||
}
|
||||
|
||||
static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
||||
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
||||
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
|
||||
U16* symbolNext = (U16*)workSpace;
|
||||
BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1);
|
||||
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
|
||||
|
||||
U32 const maxSV1 = maxSymbolValue + 1;
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* Sanity Checks */
|
||||
if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
|
||||
@@ -99,57 +120,11 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
||||
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
||||
symbolNext[s] = normalizedCounter[s];
|
||||
} } }
|
||||
ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
}
|
||||
|
||||
/* Spread symbols */
|
||||
if (highThreshold == tableSize - 1) {
|
||||
size_t const tableMask = tableSize-1;
|
||||
size_t const step = FSE_TABLESTEP(tableSize);
|
||||
/* First lay down the symbols in order.
|
||||
* We use a uint64_t to lay down 8 bytes at a time. This reduces branch
|
||||
* misses since small blocks generally have small table logs, so nearly
|
||||
* all symbols have counts <= 8. We ensure we have 8 bytes at the end of
|
||||
* our buffer to handle the over-write.
|
||||
*/
|
||||
{
|
||||
U64 const add = 0x0101010101010101ull;
|
||||
size_t pos = 0;
|
||||
U64 sv = 0;
|
||||
U32 s;
|
||||
for (s=0; s<maxSV1; ++s, sv += add) {
|
||||
int i;
|
||||
int const n = normalizedCounter[s];
|
||||
MEM_write64(spread + pos, sv);
|
||||
for (i = 8; i < n; i += 8) {
|
||||
MEM_write64(spread + pos + i, sv);
|
||||
}
|
||||
pos += n;
|
||||
}
|
||||
}
|
||||
/* Now we spread those positions across the table.
|
||||
* The benefit of doing it in two stages is that we avoid the the
|
||||
* variable size inner loop, which caused lots of branch misses.
|
||||
* Now we can run through all the positions without any branch misses.
|
||||
* We unroll the loop twice, since that is what emperically worked best.
|
||||
*/
|
||||
{
|
||||
size_t position = 0;
|
||||
size_t s;
|
||||
size_t const unroll = 2;
|
||||
assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
|
||||
for (s = 0; s < (size_t)tableSize; s += unroll) {
|
||||
size_t u;
|
||||
for (u = 0; u < unroll; ++u) {
|
||||
size_t const uPosition = (position + (u * step)) & tableMask;
|
||||
tableDecode[uPosition].symbol = spread[s + u];
|
||||
}
|
||||
position = (position + (unroll * step)) & tableMask;
|
||||
}
|
||||
assert(position == 0);
|
||||
}
|
||||
} else {
|
||||
U32 const tableMask = tableSize-1;
|
||||
{ U32 const tableMask = tableSize-1;
|
||||
U32 const step = FSE_TABLESTEP(tableSize);
|
||||
U32 s, position = 0;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
@@ -174,11 +149,6 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
|
||||
}
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
@@ -306,101 +276,36 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
short ncount[FSE_MAX_SYMBOL_VALUE + 1];
|
||||
FSE_DTable dtable[1]; /* Dynamically sized */
|
||||
} FSE_DecompressWksp;
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
unsigned maxLog, void* workSpace, size_t wkspSize,
|
||||
int bmi2)
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)cSrc;
|
||||
const BYTE* ip = istart;
|
||||
short counting[FSE_MAX_SYMBOL_VALUE+1];
|
||||
unsigned tableLog;
|
||||
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
|
||||
|
||||
DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
|
||||
if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
|
||||
|
||||
/* normal FSE decoding mode */
|
||||
{
|
||||
size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
|
||||
if (FSE_isError(NCountLength)) return NCountLength;
|
||||
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
|
||||
assert(NCountLength <= cSrcSize);
|
||||
ip += NCountLength;
|
||||
cSrcSize -= NCountLength;
|
||||
}
|
||||
size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
||||
if (FSE_isError(NCountLength)) return NCountLength;
|
||||
//if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
|
||||
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
|
||||
ip += NCountLength;
|
||||
cSrcSize -= NCountLength;
|
||||
|
||||
if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
|
||||
workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
|
||||
wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
|
||||
CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );
|
||||
|
||||
CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
|
||||
|
||||
{
|
||||
const void* ptr = wksp->dtable;
|
||||
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
|
||||
return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/* Avoids the FORCE_INLINE of the _body() function. */
|
||||
static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
|
||||
}
|
||||
|
||||
#if DYNAMIC_BMI2
|
||||
BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
|
||||
{
|
||||
#if DYNAMIC_BMI2
|
||||
if (bmi2) {
|
||||
return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
|
||||
}
|
||||
#endif
|
||||
(void)bmi2;
|
||||
return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
|
||||
return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
|
||||
}
|
||||
|
||||
|
||||
typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
||||
|
||||
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
||||
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
|
||||
U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
|
||||
return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
|
||||
}
|
||||
|
||||
size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
||||
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
|
||||
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
101
vendor/github.com/DataDog/zstd/hist.c
generated
vendored
101
vendor/github.com/DataDog/zstd/hist.c
generated
vendored
@@ -1,17 +1,36 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* hist : Histogram functions
|
||||
* part of Finite State Entropy project
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
hist : Histogram functions
|
||||
part of Finite State Entropy project
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* --- dependencies --- */
|
||||
@@ -35,7 +54,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned largestCount=0;
|
||||
|
||||
ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
|
||||
memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
|
||||
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
|
||||
|
||||
while (ip<end) {
|
||||
@@ -61,9 +80,9 @@ typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
|
||||
* this design makes better use of OoO cpus,
|
||||
* and is noticeably faster when some values are heavily repeated.
|
||||
* But it needs some additional workspace for intermediate tables.
|
||||
* `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
|
||||
* `workSpace` size must be a table of size >= HIST_WKSP_SIZE_U32.
|
||||
* @return : largest histogram frequency,
|
||||
* or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
|
||||
* or an error code (notably when histogram would be larger than *maxSymbolValuePtr). */
|
||||
static size_t HIST_count_parallel_wksp(
|
||||
unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
@@ -72,21 +91,22 @@ static size_t HIST_count_parallel_wksp(
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)source;
|
||||
const BYTE* const iend = ip+sourceSize;
|
||||
size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count);
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned max=0;
|
||||
U32* const Counting1 = workSpace;
|
||||
U32* const Counting2 = Counting1 + 256;
|
||||
U32* const Counting3 = Counting2 + 256;
|
||||
U32* const Counting4 = Counting3 + 256;
|
||||
|
||||
memset(workSpace, 0, 4*256*sizeof(unsigned));
|
||||
|
||||
/* safety checks */
|
||||
assert(*maxSymbolValuePtr <= 255);
|
||||
if (!sourceSize) {
|
||||
ZSTD_memset(count, 0, countSize);
|
||||
memset(count, 0, maxSymbolValue + 1);
|
||||
*maxSymbolValuePtr = 0;
|
||||
return 0;
|
||||
}
|
||||
ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned));
|
||||
if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */
|
||||
|
||||
/* by stripes of 16 bytes */
|
||||
{ U32 cached = MEM_read32(ip); ip += 4;
|
||||
@@ -118,18 +138,21 @@ static size_t HIST_count_parallel_wksp(
|
||||
/* finish last symbols */
|
||||
while (ip<iend) Counting1[*ip++]++;
|
||||
|
||||
{ U32 s;
|
||||
for (s=0; s<256; s++) {
|
||||
if (check) { /* verify stats will fit into destination table */
|
||||
U32 s; for (s=255; s>maxSymbolValue; s--) {
|
||||
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (Counting1[s] > max) max = Counting1[s];
|
||||
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
|
||||
} }
|
||||
|
||||
{ unsigned maxSymbolValue = 255;
|
||||
while (!Counting1[maxSymbolValue]) maxSymbolValue--;
|
||||
if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall);
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */
|
||||
}
|
||||
{ U32 s;
|
||||
if (maxSymbolValue > 255) maxSymbolValue = 255;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (count[s] > max) max = count[s];
|
||||
} }
|
||||
|
||||
while (!count[maxSymbolValue]) maxSymbolValue--;
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
return (size_t)max;
|
||||
}
|
||||
|
||||
@@ -149,6 +172,14 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
|
||||
}
|
||||
|
||||
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
||||
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
||||
}
|
||||
|
||||
/* HIST_count_wksp() :
|
||||
* Same as HIST_count(), but using an externally provided scratch buffer.
|
||||
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
||||
@@ -164,21 +195,9 @@ size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
|
||||
}
|
||||
|
||||
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
||||
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
||||
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
||||
}
|
||||
|
||||
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
49
vendor/github.com/DataDog/zstd/hist.h
generated
vendored
49
vendor/github.com/DataDog/zstd/hist.h
generated
vendored
@@ -1,21 +1,40 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* hist : Histogram functions
|
||||
* part of Finite State Entropy project
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
hist : Histogram functions
|
||||
part of Finite State Entropy project
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* --- dependencies --- */
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* --- simple histogram functions --- */
|
||||
@@ -74,5 +93,3 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
*/
|
||||
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
117
vendor/github.com/DataDog/zstd/huf.h
generated
vendored
117
vendor/github.com/DataDog/zstd/huf.h
generated
vendored
@@ -1,16 +1,35 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* huff0 huffman codec,
|
||||
* part of Finite State Entropy library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
huff0 huffman codec,
|
||||
part of Finite State Entropy library
|
||||
Copyright (C) 2013-present, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
@@ -21,7 +40,7 @@ extern "C" {
|
||||
#define HUF_H_298734234
|
||||
|
||||
/* *** Dependencies *** */
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* *** library symbols visibility *** */
|
||||
@@ -90,9 +109,9 @@ HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
|
||||
|
||||
/** HUF_compress4X_wksp() :
|
||||
* Same as HUF_compress2(), but uses externally allocated `workSpace`.
|
||||
* `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
|
||||
#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
|
||||
#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
|
||||
* `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
|
||||
#define HUF_WORKSPACE_SIZE (6 << 10)
|
||||
#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
|
||||
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
@@ -112,16 +131,14 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
||||
|
||||
/* *** Dependencies *** */
|
||||
#include "mem.h" /* U32 */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
|
||||
|
||||
/* *** Constants *** */
|
||||
#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
|
||||
#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
|
||||
#define HUF_SYMBOLVALUE_MAX 255
|
||||
|
||||
#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
||||
# error "HUF_TABLELOG_MAX is too large !"
|
||||
#endif
|
||||
@@ -136,12 +153,12 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* static allocation of HUF's Compression Table */
|
||||
/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
|
||||
typedef size_t HUF_CElt; /* consider it an incomplete type */
|
||||
#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
|
||||
#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
|
||||
#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */
|
||||
#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32))
|
||||
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
||||
HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
|
||||
U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \
|
||||
void* name##hv = &(name##hb); \
|
||||
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
||||
|
||||
/* static allocation of HUF's DTable */
|
||||
typedef U32 HUF_DTable;
|
||||
@@ -187,13 +204,10 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
|
||||
* or to save and regenerate 'CTable' using external methods.
|
||||
*/
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
||||
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
||||
size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
|
||||
size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
|
||||
int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
|
||||
|
||||
typedef enum {
|
||||
HUF_repeat_none, /**< Cannot use the previous table */
|
||||
@@ -204,13 +218,12 @@ typedef enum {
|
||||
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
* If it uses hufTable it does not modify hufTable or repeat.
|
||||
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
||||
* If preferRepeat then the old table will always be used if valid.
|
||||
* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
|
||||
* If preferRepeat then the old table will always be used if valid. */
|
||||
size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
||||
|
||||
/** HUF_buildCTable_wksp() :
|
||||
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
||||
@@ -231,27 +244,15 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
|
||||
U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/*! HUF_readStats_wksp() :
|
||||
* Same as HUF_readStats() but takes an external workspace which must be
|
||||
* 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
|
||||
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
|
||||
*/
|
||||
#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
|
||||
#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
|
||||
size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
|
||||
U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize,
|
||||
void* workspace, size_t wkspSize,
|
||||
int bmi2);
|
||||
|
||||
/** HUF_readCTable() :
|
||||
* Loading a CTable saved with HUF_writeCTable() */
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
|
||||
/** HUF_getNbBitsFromCTable() :
|
||||
/** HUF_getNbBits() :
|
||||
* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
|
||||
* Note 1 : is not inlined, as HUF_CElt definition is private */
|
||||
U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
|
||||
* Note 1 : is not inlined, as HUF_CElt definition is private
|
||||
* Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
|
||||
U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
|
||||
|
||||
/*
|
||||
* HUF_decompress() does the following:
|
||||
@@ -277,7 +278,7 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
||||
* a required workspace size greater than that specified in the following
|
||||
* macro.
|
||||
*/
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
||||
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
@@ -303,20 +304,18 @@ size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* c
|
||||
/* ====================== */
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */
|
||||
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
|
||||
/** HUF_compress1X_repeat() :
|
||||
* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
* If it uses hufTable it does not modify hufTable or repeat.
|
||||
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
||||
* If preferRepeat then the old table will always be used if valid.
|
||||
* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
|
||||
* If preferRepeat then the old table will always be used if valid. */
|
||||
size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
||||
|
||||
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
@@ -351,17 +350,9 @@ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstS
|
||||
#endif
|
||||
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
||||
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
#endif
|
||||
|
||||
#endif /* HUF_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
1085
vendor/github.com/DataDog/zstd/huf_compress.c
generated
vendored
1085
vendor/github.com/DataDog/zstd/huf_compress.c
generated
vendored
File diff suppressed because it is too large
Load Diff
1460
vendor/github.com/DataDog/zstd/huf_decompress.c
generated
vendored
1460
vendor/github.com/DataDog/zstd/huf_decompress.c
generated
vendored
File diff suppressed because it is too large
Load Diff
585
vendor/github.com/DataDog/zstd/huf_decompress_amd64.S
generated
vendored
585
vendor/github.com/DataDog/zstd/huf_decompress_amd64.S
generated
vendored
@@ -1,585 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "portability_macros.h"
|
||||
|
||||
/* Stack marking
|
||||
* ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart
|
||||
*/
|
||||
#if defined(__ELF__) && defined(__GNUC__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
|
||||
/* Calling convention:
|
||||
*
|
||||
* %rdi contains the first argument: HUF_DecompressAsmArgs*.
|
||||
* %rbp isn't maintained (no frame pointer).
|
||||
* %rsp contains the stack pointer that grows down.
|
||||
* No red-zone is assumed, only addresses >= %rsp are used.
|
||||
* All register contents are preserved.
|
||||
*
|
||||
* TODO: Support Windows calling convention.
|
||||
*/
|
||||
|
||||
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
|
||||
.global HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
|
||||
.global HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
|
||||
.global _HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
|
||||
.global _HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
|
||||
.text
|
||||
|
||||
/* Sets up register mappings for clarity.
|
||||
* op[], bits[], dtable & ip[0] each get their own register.
|
||||
* ip[1,2,3] & olimit alias var[].
|
||||
* %rax is a scratch register.
|
||||
*/
|
||||
|
||||
#define op0 rsi
|
||||
#define op1 rbx
|
||||
#define op2 rcx
|
||||
#define op3 rdi
|
||||
|
||||
#define ip0 r8
|
||||
#define ip1 r9
|
||||
#define ip2 r10
|
||||
#define ip3 r11
|
||||
|
||||
#define bits0 rbp
|
||||
#define bits1 rdx
|
||||
#define bits2 r12
|
||||
#define bits3 r13
|
||||
#define dtable r14
|
||||
#define olimit r15
|
||||
|
||||
/* var[] aliases ip[1,2,3] & olimit
|
||||
* ip[1,2,3] are saved every iteration.
|
||||
* olimit is only used in compute_olimit.
|
||||
*/
|
||||
#define var0 r15
|
||||
#define var1 r9
|
||||
#define var2 r10
|
||||
#define var3 r11
|
||||
|
||||
/* 32-bit var registers */
|
||||
#define vard0 r15d
|
||||
#define vard1 r9d
|
||||
#define vard2 r10d
|
||||
#define vard3 r11d
|
||||
|
||||
/* Calls X(N) for each stream 0, 1, 2, 3. */
|
||||
#define FOR_EACH_STREAM(X) \
|
||||
X(0); \
|
||||
X(1); \
|
||||
X(2); \
|
||||
X(3)
|
||||
|
||||
/* Calls X(N, idx) for each stream 0, 1, 2, 3. */
|
||||
#define FOR_EACH_STREAM_WITH_INDEX(X, idx) \
|
||||
X(0, idx); \
|
||||
X(1, idx); \
|
||||
X(2, idx); \
|
||||
X(3, idx)
|
||||
|
||||
/* Define both _HUF_* & HUF_* symbols because MacOS
|
||||
* C symbols are prefixed with '_' & Linux symbols aren't.
|
||||
*/
|
||||
_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
/* Save all registers - even if they are callee saved for simplicity. */
|
||||
push %rax
|
||||
push %rbx
|
||||
push %rcx
|
||||
push %rdx
|
||||
push %rbp
|
||||
push %rsi
|
||||
push %rdi
|
||||
push %r8
|
||||
push %r9
|
||||
push %r10
|
||||
push %r11
|
||||
push %r12
|
||||
push %r13
|
||||
push %r14
|
||||
push %r15
|
||||
|
||||
/* Read HUF_DecompressAsmArgs* args from %rax */
|
||||
movq %rdi, %rax
|
||||
movq 0(%rax), %ip0
|
||||
movq 8(%rax), %ip1
|
||||
movq 16(%rax), %ip2
|
||||
movq 24(%rax), %ip3
|
||||
movq 32(%rax), %op0
|
||||
movq 40(%rax), %op1
|
||||
movq 48(%rax), %op2
|
||||
movq 56(%rax), %op3
|
||||
movq 64(%rax), %bits0
|
||||
movq 72(%rax), %bits1
|
||||
movq 80(%rax), %bits2
|
||||
movq 88(%rax), %bits3
|
||||
movq 96(%rax), %dtable
|
||||
push %rax /* argument */
|
||||
push 104(%rax) /* ilimit */
|
||||
push 112(%rax) /* oend */
|
||||
push %olimit /* olimit space */
|
||||
|
||||
subq $24, %rsp
|
||||
|
||||
.L_4X1_compute_olimit:
|
||||
/* Computes how many iterations we can do safely
|
||||
* %r15, %rax may be clobbered
|
||||
* rbx, rdx must be saved
|
||||
* op3 & ip0 mustn't be clobbered
|
||||
*/
|
||||
movq %rbx, 0(%rsp)
|
||||
movq %rdx, 8(%rsp)
|
||||
|
||||
movq 32(%rsp), %rax /* rax = oend */
|
||||
subq %op3, %rax /* rax = oend - op3 */
|
||||
|
||||
/* r15 = (oend - op3) / 5 */
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
mulq %rdx
|
||||
movq %rdx, %r15
|
||||
shrq $2, %r15
|
||||
|
||||
movq %ip0, %rax /* rax = ip0 */
|
||||
movq 40(%rsp), %rdx /* rdx = ilimit */
|
||||
subq %rdx, %rax /* rax = ip0 - ilimit */
|
||||
movq %rax, %rbx /* rbx = ip0 - ilimit */
|
||||
|
||||
/* rdx = (ip0 - ilimit) / 7 */
|
||||
movabsq $2635249153387078803, %rdx
|
||||
mulq %rdx
|
||||
subq %rdx, %rbx
|
||||
shrq %rbx
|
||||
addq %rbx, %rdx
|
||||
shrq $2, %rdx
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
|
||||
/* r15 = r15 * 5 */
|
||||
leaq (%r15, %r15, 4), %r15
|
||||
|
||||
/* olimit = op3 + r15 */
|
||||
addq %op3, %olimit
|
||||
|
||||
movq 8(%rsp), %rdx
|
||||
movq 0(%rsp), %rbx
|
||||
|
||||
/* If (op3 + 20 > olimit) */
|
||||
movq %op3, %rax /* rax = op3 */
|
||||
addq $20, %rax /* rax = op3 + 20 */
|
||||
cmpq %rax, %olimit /* op3 + 20 > olimit */
|
||||
jb .L_4X1_exit
|
||||
|
||||
/* If (ip1 < ip0) go to exit */
|
||||
cmpq %ip0, %ip1
|
||||
jb .L_4X1_exit
|
||||
|
||||
/* If (ip2 < ip1) go to exit */
|
||||
cmpq %ip1, %ip2
|
||||
jb .L_4X1_exit
|
||||
|
||||
/* If (ip3 < ip2) go to exit */
|
||||
cmpq %ip2, %ip3
|
||||
jb .L_4X1_exit
|
||||
|
||||
/* Reads top 11 bits from bits[n]
|
||||
* Loads dt[bits[n]] into var[n]
|
||||
*/
|
||||
#define GET_NEXT_DELT(n) \
|
||||
movq $53, %var##n; \
|
||||
shrxq %var##n, %bits##n, %var##n; \
|
||||
movzwl (%dtable,%var##n,2),%vard##n
|
||||
|
||||
/* var[n] must contain the DTable entry computed with GET_NEXT_DELT
|
||||
* Moves var[n] to %rax
|
||||
* bits[n] <<= var[n] & 63
|
||||
* op[n][idx] = %rax >> 8
|
||||
* %ah is a way to access bits [8, 16) of %rax
|
||||
*/
|
||||
#define DECODE_FROM_DELT(n, idx) \
|
||||
movq %var##n, %rax; \
|
||||
shlxq %var##n, %bits##n, %bits##n; \
|
||||
movb %ah, idx(%op##n)
|
||||
|
||||
/* Assumes GET_NEXT_DELT has been called.
|
||||
* Calls DECODE_FROM_DELT then GET_NEXT_DELT
|
||||
*/
|
||||
#define DECODE_AND_GET_NEXT(n, idx) \
|
||||
DECODE_FROM_DELT(n, idx); \
|
||||
GET_NEXT_DELT(n) \
|
||||
|
||||
/* // ctz & nbBytes is stored in bits[n]
|
||||
* // nbBits is stored in %rax
|
||||
* ctz = CTZ[bits[n]]
|
||||
* nbBits = ctz & 7
|
||||
* nbBytes = ctz >> 3
|
||||
* op[n] += 5
|
||||
* ip[n] -= nbBytes
|
||||
* // Note: x86-64 is little-endian ==> no bswap
|
||||
* bits[n] = MEM_readST(ip[n]) | 1
|
||||
* bits[n] <<= nbBits
|
||||
*/
|
||||
#define RELOAD_BITS(n) \
|
||||
bsfq %bits##n, %bits##n; \
|
||||
movq %bits##n, %rax; \
|
||||
andq $7, %rax; \
|
||||
shrq $3, %bits##n; \
|
||||
leaq 5(%op##n), %op##n; \
|
||||
subq %bits##n, %ip##n; \
|
||||
movq (%ip##n), %bits##n; \
|
||||
orq $1, %bits##n; \
|
||||
shlx %rax, %bits##n, %bits##n
|
||||
|
||||
/* Store clobbered variables on the stack */
|
||||
movq %olimit, 24(%rsp)
|
||||
movq %ip1, 0(%rsp)
|
||||
movq %ip2, 8(%rsp)
|
||||
movq %ip3, 16(%rsp)
|
||||
|
||||
/* Call GET_NEXT_DELT for each stream */
|
||||
FOR_EACH_STREAM(GET_NEXT_DELT)
|
||||
|
||||
.p2align 6
|
||||
|
||||
.L_4X1_loop_body:
|
||||
/* Decode 5 symbols in each of the 4 streams (20 total)
|
||||
* Must have called GET_NEXT_DELT for each stream
|
||||
*/
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 0)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 1)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 2)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 3)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE_FROM_DELT, 4)
|
||||
|
||||
/* Load ip[1,2,3] from stack (var[] aliases them)
|
||||
* ip[] is needed for RELOAD_BITS
|
||||
* Each will be stored back to the stack after RELOAD
|
||||
*/
|
||||
movq 0(%rsp), %ip1
|
||||
movq 8(%rsp), %ip2
|
||||
movq 16(%rsp), %ip3
|
||||
|
||||
/* Reload each stream & fetch the next table entry
|
||||
* to prepare for the next iteration
|
||||
*/
|
||||
RELOAD_BITS(0)
|
||||
GET_NEXT_DELT(0)
|
||||
|
||||
RELOAD_BITS(1)
|
||||
movq %ip1, 0(%rsp)
|
||||
GET_NEXT_DELT(1)
|
||||
|
||||
RELOAD_BITS(2)
|
||||
movq %ip2, 8(%rsp)
|
||||
GET_NEXT_DELT(2)
|
||||
|
||||
RELOAD_BITS(3)
|
||||
movq %ip3, 16(%rsp)
|
||||
GET_NEXT_DELT(3)
|
||||
|
||||
/* If op3 < olimit: continue the loop */
|
||||
cmp %op3, 24(%rsp)
|
||||
ja .L_4X1_loop_body
|
||||
|
||||
/* Reload ip[1,2,3] from stack */
|
||||
movq 0(%rsp), %ip1
|
||||
movq 8(%rsp), %ip2
|
||||
movq 16(%rsp), %ip3
|
||||
|
||||
/* Re-compute olimit */
|
||||
jmp .L_4X1_compute_olimit
|
||||
|
||||
#undef GET_NEXT_DELT
|
||||
#undef DECODE_FROM_DELT
|
||||
#undef DECODE
|
||||
#undef RELOAD_BITS
|
||||
.L_4X1_exit:
|
||||
addq $24, %rsp
|
||||
|
||||
/* Restore stack (oend & olimit) */
|
||||
pop %rax /* olimit */
|
||||
pop %rax /* oend */
|
||||
pop %rax /* ilimit */
|
||||
pop %rax /* arg */
|
||||
|
||||
/* Save ip / op / bits */
|
||||
movq %ip0, 0(%rax)
|
||||
movq %ip1, 8(%rax)
|
||||
movq %ip2, 16(%rax)
|
||||
movq %ip3, 24(%rax)
|
||||
movq %op0, 32(%rax)
|
||||
movq %op1, 40(%rax)
|
||||
movq %op2, 48(%rax)
|
||||
movq %op3, 56(%rax)
|
||||
movq %bits0, 64(%rax)
|
||||
movq %bits1, 72(%rax)
|
||||
movq %bits2, 80(%rax)
|
||||
movq %bits3, 88(%rax)
|
||||
|
||||
/* Restore registers */
|
||||
pop %r15
|
||||
pop %r14
|
||||
pop %r13
|
||||
pop %r12
|
||||
pop %r11
|
||||
pop %r10
|
||||
pop %r9
|
||||
pop %r8
|
||||
pop %rdi
|
||||
pop %rsi
|
||||
pop %rbp
|
||||
pop %rdx
|
||||
pop %rcx
|
||||
pop %rbx
|
||||
pop %rax
|
||||
ret
|
||||
|
||||
_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
/* Save all registers - even if they are callee saved for simplicity. */
|
||||
push %rax
|
||||
push %rbx
|
||||
push %rcx
|
||||
push %rdx
|
||||
push %rbp
|
||||
push %rsi
|
||||
push %rdi
|
||||
push %r8
|
||||
push %r9
|
||||
push %r10
|
||||
push %r11
|
||||
push %r12
|
||||
push %r13
|
||||
push %r14
|
||||
push %r15
|
||||
|
||||
movq %rdi, %rax
|
||||
movq 0(%rax), %ip0
|
||||
movq 8(%rax), %ip1
|
||||
movq 16(%rax), %ip2
|
||||
movq 24(%rax), %ip3
|
||||
movq 32(%rax), %op0
|
||||
movq 40(%rax), %op1
|
||||
movq 48(%rax), %op2
|
||||
movq 56(%rax), %op3
|
||||
movq 64(%rax), %bits0
|
||||
movq 72(%rax), %bits1
|
||||
movq 80(%rax), %bits2
|
||||
movq 88(%rax), %bits3
|
||||
movq 96(%rax), %dtable
|
||||
push %rax /* argument */
|
||||
push %rax /* olimit */
|
||||
push 104(%rax) /* ilimit */
|
||||
|
||||
movq 112(%rax), %rax
|
||||
push %rax /* oend3 */
|
||||
|
||||
movq %op3, %rax
|
||||
push %rax /* oend2 */
|
||||
|
||||
movq %op2, %rax
|
||||
push %rax /* oend1 */
|
||||
|
||||
movq %op1, %rax
|
||||
push %rax /* oend0 */
|
||||
|
||||
/* Scratch space */
|
||||
subq $8, %rsp
|
||||
|
||||
.L_4X2_compute_olimit:
|
||||
/* Computes how many iterations we can do safely
|
||||
* %r15, %rax may be clobbered
|
||||
* rdx must be saved
|
||||
* op[1,2,3,4] & ip0 mustn't be clobbered
|
||||
*/
|
||||
movq %rdx, 0(%rsp)
|
||||
|
||||
/* We can consume up to 7 input bytes each iteration. */
|
||||
movq %ip0, %rax /* rax = ip0 */
|
||||
movq 40(%rsp), %rdx /* rdx = ilimit */
|
||||
subq %rdx, %rax /* rax = ip0 - ilimit */
|
||||
movq %rax, %r15 /* r15 = ip0 - ilimit */
|
||||
|
||||
/* rdx = rax / 7 */
|
||||
movabsq $2635249153387078803, %rdx
|
||||
mulq %rdx
|
||||
subq %rdx, %r15
|
||||
shrq %r15
|
||||
addq %r15, %rdx
|
||||
shrq $2, %rdx
|
||||
|
||||
/* r15 = (ip0 - ilimit) / 7 */
|
||||
movq %rdx, %r15
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 8(%rsp), %rax /* rax = oend0 */
|
||||
subq %op0, %rax /* rax = oend0 - op0 */
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 16(%rsp), %rax /* rax = oend1 */
|
||||
subq %op1, %rax /* rax = oend1 - op1 */
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 24(%rsp), %rax /* rax = oend2 */
|
||||
subq %op2, %rax /* rax = oend2 - op2 */
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 32(%rsp), %rax /* rax = oend3 */
|
||||
subq %op3, %rax /* rax = oend3 - op3 */
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
|
||||
/* olimit = op3 + 5 * r15 */
|
||||
movq %r15, %rax
|
||||
leaq (%op3, %rax, 4), %olimit
|
||||
addq %rax, %olimit
|
||||
|
||||
movq 0(%rsp), %rdx
|
||||
|
||||
/* If (op3 + 10 > olimit) */
|
||||
movq %op3, %rax /* rax = op3 */
|
||||
addq $10, %rax /* rax = op3 + 10 */
|
||||
cmpq %rax, %olimit /* op3 + 10 > olimit */
|
||||
jb .L_4X2_exit
|
||||
|
||||
/* If (ip1 < ip0) go to exit */
|
||||
cmpq %ip0, %ip1
|
||||
jb .L_4X2_exit
|
||||
|
||||
/* If (ip2 < ip1) go to exit */
|
||||
cmpq %ip1, %ip2
|
||||
jb .L_4X2_exit
|
||||
|
||||
/* If (ip3 < ip2) go to exit */
|
||||
cmpq %ip2, %ip3
|
||||
jb .L_4X2_exit
|
||||
|
||||
#define DECODE(n, idx) \
|
||||
movq %bits##n, %rax; \
|
||||
shrq $53, %rax; \
|
||||
movzwl 0(%dtable,%rax,4),%r8d; \
|
||||
movzbl 2(%dtable,%rax,4),%r15d; \
|
||||
movzbl 3(%dtable,%rax,4),%eax; \
|
||||
movw %r8w, (%op##n); \
|
||||
shlxq %r15, %bits##n, %bits##n; \
|
||||
addq %rax, %op##n
|
||||
|
||||
#define RELOAD_BITS(n) \
|
||||
bsfq %bits##n, %bits##n; \
|
||||
movq %bits##n, %rax; \
|
||||
shrq $3, %bits##n; \
|
||||
andq $7, %rax; \
|
||||
subq %bits##n, %ip##n; \
|
||||
movq (%ip##n), %bits##n; \
|
||||
orq $1, %bits##n; \
|
||||
shlxq %rax, %bits##n, %bits##n
|
||||
|
||||
|
||||
movq %olimit, 48(%rsp)
|
||||
|
||||
.p2align 6
|
||||
|
||||
.L_4X2_loop_body:
|
||||
/* We clobber r8, so store it on the stack */
|
||||
movq %r8, 0(%rsp)
|
||||
|
||||
/* Decode 5 symbols from each of the 4 streams (20 symbols total). */
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE, 0)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE, 1)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE, 2)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE, 3)
|
||||
FOR_EACH_STREAM_WITH_INDEX(DECODE, 4)
|
||||
|
||||
/* Reload r8 */
|
||||
movq 0(%rsp), %r8
|
||||
|
||||
FOR_EACH_STREAM(RELOAD_BITS)
|
||||
|
||||
cmp %op3, 48(%rsp)
|
||||
ja .L_4X2_loop_body
|
||||
jmp .L_4X2_compute_olimit
|
||||
|
||||
#undef DECODE
|
||||
#undef RELOAD_BITS
|
||||
.L_4X2_exit:
|
||||
addq $8, %rsp
|
||||
/* Restore stack (oend & olimit) */
|
||||
pop %rax /* oend0 */
|
||||
pop %rax /* oend1 */
|
||||
pop %rax /* oend2 */
|
||||
pop %rax /* oend3 */
|
||||
pop %rax /* ilimit */
|
||||
pop %rax /* olimit */
|
||||
pop %rax /* arg */
|
||||
|
||||
/* Save ip / op / bits */
|
||||
movq %ip0, 0(%rax)
|
||||
movq %ip1, 8(%rax)
|
||||
movq %ip2, 16(%rax)
|
||||
movq %ip3, 24(%rax)
|
||||
movq %op0, 32(%rax)
|
||||
movq %op1, 40(%rax)
|
||||
movq %op2, 48(%rax)
|
||||
movq %op3, 56(%rax)
|
||||
movq %bits0, 64(%rax)
|
||||
movq %bits1, 72(%rax)
|
||||
movq %bits2, 80(%rax)
|
||||
movq %bits3, 88(%rax)
|
||||
|
||||
/* Restore registers */
|
||||
pop %r15
|
||||
pop %r14
|
||||
pop %r13
|
||||
pop %r12
|
||||
pop %r11
|
||||
pop %r10
|
||||
pop %r9
|
||||
pop %r8
|
||||
pop %rdi
|
||||
pop %rsi
|
||||
pop %rbp
|
||||
pop %rdx
|
||||
pop %rcx
|
||||
pop %rbx
|
||||
pop %rax
|
||||
ret
|
||||
|
||||
#endif
|
||||
188
vendor/github.com/DataDog/zstd/mem.h
generated
vendored
188
vendor/github.com/DataDog/zstd/mem.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -19,10 +18,8 @@ extern "C" {
|
||||
/*-****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include "compiler.h" /* __has_builtin */
|
||||
#include "debug.h" /* DEBUG_STATIC_ASSERT */
|
||||
#include "zstd_deps.h" /* ZSTD_memcpy */
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <string.h> /* memcpy */
|
||||
|
||||
|
||||
/*-****************************************
|
||||
@@ -42,18 +39,94 @@ extern "C" {
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
#ifndef __has_builtin
|
||||
# define __has_builtin(x) 0 /* compat. with non-clang compilers */
|
||||
#endif
|
||||
|
||||
/* code only tested on 32 and 64 bits systems */
|
||||
#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
|
||||
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
||||
|
||||
/* detects whether we are being compiled under msan */
|
||||
#if defined (__has_feature)
|
||||
# if __has_feature(memory_sanitizer)
|
||||
# define MEMORY_SANITIZER 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if defined (MEMORY_SANITIZER)
|
||||
/* Not all platforms that support msan provide sanitizers/msan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
* include the header file... */
|
||||
|
||||
#include <stdint.h> /* intptr_t */
|
||||
|
||||
/* Make memory region fully initialized (without changing its contents). */
|
||||
void __msan_unpoison(const volatile void *a, size_t size);
|
||||
|
||||
/* Make memory region fully uninitialized (without changing its contents).
|
||||
This is a legacy interface that does not update origin information. Use
|
||||
__msan_allocated_memory() instead. */
|
||||
void __msan_poison(const volatile void *a, size_t size);
|
||||
|
||||
/* Returns the offset of the first (at least partially) poisoned byte in the
|
||||
memory range, or -1 if the whole range is good. */
|
||||
intptr_t __msan_test_shadow(const volatile void *x, size_t size);
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under asan */
|
||||
#if defined (__has_feature)
|
||||
# if __has_feature(address_sanitizer)
|
||||
# define ADDRESS_SANITIZER 1
|
||||
# endif
|
||||
#elif defined(__SANITIZE_ADDRESS__)
|
||||
# define ADDRESS_SANITIZER 1
|
||||
#endif
|
||||
|
||||
#if defined (ADDRESS_SANITIZER)
|
||||
/* Not all platforms that support asan provide sanitizers/asan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
* include the header file... */
|
||||
|
||||
/**
|
||||
* Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
|
||||
*
|
||||
* This memory must be previously allocated by your program. Instrumented
|
||||
* code is forbidden from accessing addresses in this region until it is
|
||||
* unpoisoned. This function is not guaranteed to poison the entire region -
|
||||
* it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
|
||||
* alignment restrictions.
|
||||
*
|
||||
* \note This function is not thread-safe because no two threads can poison or
|
||||
* unpoison memory in the same memory region simultaneously.
|
||||
*
|
||||
* \param addr Start of memory region.
|
||||
* \param size Size of memory region. */
|
||||
void __asan_poison_memory_region(void const volatile *addr, size_t size);
|
||||
|
||||
/**
|
||||
* Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
|
||||
*
|
||||
* This memory must be previously allocated by your program. Accessing
|
||||
* addresses in this region is allowed until this region is poisoned again.
|
||||
* This function could unpoison a super-region of <c>[addr, addr+size)</c> due
|
||||
* to ASan alignment restrictions.
|
||||
*
|
||||
* \note This function is not thread-safe because no two threads can
|
||||
* poison or unpoison memory in the same memory region simultaneously.
|
||||
*
|
||||
* \param addr Start of memory region.
|
||||
* \param size Size of memory region. */
|
||||
void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
|
||||
#endif
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Basic Types
|
||||
*****************************************************************/
|
||||
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
||||
# if defined(_AIX)
|
||||
# include <inttypes.h>
|
||||
# else
|
||||
# include <stdint.h> /* intptr_t */
|
||||
# endif
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint8_t U8;
|
||||
typedef int8_t S8;
|
||||
typedef uint16_t U16;
|
||||
typedef int16_t S16;
|
||||
typedef uint32_t U32;
|
||||
@@ -66,8 +139,6 @@ extern "C" {
|
||||
# error "this implementation requires char to be exactly 8-bit type"
|
||||
#endif
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned char U8;
|
||||
typedef signed char S8;
|
||||
#if USHRT_MAX != 65535
|
||||
# error "this implementation requires short to be exactly 16-bit type"
|
||||
#endif
|
||||
@@ -86,53 +157,7 @@ extern "C" {
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Memory I/O API
|
||||
*****************************************************************/
|
||||
/*=== Static platform detection ===*/
|
||||
MEM_STATIC unsigned MEM_32bits(void);
|
||||
MEM_STATIC unsigned MEM_64bits(void);
|
||||
MEM_STATIC unsigned MEM_isLittleEndian(void);
|
||||
|
||||
/*=== Native unaligned read/write ===*/
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr);
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr);
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr);
|
||||
MEM_STATIC size_t MEM_readST(const void* memPtr);
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value);
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value);
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value);
|
||||
|
||||
/*=== Little endian unaligned read/write ===*/
|
||||
MEM_STATIC U16 MEM_readLE16(const void* memPtr);
|
||||
MEM_STATIC U32 MEM_readLE24(const void* memPtr);
|
||||
MEM_STATIC U32 MEM_readLE32(const void* memPtr);
|
||||
MEM_STATIC U64 MEM_readLE64(const void* memPtr);
|
||||
MEM_STATIC size_t MEM_readLEST(const void* memPtr);
|
||||
|
||||
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val);
|
||||
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val);
|
||||
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32);
|
||||
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64);
|
||||
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val);
|
||||
|
||||
/*=== Big endian unaligned read/write ===*/
|
||||
MEM_STATIC U32 MEM_readBE32(const void* memPtr);
|
||||
MEM_STATIC U64 MEM_readBE64(const void* memPtr);
|
||||
MEM_STATIC size_t MEM_readBEST(const void* memPtr);
|
||||
|
||||
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32);
|
||||
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64);
|
||||
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val);
|
||||
|
||||
/*=== Byteswap ===*/
|
||||
MEM_STATIC U32 MEM_swap32(U32 in);
|
||||
MEM_STATIC U64 MEM_swap64(U64 in);
|
||||
MEM_STATIC size_t MEM_swapST(size_t in);
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Memory I/O Implementation
|
||||
* Memory I/O
|
||||
*****************************************************************/
|
||||
/* MEM_FORCE_MEMORY_ACCESS :
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
@@ -148,7 +173,9 @@ MEM_STATIC size_t MEM_swapST(size_t in);
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define MEM_FORCE_MEMORY_ACCESS 2
|
||||
# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define MEM_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
@@ -158,22 +185,8 @@ MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
|
||||
|
||||
MEM_STATIC unsigned MEM_isLittleEndian(void)
|
||||
{
|
||||
#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
return 1;
|
||||
#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
|
||||
return 0;
|
||||
#elif defined(__clang__) && __LITTLE_ENDIAN__
|
||||
return 1;
|
||||
#elif defined(__clang__) && __BIG_ENDIAN__
|
||||
return 0;
|
||||
#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
|
||||
return 1;
|
||||
#elif defined(__DMC__) && defined(_M_IX86)
|
||||
return 1;
|
||||
#else
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
|
||||
return one.c[0];
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
||||
@@ -223,37 +236,37 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v =
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
||||
{
|
||||
U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr)
|
||||
{
|
||||
U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr)
|
||||
{
|
||||
U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readST(const void* memPtr)
|
||||
{
|
||||
size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
||||
{
|
||||
ZSTD_memcpy(memPtr, &value, sizeof(value));
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
|
||||
{
|
||||
ZSTD_memcpy(memPtr, &value, sizeof(value));
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
||||
{
|
||||
ZSTD_memcpy(memPtr, &value, sizeof(value));
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
||||
@@ -325,7 +338,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
||||
|
||||
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
||||
{
|
||||
return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
|
||||
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
|
||||
@@ -432,14 +445,9 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
|
||||
MEM_writeBE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
/* code only tested on 32 and 64 bits systems */
|
||||
MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MEM_H_MODULE */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
60
vendor/github.com/DataDog/zstd/pool.c
generated
vendored
60
vendor/github.com/DataDog/zstd/pool.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -11,9 +10,9 @@
|
||||
|
||||
|
||||
/* ====== Dependencies ======= */
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "debug.h" /* assert */
|
||||
#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */
|
||||
#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */
|
||||
#include "pool.h"
|
||||
|
||||
/* ====== Compiler specifics ====== */
|
||||
@@ -87,7 +86,7 @@ static void* POOL_thread(void* opaque) {
|
||||
{ POOL_job const job = ctx->queue[ctx->queueHead];
|
||||
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
|
||||
ctx->numThreadsBusy++;
|
||||
ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
|
||||
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
|
||||
/* Unlock the mutex, signal a pusher, and run the job */
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
@@ -106,30 +105,24 @@ static void* POOL_thread(void* opaque) {
|
||||
assert(0); /* Unreachable */
|
||||
}
|
||||
|
||||
/* ZSTD_createThreadPool() : public access point */
|
||||
POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
|
||||
return POOL_create (numThreads, 0);
|
||||
}
|
||||
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
||||
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
ZSTD_customMem customMem)
|
||||
{
|
||||
ZSTD_customMem customMem) {
|
||||
POOL_ctx* ctx;
|
||||
/* Check parameters */
|
||||
if (!numThreads) { return NULL; }
|
||||
/* Allocate the context and zero initialize */
|
||||
ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem);
|
||||
ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
|
||||
if (!ctx) { return NULL; }
|
||||
/* Initialize the job queue.
|
||||
* It needs one extra space since one space is wasted to differentiate
|
||||
* empty and full queues.
|
||||
*/
|
||||
ctx->queueSize = queueSize + 1;
|
||||
ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
||||
ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
||||
ctx->queueHead = 0;
|
||||
ctx->queueTail = 0;
|
||||
ctx->numThreadsBusy = 0;
|
||||
@@ -143,7 +136,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
}
|
||||
ctx->shutdown = 0;
|
||||
/* Allocate space for the thread handles */
|
||||
ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
||||
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
||||
ctx->threadCapacity = 0;
|
||||
ctx->customMem = customMem;
|
||||
/* Check for errors */
|
||||
@@ -186,16 +179,14 @@ void POOL_free(POOL_ctx *ctx) {
|
||||
ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
|
||||
ZSTD_customFree(ctx->queue, ctx->customMem);
|
||||
ZSTD_customFree(ctx->threads, ctx->customMem);
|
||||
ZSTD_customFree(ctx, ctx->customMem);
|
||||
ZSTD_free(ctx->queue, ctx->customMem);
|
||||
ZSTD_free(ctx->threads, ctx->customMem);
|
||||
ZSTD_free(ctx, ctx->customMem);
|
||||
}
|
||||
|
||||
void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
|
||||
POOL_free (pool);
|
||||
}
|
||||
|
||||
size_t POOL_sizeof(const POOL_ctx* ctx) {
|
||||
|
||||
size_t POOL_sizeof(POOL_ctx *ctx) {
|
||||
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
||||
return sizeof(*ctx)
|
||||
+ ctx->queueSize * sizeof(POOL_job)
|
||||
@@ -212,11 +203,11 @@ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
|
||||
return 0;
|
||||
}
|
||||
/* numThreads > threadCapacity */
|
||||
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
||||
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
||||
if (!threadPool) return 1;
|
||||
/* replace existing thread pool */
|
||||
ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
||||
ZSTD_customFree(ctx->threads, ctx->customMem);
|
||||
memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
||||
ZSTD_free(ctx->threads, ctx->customMem);
|
||||
ctx->threads = threadPool;
|
||||
/* Initialize additional threads */
|
||||
{ size_t threadId;
|
||||
@@ -260,8 +251,7 @@ static int isQueueFull(POOL_ctx const* ctx) {
|
||||
}
|
||||
|
||||
|
||||
static void
|
||||
POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
||||
static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
||||
{
|
||||
POOL_job const job = {function, opaque};
|
||||
assert(ctx != NULL);
|
||||
@@ -311,23 +301,21 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
|
||||
struct POOL_ctx_s {
|
||||
int dummy;
|
||||
};
|
||||
static POOL_ctx g_poolCtx;
|
||||
static POOL_ctx g_ctx;
|
||||
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
||||
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
POOL_ctx*
|
||||
POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
|
||||
{
|
||||
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
|
||||
(void)numThreads;
|
||||
(void)queueSize;
|
||||
(void)customMem;
|
||||
return &g_poolCtx;
|
||||
return &g_ctx;
|
||||
}
|
||||
|
||||
void POOL_free(POOL_ctx* ctx) {
|
||||
assert(!ctx || ctx == &g_poolCtx);
|
||||
assert(!ctx || ctx == &g_ctx);
|
||||
(void)ctx;
|
||||
}
|
||||
|
||||
@@ -347,12 +335,10 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t POOL_sizeof(const POOL_ctx* ctx) {
|
||||
size_t POOL_sizeof(POOL_ctx* ctx) {
|
||||
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
||||
assert(ctx == &g_poolCtx);
|
||||
assert(ctx == &g_ctx);
|
||||
return sizeof(*ctx);
|
||||
}
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
11
vendor/github.com/DataDog/zstd/pool.h
generated
vendored
11
vendor/github.com/DataDog/zstd/pool.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -17,7 +16,7 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
#include "zstd_deps.h"
|
||||
#include <stddef.h> /* size_t */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
|
||||
#include "zstd.h"
|
||||
|
||||
@@ -54,7 +53,7 @@ int POOL_resize(POOL_ctx* ctx, size_t numThreads);
|
||||
* @return threadpool memory usage
|
||||
* note : compatible with NULL (returns 0 in this case)
|
||||
*/
|
||||
size_t POOL_sizeof(const POOL_ctx* ctx);
|
||||
size_t POOL_sizeof(POOL_ctx* ctx);
|
||||
|
||||
/*! POOL_function :
|
||||
* The function type that can be added to a thread pool.
|
||||
@@ -71,7 +70,7 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
|
||||
|
||||
/*! POOL_tryAdd() :
|
||||
* Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
|
||||
* Add the job `function(opaque)` to thread pool _if_ a worker is available.
|
||||
* Returns immediately even if not (does not block).
|
||||
* @return : 1 if successful, 0 if not.
|
||||
*/
|
||||
@@ -83,5 +82,3 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
140
vendor/github.com/DataDog/zstd/portability_macros.h
generated
vendored
140
vendor/github.com/DataDog/zstd/portability_macros.h
generated
vendored
@@ -1,140 +0,0 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_PORTABILITY_MACROS_H
|
||||
#define ZSTD_PORTABILITY_MACROS_H
|
||||
|
||||
/**
|
||||
* This header file contains macro defintions to support portability.
|
||||
* This header is shared between C and ASM code, so it MUST only
|
||||
* contain macro definitions. It MUST not contain any C code.
|
||||
*
|
||||
* This header ONLY defines macros to detect platforms/feature support.
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) 0
|
||||
#endif
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_builtin
|
||||
# define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_feature
|
||||
# define __has_feature(x) 0
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under msan */
|
||||
#ifndef ZSTD_MEMORY_SANITIZER
|
||||
# if __has_feature(memory_sanitizer)
|
||||
# define ZSTD_MEMORY_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_MEMORY_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under asan */
|
||||
#ifndef ZSTD_ADDRESS_SANITIZER
|
||||
# if __has_feature(address_sanitizer)
|
||||
# define ZSTD_ADDRESS_SANITIZER 1
|
||||
# elif defined(__SANITIZE_ADDRESS__)
|
||||
# define ZSTD_ADDRESS_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_ADDRESS_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under dfsan */
|
||||
#ifndef ZSTD_DATAFLOW_SANITIZER
|
||||
# if __has_feature(dataflow_sanitizer)
|
||||
# define ZSTD_DATAFLOW_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_DATAFLOW_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* Mark the internal assembly functions as hidden */
|
||||
#ifdef __ELF__
|
||||
# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
|
||||
#else
|
||||
# define ZSTD_HIDE_ASM_FUNCTION(func)
|
||||
#endif
|
||||
|
||||
/* Enable runtime BMI2 dispatch based on the CPU.
|
||||
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
||||
*/
|
||||
#ifndef DYNAMIC_BMI2
|
||||
#if ((defined(__clang__) && __has_attribute(__target__)) \
|
||||
|| (defined(__GNUC__) \
|
||||
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
|
||||
&& (defined(__x86_64__) || defined(_M_X64)) \
|
||||
&& !defined(__BMI2__)
|
||||
# define DYNAMIC_BMI2 1
|
||||
#else
|
||||
# define DYNAMIC_BMI2 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Only enable assembly for GNUC comptabile compilers,
|
||||
* because other platforms may not support GAS assembly syntax.
|
||||
*
|
||||
* Only enable assembly for Linux / MacOS, other platforms may
|
||||
* work, but they haven't been tested. This could likely be
|
||||
* extended to BSD systems.
|
||||
*
|
||||
* Disable assembly when MSAN is enabled, because MSAN requires
|
||||
* 100% of code to be instrumented to work.
|
||||
*/
|
||||
#if defined(__GNUC__)
|
||||
# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
|
||||
# if ZSTD_MEMORY_SANITIZER
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
# elif ZSTD_DATAFLOW_SANITIZER
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
# else
|
||||
# define ZSTD_ASM_SUPPORTED 1
|
||||
# endif
|
||||
# else
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
# endif
|
||||
#else
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Determines whether we should enable assembly for x86-64
|
||||
* with BMI2.
|
||||
*
|
||||
* Enable if all of the following conditions hold:
|
||||
* - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
|
||||
* - Assembly is supported
|
||||
* - We are compiling for x86-64 and either:
|
||||
* - DYNAMIC_BMI2 is enabled
|
||||
* - BMI2 is supported at compile time
|
||||
*/
|
||||
#if !defined(ZSTD_DISABLE_ASM) && \
|
||||
ZSTD_ASM_SUPPORTED && \
|
||||
defined(__x86_64__) && \
|
||||
(DYNAMIC_BMI2 || defined(__BMI2__))
|
||||
# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
|
||||
#else
|
||||
# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_PORTABILITY_MACROS_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
21
vendor/github.com/DataDog/zstd/threading.c
generated
vendored
21
vendor/github.com/DataDog/zstd/threading.c
generated
vendored
@@ -1,15 +1,13 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/**
|
||||
* Copyright (c) 2016 Tino Reichardt
|
||||
* All rights reserved.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*/
|
||||
|
||||
/**
|
||||
@@ -79,12 +77,11 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
||||
|
||||
#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
|
||||
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#include "zstd_deps.h"
|
||||
#include <stdlib.h>
|
||||
|
||||
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
|
||||
{
|
||||
*mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t));
|
||||
*mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t));
|
||||
if (!*mutex)
|
||||
return 1;
|
||||
return pthread_mutex_init(*mutex, attr);
|
||||
@@ -96,14 +93,14 @@ int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
|
||||
return 0;
|
||||
{
|
||||
int const ret = pthread_mutex_destroy(*mutex);
|
||||
ZSTD_free(*mutex);
|
||||
free(*mutex);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
|
||||
{
|
||||
*cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t));
|
||||
*cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t));
|
||||
if (!*cond)
|
||||
return 1;
|
||||
return pthread_cond_init(*cond, attr);
|
||||
@@ -115,11 +112,9 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
|
||||
return 0;
|
||||
{
|
||||
int const ret = pthread_cond_destroy(*cond);
|
||||
ZSTD_free(*cond);
|
||||
free(*cond);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
10
vendor/github.com/DataDog/zstd/threading.h
generated
vendored
10
vendor/github.com/DataDog/zstd/threading.h
generated
vendored
@@ -1,15 +1,13 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/**
|
||||
* Copyright (c) 2016 Tino Reichardt
|
||||
* All rights reserved.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*/
|
||||
|
||||
#ifndef THREADING_H_938743
|
||||
@@ -154,5 +152,3 @@ typedef int ZSTD_pthread_cond_t;
|
||||
#endif
|
||||
|
||||
#endif /* THREADING_H_938743 */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
8
vendor/github.com/DataDog/zstd/travis_test_32.sh
generated
vendored
8
vendor/github.com/DataDog/zstd/travis_test_32.sh
generated
vendored
@@ -1,8 +1,6 @@
|
||||
#!/bin/bash
|
||||
# Get utilities
|
||||
#yum -y -q -e 0 install wget tar unzip gcc
|
||||
apt-get update
|
||||
apt-get -y install wget tar unzip gcc
|
||||
yum -y -q -e 0 install wget tar unzip gcc
|
||||
|
||||
# Get Go
|
||||
wget -q https://dl.google.com/go/go1.13.linux-386.tar.gz
|
||||
@@ -15,5 +13,5 @@ unzip mr.zip
|
||||
|
||||
# Build and run tests
|
||||
go build
|
||||
DISABLE_BIG_TESTS=1 PAYLOAD=$(pwd)/mr go test -v
|
||||
DISABLE_BIG_TESTS=1 PAYLOAD=$(pwd)/mr go test -bench .
|
||||
PAYLOAD=$(pwd)/mr go test -v
|
||||
PAYLOAD=$(pwd)/mr go test -bench .
|
||||
|
||||
891
vendor/github.com/DataDog/zstd/xxhash.c
generated
vendored
891
vendor/github.com/DataDog/zstd/xxhash.c
generated
vendored
@@ -1,27 +1,882 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* xxHash - Fast Hash algorithm
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - xxHash homepage: http://www.xxhash.com
|
||||
* - xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
* xxHash - Fast Hash algorithm
|
||||
* Copyright (C) 2012-2016, Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - xxHash homepage: http://www.xxhash.com
|
||||
* - xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* xxhash.c instantiates functions defined in xxhash.h
|
||||
/* *************************************
|
||||
* Tuning parameters
|
||||
***************************************/
|
||||
/*!XXH_FORCE_MEMORY_ACCESS :
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
* Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
|
||||
* It can generate buggy code on targets which do not support unaligned memory accesses.
|
||||
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
||||
* See http://stackoverflow.com/a/32095106/646947 for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define XXH_FORCE_MEMORY_ACCESS 2
|
||||
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
||||
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \
|
||||
defined(__ICCARM__)
|
||||
# define XXH_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
|
||||
#define XXH_IMPLEMENTATION /* access definitions */
|
||||
/*!XXH_ACCEPT_NULL_INPUT_POINTER :
|
||||
* If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
|
||||
* When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
|
||||
* By default, this option is disabled. To enable it, uncomment below define :
|
||||
*/
|
||||
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
|
||||
|
||||
/*!XXH_FORCE_NATIVE_FORMAT :
|
||||
* By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
|
||||
* Results are therefore identical for little-endian and big-endian CPU.
|
||||
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
|
||||
* Should endian-independence be of no importance for your application, you may set the #define below to 1,
|
||||
* to improve speed for Big-endian CPU.
|
||||
* This option has no impact on Little_Endian CPU.
|
||||
*/
|
||||
#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
|
||||
# define XXH_FORCE_NATIVE_FORMAT 0
|
||||
#endif
|
||||
|
||||
/*!XXH_FORCE_ALIGN_CHECK :
|
||||
* This is a minor performance trick, only useful with lots of very small keys.
|
||||
* It means : check for aligned/unaligned input.
|
||||
* The check costs one initial branch per hash; set to 0 when the input data
|
||||
* is guaranteed to be aligned.
|
||||
*/
|
||||
#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
|
||||
# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
|
||||
# define XXH_FORCE_ALIGN_CHECK 0
|
||||
# else
|
||||
# define XXH_FORCE_ALIGN_CHECK 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Includes & Memory related functions
|
||||
***************************************/
|
||||
/* Modify the local functions below should you wish to use some other memory routines */
|
||||
/* for malloc(), free() */
|
||||
#include <stdlib.h>
|
||||
#include <stddef.h> /* size_t */
|
||||
static void* XXH_malloc(size_t s) { return malloc(s); }
|
||||
static void XXH_free (void* p) { free(p); }
|
||||
/* for memcpy() */
|
||||
#include <string.h>
|
||||
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
|
||||
|
||||
#ifndef XXH_STATIC_LINKING_ONLY
|
||||
# define XXH_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "xxhash.h"
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
/* *************************************
|
||||
* Compiler Specific Options
|
||||
***************************************/
|
||||
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# define INLINE_KEYWORD inline
|
||||
#else
|
||||
# define INLINE_KEYWORD
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define FORCE_INLINE_ATTR __attribute__((always_inline))
|
||||
#elif defined(_MSC_VER)
|
||||
# define FORCE_INLINE_ATTR __forceinline
|
||||
#else
|
||||
# define FORCE_INLINE_ATTR
|
||||
#endif
|
||||
|
||||
#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
|
||||
|
||||
|
||||
#ifdef _MSC_VER
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Basic Types
|
||||
***************************************/
|
||||
#ifndef MEM_MODULE
|
||||
# define MEM_MODULE
|
||||
# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
# else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
|
||||
|
||||
/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
|
||||
static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
||||
static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
||||
|
||||
#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
||||
|
||||
static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
||||
static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
||||
|
||||
#else
|
||||
|
||||
/* portable and safe solution. Generally efficient.
|
||||
* see : http://stackoverflow.com/a/32095106/646947
|
||||
*/
|
||||
|
||||
static U32 XXH_read32(const void* memPtr)
|
||||
{
|
||||
U32 val;
|
||||
memcpy(&val, memPtr, sizeof(val));
|
||||
return val;
|
||||
}
|
||||
|
||||
static U64 XXH_read64(const void* memPtr)
|
||||
{
|
||||
U64 val;
|
||||
memcpy(&val, memPtr, sizeof(val));
|
||||
return val;
|
||||
}
|
||||
|
||||
#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Compiler-specific Functions and Macros
|
||||
******************************************/
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
|
||||
/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
|
||||
#if defined(_MSC_VER)
|
||||
# define XXH_rotl32(x,r) _rotl(x,r)
|
||||
# define XXH_rotl64(x,r) _rotl64(x,r)
|
||||
#else
|
||||
#if defined(__ICCARM__)
|
||||
# include <intrinsics.h>
|
||||
# define XXH_rotl32(x,r) __ROR(x,(32 - r))
|
||||
#else
|
||||
# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
#endif
|
||||
# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# define XXH_swap32 _byteswap_ulong
|
||||
# define XXH_swap64 _byteswap_uint64
|
||||
#elif GCC_VERSION >= 403
|
||||
# define XXH_swap32 __builtin_bswap32
|
||||
# define XXH_swap64 __builtin_bswap64
|
||||
#else
|
||||
static U32 XXH_swap32 (U32 x)
|
||||
{
|
||||
return ((x << 24) & 0xff000000 ) |
|
||||
((x << 8) & 0x00ff0000 ) |
|
||||
((x >> 8) & 0x0000ff00 ) |
|
||||
((x >> 24) & 0x000000ff );
|
||||
}
|
||||
static U64 XXH_swap64 (U64 x)
|
||||
{
|
||||
return ((x << 56) & 0xff00000000000000ULL) |
|
||||
((x << 40) & 0x00ff000000000000ULL) |
|
||||
((x << 24) & 0x0000ff0000000000ULL) |
|
||||
((x << 8) & 0x000000ff00000000ULL) |
|
||||
((x >> 8) & 0x00000000ff000000ULL) |
|
||||
((x >> 24) & 0x0000000000ff0000ULL) |
|
||||
((x >> 40) & 0x000000000000ff00ULL) |
|
||||
((x >> 56) & 0x00000000000000ffULL);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Architecture Macros
|
||||
***************************************/
|
||||
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
|
||||
|
||||
/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
|
||||
#ifndef XXH_CPU_LITTLE_ENDIAN
|
||||
static const int g_one = 1;
|
||||
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
|
||||
#endif
|
||||
|
||||
|
||||
/* ***************************
|
||||
* Memory reads
|
||||
*****************************/
|
||||
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
if (align==XXH_unaligned)
|
||||
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
|
||||
else
|
||||
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
|
||||
{
|
||||
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
|
||||
}
|
||||
|
||||
static U32 XXH_readBE32(const void* ptr)
|
||||
{
|
||||
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
if (align==XXH_unaligned)
|
||||
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
|
||||
else
|
||||
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
|
||||
{
|
||||
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
|
||||
}
|
||||
|
||||
static U64 XXH_readBE64(const void* ptr)
|
||||
{
|
||||
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
|
||||
}
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
static const U32 PRIME32_1 = 2654435761U;
|
||||
static const U32 PRIME32_2 = 2246822519U;
|
||||
static const U32 PRIME32_3 = 3266489917U;
|
||||
static const U32 PRIME32_4 = 668265263U;
|
||||
static const U32 PRIME32_5 = 374761393U;
|
||||
|
||||
static const U64 PRIME64_1 = 11400714785074694791ULL;
|
||||
static const U64 PRIME64_2 = 14029467366897019727ULL;
|
||||
static const U64 PRIME64_3 = 1609587929392839161ULL;
|
||||
static const U64 PRIME64_4 = 9650029242287828579ULL;
|
||||
static const U64 PRIME64_5 = 2870177450012600261ULL;
|
||||
|
||||
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
|
||||
|
||||
|
||||
/* **************************
|
||||
* Utils
|
||||
****************************/
|
||||
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)
|
||||
{
|
||||
memcpy(dstState, srcState, sizeof(*dstState));
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState)
|
||||
{
|
||||
memcpy(dstState, srcState, sizeof(*dstState));
|
||||
}
|
||||
|
||||
|
||||
/* ***************************
|
||||
* Simple Hash Functions
|
||||
*****************************/
|
||||
|
||||
static U32 XXH32_round(U32 seed, U32 input)
|
||||
{
|
||||
seed += input * PRIME32_2;
|
||||
seed = XXH_rotl32(seed, 13);
|
||||
seed *= PRIME32_1;
|
||||
return seed;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* bEnd = p + len;
|
||||
U32 h32;
|
||||
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (p==NULL) {
|
||||
len=0;
|
||||
bEnd=p=(const BYTE*)(size_t)16;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (len>=16) {
|
||||
const BYTE* const limit = bEnd - 16;
|
||||
U32 v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
U32 v2 = seed + PRIME32_2;
|
||||
U32 v3 = seed + 0;
|
||||
U32 v4 = seed - PRIME32_1;
|
||||
|
||||
do {
|
||||
v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
|
||||
v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
|
||||
v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
|
||||
v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
|
||||
} while (p<=limit);
|
||||
|
||||
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
|
||||
} else {
|
||||
h32 = seed + PRIME32_5;
|
||||
}
|
||||
|
||||
h32 += (U32) len;
|
||||
|
||||
while (p+4<=bEnd) {
|
||||
h32 += XXH_get32bits(p) * PRIME32_3;
|
||||
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h32 += (*p) * PRIME32_5;
|
||||
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
|
||||
p++;
|
||||
}
|
||||
|
||||
h32 ^= h32 >> 15;
|
||||
h32 *= PRIME32_2;
|
||||
h32 ^= h32 >> 13;
|
||||
h32 *= PRIME32_3;
|
||||
h32 ^= h32 >> 16;
|
||||
|
||||
return h32;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
|
||||
{
|
||||
#if 0
|
||||
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
|
||||
XXH32_CREATESTATE_STATIC(state);
|
||||
XXH32_reset(state, seed);
|
||||
XXH32_update(state, input, len);
|
||||
return XXH32_digest(state);
|
||||
#else
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if (XXH_FORCE_ALIGN_CHECK) {
|
||||
if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
|
||||
else
|
||||
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
|
||||
} }
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
|
||||
else
|
||||
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
static U64 XXH64_round(U64 acc, U64 input)
|
||||
{
|
||||
acc += input * PRIME64_2;
|
||||
acc = XXH_rotl64(acc, 31);
|
||||
acc *= PRIME64_1;
|
||||
return acc;
|
||||
}
|
||||
|
||||
static U64 XXH64_mergeRound(U64 acc, U64 val)
|
||||
{
|
||||
val = XXH64_round(0, val);
|
||||
acc ^= val;
|
||||
acc = acc * PRIME64_1 + PRIME64_4;
|
||||
return acc;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* const bEnd = p + len;
|
||||
U64 h64;
|
||||
#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (p==NULL) {
|
||||
len=0;
|
||||
bEnd=p=(const BYTE*)(size_t)32;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (len>=32) {
|
||||
const BYTE* const limit = bEnd - 32;
|
||||
U64 v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
U64 v2 = seed + PRIME64_2;
|
||||
U64 v3 = seed + 0;
|
||||
U64 v4 = seed - PRIME64_1;
|
||||
|
||||
do {
|
||||
v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
|
||||
v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
|
||||
v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
|
||||
v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
|
||||
} while (p<=limit);
|
||||
|
||||
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
|
||||
h64 = XXH64_mergeRound(h64, v1);
|
||||
h64 = XXH64_mergeRound(h64, v2);
|
||||
h64 = XXH64_mergeRound(h64, v3);
|
||||
h64 = XXH64_mergeRound(h64, v4);
|
||||
|
||||
} else {
|
||||
h64 = seed + PRIME64_5;
|
||||
}
|
||||
|
||||
h64 += (U64) len;
|
||||
|
||||
while (p+8<=bEnd) {
|
||||
U64 const k1 = XXH64_round(0, XXH_get64bits(p));
|
||||
h64 ^= k1;
|
||||
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
|
||||
p+=8;
|
||||
}
|
||||
|
||||
if (p+4<=bEnd) {
|
||||
h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
|
||||
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h64 ^= (*p) * PRIME64_5;
|
||||
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
|
||||
p++;
|
||||
}
|
||||
|
||||
h64 ^= h64 >> 33;
|
||||
h64 *= PRIME64_2;
|
||||
h64 ^= h64 >> 29;
|
||||
h64 *= PRIME64_3;
|
||||
h64 ^= h64 >> 32;
|
||||
|
||||
return h64;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
|
||||
{
|
||||
#if 0
|
||||
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
|
||||
XXH64_CREATESTATE_STATIC(state);
|
||||
XXH64_reset(state, seed);
|
||||
XXH64_update(state, input, len);
|
||||
return XXH64_digest(state);
|
||||
#else
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if (XXH_FORCE_ALIGN_CHECK) {
|
||||
if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
|
||||
else
|
||||
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
|
||||
} }
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
|
||||
else
|
||||
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/* **************************************************
|
||||
* Advanced Hash Functions
|
||||
****************************************************/
|
||||
|
||||
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
|
||||
{
|
||||
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
|
||||
}
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
|
||||
{
|
||||
XXH_free(statePtr);
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
|
||||
{
|
||||
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
|
||||
}
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
|
||||
{
|
||||
XXH_free(statePtr);
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
|
||||
/*** Hash feed ***/
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
|
||||
{
|
||||
XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
|
||||
memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */
|
||||
state.v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
state.v2 = seed + PRIME32_2;
|
||||
state.v3 = seed + 0;
|
||||
state.v4 = seed - PRIME32_1;
|
||||
memcpy(statePtr, &state, sizeof(state));
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
|
||||
{
|
||||
XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
|
||||
memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */
|
||||
state.v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
state.v2 = seed + PRIME64_2;
|
||||
state.v3 = seed + 0;
|
||||
state.v4 = seed - PRIME64_1;
|
||||
memcpy(statePtr, &state, sizeof(state));
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* const bEnd = p + len;
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (input==NULL) return XXH_ERROR;
|
||||
#endif
|
||||
|
||||
state->total_len_32 += (unsigned)len;
|
||||
state->large_len |= (len>=16) | (state->total_len_32>=16);
|
||||
|
||||
if (state->memsize + len < 16) { /* fill in tmp buffer */
|
||||
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
|
||||
state->memsize += (unsigned)len;
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
if (state->memsize) { /* some data left from previous update */
|
||||
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
|
||||
{ const U32* p32 = state->mem32;
|
||||
state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
|
||||
state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
|
||||
state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
|
||||
state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
|
||||
}
|
||||
p += 16-state->memsize;
|
||||
state->memsize = 0;
|
||||
}
|
||||
|
||||
if (p <= bEnd-16) {
|
||||
const BYTE* const limit = bEnd - 16;
|
||||
U32 v1 = state->v1;
|
||||
U32 v2 = state->v2;
|
||||
U32 v3 = state->v3;
|
||||
U32 v4 = state->v4;
|
||||
|
||||
do {
|
||||
v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
|
||||
v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
|
||||
v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
|
||||
v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
|
||||
} while (p<=limit);
|
||||
|
||||
state->v1 = v1;
|
||||
state->v2 = v2;
|
||||
state->v3 = v3;
|
||||
state->v4 = v4;
|
||||
}
|
||||
|
||||
if (p < bEnd) {
|
||||
XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
|
||||
state->memsize = (unsigned)(bEnd-p);
|
||||
}
|
||||
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
|
||||
else
|
||||
return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
|
||||
{
|
||||
const BYTE * p = (const BYTE*)state->mem32;
|
||||
const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
|
||||
U32 h32;
|
||||
|
||||
if (state->large_len) {
|
||||
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
|
||||
} else {
|
||||
h32 = state->v3 /* == seed */ + PRIME32_5;
|
||||
}
|
||||
|
||||
h32 += state->total_len_32;
|
||||
|
||||
while (p+4<=bEnd) {
|
||||
h32 += XXH_readLE32(p, endian) * PRIME32_3;
|
||||
h32 = XXH_rotl32(h32, 17) * PRIME32_4;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h32 += (*p) * PRIME32_5;
|
||||
h32 = XXH_rotl32(h32, 11) * PRIME32_1;
|
||||
p++;
|
||||
}
|
||||
|
||||
h32 ^= h32 >> 15;
|
||||
h32 *= PRIME32_2;
|
||||
h32 ^= h32 >> 13;
|
||||
h32 *= PRIME32_3;
|
||||
h32 ^= h32 >> 16;
|
||||
|
||||
return h32;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH32_digest_endian(state_in, XXH_littleEndian);
|
||||
else
|
||||
return XXH32_digest_endian(state_in, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* **** XXH64 **** */
|
||||
|
||||
FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)input;
|
||||
const BYTE* const bEnd = p + len;
|
||||
|
||||
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
|
||||
if (input==NULL) return XXH_ERROR;
|
||||
#endif
|
||||
|
||||
state->total_len += len;
|
||||
|
||||
if (state->memsize + len < 32) { /* fill in tmp buffer */
|
||||
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
|
||||
state->memsize += (U32)len;
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
if (state->memsize) { /* tmp buffer is full */
|
||||
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
|
||||
state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
|
||||
state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
|
||||
state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
|
||||
state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
|
||||
p += 32-state->memsize;
|
||||
state->memsize = 0;
|
||||
}
|
||||
|
||||
if (p+32 <= bEnd) {
|
||||
const BYTE* const limit = bEnd - 32;
|
||||
U64 v1 = state->v1;
|
||||
U64 v2 = state->v2;
|
||||
U64 v3 = state->v3;
|
||||
U64 v4 = state->v4;
|
||||
|
||||
do {
|
||||
v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
|
||||
v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
|
||||
v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
|
||||
v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
|
||||
} while (p<=limit);
|
||||
|
||||
state->v1 = v1;
|
||||
state->v2 = v2;
|
||||
state->v3 = v3;
|
||||
state->v4 = v4;
|
||||
}
|
||||
|
||||
if (p < bEnd) {
|
||||
XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
|
||||
state->memsize = (unsigned)(bEnd-p);
|
||||
}
|
||||
|
||||
return XXH_OK;
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
|
||||
else
|
||||
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
|
||||
{
|
||||
const BYTE * p = (const BYTE*)state->mem64;
|
||||
const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
|
||||
U64 h64;
|
||||
|
||||
if (state->total_len >= 32) {
|
||||
U64 const v1 = state->v1;
|
||||
U64 const v2 = state->v2;
|
||||
U64 const v3 = state->v3;
|
||||
U64 const v4 = state->v4;
|
||||
|
||||
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
|
||||
h64 = XXH64_mergeRound(h64, v1);
|
||||
h64 = XXH64_mergeRound(h64, v2);
|
||||
h64 = XXH64_mergeRound(h64, v3);
|
||||
h64 = XXH64_mergeRound(h64, v4);
|
||||
} else {
|
||||
h64 = state->v3 + PRIME64_5;
|
||||
}
|
||||
|
||||
h64 += (U64) state->total_len;
|
||||
|
||||
while (p+8<=bEnd) {
|
||||
U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
|
||||
h64 ^= k1;
|
||||
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
|
||||
p+=8;
|
||||
}
|
||||
|
||||
if (p+4<=bEnd) {
|
||||
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
|
||||
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
p+=4;
|
||||
}
|
||||
|
||||
while (p<bEnd) {
|
||||
h64 ^= (*p) * PRIME64_5;
|
||||
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
|
||||
p++;
|
||||
}
|
||||
|
||||
h64 ^= h64 >> 33;
|
||||
h64 *= PRIME64_2;
|
||||
h64 ^= h64 >> 29;
|
||||
h64 *= PRIME64_3;
|
||||
h64 ^= h64 >> 32;
|
||||
|
||||
return h64;
|
||||
}
|
||||
|
||||
|
||||
XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
|
||||
{
|
||||
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
|
||||
|
||||
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
|
||||
return XXH64_digest_endian(state_in, XXH_littleEndian);
|
||||
else
|
||||
return XXH64_digest_endian(state_in, XXH_bigEndian);
|
||||
}
|
||||
|
||||
|
||||
/* **************************
|
||||
* Canonical representation
|
||||
****************************/
|
||||
|
||||
/*! Default XXH result types are basic unsigned 32 and 64 bits.
|
||||
* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
|
||||
* These functions allow transformation of hash result into and from its canonical format.
|
||||
* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
|
||||
*/
|
||||
|
||||
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
|
||||
{
|
||||
XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
|
||||
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
|
||||
memcpy(dst, &hash, sizeof(*dst));
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
|
||||
{
|
||||
XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
|
||||
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
|
||||
memcpy(dst, &hash, sizeof(*dst));
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
|
||||
{
|
||||
return XXH_readBE32(src);
|
||||
}
|
||||
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
|
||||
{
|
||||
return XXH_readBE64(src);
|
||||
}
|
||||
|
||||
5794
vendor/github.com/DataDog/zstd/xxhash.h
generated
vendored
5794
vendor/github.com/DataDog/zstd/xxhash.h
generated
vendored
File diff suppressed because it is too large
Load Diff
5
vendor/github.com/DataDog/zstd/zbuff.h
generated
vendored
5
vendor/github.com/DataDog/zstd/zbuff.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -213,5 +212,3 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
5
vendor/github.com/DataDog/zstd/zbuff_common.c
generated
vendored
5
vendor/github.com/DataDog/zstd/zbuff_common.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -25,5 +24,3 @@ unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
/*! ZBUFF_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
31
vendor/github.com/DataDog/zstd/zbuff_compress.c
generated
vendored
31
vendor/github.com/DataDog/zstd/zbuff_compress.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -16,7 +15,6 @@
|
||||
***************************************/
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h"
|
||||
#include "error_private.h"
|
||||
|
||||
|
||||
/*-***********************************************************
|
||||
@@ -75,32 +73,13 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* preserve "0 == unknown" behavior */
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setPledgedSrcSize(zbc, pledgedSrcSize), "");
|
||||
|
||||
FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_windowLog, params.cParams.windowLog), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_hashLog, params.cParams.hashLog), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_chainLog, params.cParams.chainLog), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_searchLog, params.cParams.searchLog), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_minMatch, params.cParams.minMatch), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_targetLength, params.cParams.targetLength), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_strategy, params.cParams.strategy), "");
|
||||
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_contentSizeFlag, params.fParams.contentSizeFlag), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_checksumFlag, params.fParams.checksumFlag), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_dictIDFlag, params.fParams.noDictIDFlag), "");
|
||||
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_loadDictionary(zbc, dict, dictSize), "");
|
||||
return 0;
|
||||
return ZSTD_initCStream_advanced(zbc, dict, dictSize, params, pledgedSrcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(zbc, ZSTD_c_compressionLevel, compressionLevel), "");
|
||||
FORWARD_IF_ERROR(ZSTD_CCtx_loadDictionary(zbc, dict, dictSize), "");
|
||||
return 0;
|
||||
return ZSTD_initCStream_usingDict(zbc, dict, dictSize, compressionLevel);
|
||||
}
|
||||
|
||||
size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
|
||||
@@ -166,5 +145,3 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
***************************************/
|
||||
size_t ZBUFF_recommendedCInSize(void) { return ZSTD_CStreamInSize(); }
|
||||
size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_CStreamOutSize(); }
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
5
vendor/github.com/DataDog/zstd/zbuff_decompress.c
generated
vendored
5
vendor/github.com/DataDog/zstd/zbuff_decompress.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -74,5 +73,3 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
***************************************/
|
||||
size_t ZBUFF_recommendedDInSize(void) { return ZSTD_DStreamInSize(); }
|
||||
size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_DStreamOutSize(); }
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
209
vendor/github.com/DataDog/zstd/zdict.c
generated
vendored
209
vendor/github.com/DataDog/zstd/zdict.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -24,13 +23,9 @@
|
||||
/* Unix Large Files support (>4GB) */
|
||||
#define _FILE_OFFSET_BITS 64
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
|
||||
# ifndef _LARGEFILE_SOURCE
|
||||
# define _LARGEFILE_SOURCE
|
||||
# endif
|
||||
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
|
||||
# ifndef _LARGEFILE64_SOURCE
|
||||
# define _LARGEFILE64_SOURCE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
@@ -42,19 +37,17 @@
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <time.h> /* clock */
|
||||
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
|
||||
#include "mem.h" /* read */
|
||||
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
#include "zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
|
||||
#include "zdict.h"
|
||||
#include "divsufsort.h"
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@@ -68,15 +61,14 @@
|
||||
|
||||
#define NOISELENGTH 32
|
||||
|
||||
static const int g_compressionLevel_default = 3;
|
||||
static const U32 g_selectivity_default = 9;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#undef DISPLAY
|
||||
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
|
||||
#undef DISPLAYLEVEL
|
||||
#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||
|
||||
static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
|
||||
@@ -107,26 +99,6 @@ unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
|
||||
return MEM_readLE32((const char*)dictBuffer + 4);
|
||||
}
|
||||
|
||||
size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
|
||||
{
|
||||
size_t headerSize;
|
||||
if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
|
||||
|
||||
{ ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
|
||||
U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
|
||||
if (!bs || !wksp) {
|
||||
headerSize = ERROR(memory_allocation);
|
||||
} else {
|
||||
ZSTD_reset_compressedBlockState(bs);
|
||||
headerSize = ZSTD_loadCEntropy(bs, wksp, dictBuffer, dictSize);
|
||||
}
|
||||
|
||||
free(bs);
|
||||
free(wksp);
|
||||
}
|
||||
|
||||
return headerSize;
|
||||
}
|
||||
|
||||
/*-********************************************************
|
||||
* Dictionary training functions
|
||||
@@ -136,32 +108,22 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward64(&r, (U64)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
unsigned long r = 0;
|
||||
_BitScanForward64( &r, (U64)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_ctzll((U64)val) >> 3);
|
||||
return (__builtin_ctzll((U64)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
|
||||
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, (U32)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
unsigned long r=0;
|
||||
_BitScanForward( &r, (U32)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_ctz((U32)val) >> 3);
|
||||
return (__builtin_ctz((U32)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
|
||||
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
||||
@@ -170,16 +132,11 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
|
||||
} else { /* Big Endian CPU */
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse64(&r, val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse64( &r, val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_clzll(val) >> 3);
|
||||
return (__builtin_clzll(val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
|
||||
@@ -190,16 +147,11 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, (unsigned long)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse( &r, (unsigned long)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_clz((U32)val) >> 3);
|
||||
return (__builtin_clz((U32)val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
|
||||
@@ -256,7 +208,7 @@ static dictItem ZDICT_analyzePos(
|
||||
U32 savings[LLIMIT] = {0};
|
||||
const BYTE* b = (const BYTE*)buffer;
|
||||
size_t maxLength = LLIMIT;
|
||||
size_t pos = (size_t)suffix[start];
|
||||
size_t pos = suffix[start];
|
||||
U32 end = start;
|
||||
dictItem solution;
|
||||
|
||||
@@ -390,7 +342,7 @@ static dictItem ZDICT_analyzePos(
|
||||
savings[i] = savings[i-1] + (lengthList[i] * (i-3));
|
||||
|
||||
DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
|
||||
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength);
|
||||
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength);
|
||||
|
||||
solution.pos = (U32)pos;
|
||||
solution.length = (U32)maxLength;
|
||||
@@ -400,7 +352,7 @@ static dictItem ZDICT_analyzePos(
|
||||
{ U32 id;
|
||||
for (id=start; id<end; id++) {
|
||||
U32 p, pEnd, length;
|
||||
U32 const testedPos = (U32)suffix[id];
|
||||
U32 const testedPos = suffix[id];
|
||||
if (testedPos == pos)
|
||||
length = solution.length;
|
||||
else {
|
||||
@@ -463,7 +415,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
|
||||
|
||||
if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
|
||||
/* append */
|
||||
int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length);
|
||||
int const addedLength = (int)eltEnd - (table[u].pos + table[u].length);
|
||||
table[u].savings += elt.length / 8; /* rough approx bonus */
|
||||
if (addedLength > 0) { /* otherwise, elt fully included into existing */
|
||||
table[u].length += addedLength;
|
||||
@@ -556,7 +508,6 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
|
||||
clock_t displayClock = 0;
|
||||
clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
|
||||
|
||||
# undef DISPLAYUPDATE
|
||||
# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
|
||||
if (ZDICT_clockSpan(displayClock) > refreshRate) \
|
||||
{ displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
||||
@@ -637,12 +588,12 @@ typedef struct
|
||||
|
||||
#define MAXREPOFFSET 1024
|
||||
|
||||
static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
|
||||
static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
|
||||
unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets,
|
||||
const void* src, size_t srcSize,
|
||||
U32 notificationLevel)
|
||||
{
|
||||
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params->cParams.windowLog);
|
||||
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog);
|
||||
size_t cSize;
|
||||
|
||||
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
|
||||
@@ -683,8 +634,8 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
|
||||
|
||||
if (nbSeq >= 2) { /* rep offsets */
|
||||
const seqDef* const seq = seqStorePtr->sequencesStart;
|
||||
U32 offset1 = seq[0].offBase - ZSTD_REP_NUM;
|
||||
U32 offset2 = seq[1].offBase - ZSTD_REP_NUM;
|
||||
U32 offset1 = seq[0].offset - 3;
|
||||
U32 offset2 = seq[1].offset - 3;
|
||||
if (offset1 >= MAXREPOFFSET) offset1 = 0;
|
||||
if (offset2 >= MAXREPOFFSET) offset2 = 0;
|
||||
repOffsets[offset1] += 3;
|
||||
@@ -731,7 +682,7 @@ static void ZDICT_flatLit(unsigned* countLit)
|
||||
|
||||
#define OFFCODE_MAX 30 /* only applicable to first block */
|
||||
static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
int compressionLevel,
|
||||
unsigned compressionLevel,
|
||||
const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
unsigned notificationLevel)
|
||||
@@ -766,7 +717,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
memset(repOffset, 0, sizeof(repOffset));
|
||||
repOffset[1] = repOffset[4] = repOffset[8] = 1;
|
||||
memset(bestRepOffset, 0, sizeof(bestRepOffset));
|
||||
if (compressionLevel==0) compressionLevel = ZSTD_CLEVEL_DEFAULT;
|
||||
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
|
||||
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
|
||||
|
||||
esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem);
|
||||
@@ -780,20 +731,13 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
|
||||
/* collect stats on all samples */
|
||||
for (u=0; u<nbFiles; u++) {
|
||||
ZDICT_countEStats(esr, ¶ms,
|
||||
ZDICT_countEStats(esr, params,
|
||||
countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
|
||||
(const char*)srcBuffer + pos, fileSizes[u],
|
||||
notificationLevel);
|
||||
pos += fileSizes[u];
|
||||
}
|
||||
|
||||
if (notificationLevel >= 4) {
|
||||
/* writeStats */
|
||||
DISPLAYLEVEL(4, "Offset Code Frequencies : \n");
|
||||
for (u=0; u<=offcodeMax; u++) {
|
||||
DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]);
|
||||
} }
|
||||
|
||||
/* analyze, build stats, starting with literals */
|
||||
{ size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
|
||||
if (HUF_isError(maxNbBits)) {
|
||||
@@ -818,7 +762,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
/* note : the result of this phase should be used to better appreciate the impact on statistics */
|
||||
|
||||
total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u];
|
||||
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax, /* useLowProbCount */ 1);
|
||||
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = errorCode;
|
||||
DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
|
||||
@@ -827,7 +771,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
Offlog = (U32)errorCode;
|
||||
|
||||
total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u];
|
||||
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML, /* useLowProbCount */ 1);
|
||||
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = errorCode;
|
||||
DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
|
||||
@@ -836,7 +780,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
mlLog = (U32)errorCode;
|
||||
|
||||
total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
|
||||
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL, /* useLowProbCount */ 1);
|
||||
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = errorCode;
|
||||
DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
|
||||
@@ -900,7 +844,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset);
|
||||
#else
|
||||
/* at this stage, we don't use the result of "most common first offset",
|
||||
* as the impact of statistics is not properly evaluated */
|
||||
as the impact of statistics is not properly evaluated */
|
||||
MEM_writeLE32(dstPtr+0, repStartValue[0]);
|
||||
MEM_writeLE32(dstPtr+4, repStartValue[1]);
|
||||
MEM_writeLE32(dstPtr+8, repStartValue[2]);
|
||||
@@ -916,17 +860,6 @@ _cleanup:
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @returns the maximum repcode value
|
||||
*/
|
||||
static U32 ZDICT_maxRep(U32 const reps[ZSTD_REP_NUM])
|
||||
{
|
||||
U32 maxRep = reps[0];
|
||||
int r;
|
||||
for (r = 1; r < ZSTD_REP_NUM; ++r)
|
||||
maxRep = MAX(maxRep, reps[r]);
|
||||
return maxRep;
|
||||
}
|
||||
|
||||
size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* customDictContent, size_t dictContentSize,
|
||||
@@ -936,15 +869,13 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
||||
size_t hSize;
|
||||
#define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
|
||||
BYTE header[HBUFFSIZE];
|
||||
int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
|
||||
int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
|
||||
U32 const notificationLevel = params.notificationLevel;
|
||||
/* The final dictionary content must be at least as large as the largest repcode */
|
||||
size_t const minContentSize = (size_t)ZDICT_maxRep(repStartValue);
|
||||
size_t paddingSize;
|
||||
|
||||
/* check conditions */
|
||||
DEBUGLOG(4, "ZDICT_finalizeDictionary");
|
||||
if (dictBufferCapacity < dictContentSize) return ERROR(dstSize_tooSmall);
|
||||
if (dictContentSize < ZDICT_CONTENTSIZE_MIN) return ERROR(srcSize_wrong);
|
||||
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall);
|
||||
|
||||
/* dictionary header */
|
||||
@@ -968,43 +899,12 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
||||
hSize += eSize;
|
||||
}
|
||||
|
||||
/* Shrink the content size if it doesn't fit in the buffer */
|
||||
if (hSize + dictContentSize > dictBufferCapacity) {
|
||||
dictContentSize = dictBufferCapacity - hSize;
|
||||
}
|
||||
|
||||
/* Pad the dictionary content with zeros if it is too small */
|
||||
if (dictContentSize < minContentSize) {
|
||||
RETURN_ERROR_IF(hSize + minContentSize > dictBufferCapacity, dstSize_tooSmall,
|
||||
"dictBufferCapacity too small to fit max repcode");
|
||||
paddingSize = minContentSize - dictContentSize;
|
||||
} else {
|
||||
paddingSize = 0;
|
||||
}
|
||||
|
||||
{
|
||||
size_t const dictSize = hSize + paddingSize + dictContentSize;
|
||||
|
||||
/* The dictionary consists of the header, optional padding, and the content.
|
||||
* The padding comes before the content because the "best" position in the
|
||||
* dictionary is the last byte.
|
||||
*/
|
||||
BYTE* const outDictHeader = (BYTE*)dictBuffer;
|
||||
BYTE* const outDictPadding = outDictHeader + hSize;
|
||||
BYTE* const outDictContent = outDictPadding + paddingSize;
|
||||
|
||||
assert(dictSize <= dictBufferCapacity);
|
||||
assert(outDictContent + dictContentSize == (BYTE*)dictBuffer + dictSize);
|
||||
|
||||
/* First copy the customDictContent into its final location.
|
||||
* `customDictContent` and `dictBuffer` may overlap, so we must
|
||||
* do this before any other writes into the output buffer.
|
||||
* Then copy the header & padding into the output buffer.
|
||||
*/
|
||||
memmove(outDictContent, customDictContent, dictContentSize);
|
||||
memcpy(outDictHeader, header, hSize);
|
||||
memset(outDictPadding, 0, paddingSize);
|
||||
|
||||
/* copy elements in final buffer ; note : src and dst buffer can overlap */
|
||||
if (hSize + dictContentSize > dictBufferCapacity) dictContentSize = dictBufferCapacity - hSize;
|
||||
{ size_t const dictSize = hSize + dictContentSize;
|
||||
char* dictEnd = (char*)dictBuffer + dictSize;
|
||||
memmove(dictEnd - dictContentSize, customDictContent, dictContentSize);
|
||||
memcpy(dictBuffer, header, hSize);
|
||||
return dictSize;
|
||||
}
|
||||
}
|
||||
@@ -1015,7 +915,7 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t params)
|
||||
{
|
||||
int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
|
||||
int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
|
||||
U32 const notificationLevel = params.notificationLevel;
|
||||
size_t hSize = 8;
|
||||
|
||||
@@ -1044,11 +944,16 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
|
||||
return MIN(dictBufferCapacity, hSize+dictContentSize);
|
||||
}
|
||||
|
||||
/* Hidden declaration for dbio.c */
|
||||
size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
void* dictBuffer, size_t maxDictSize,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t params);
|
||||
/*! ZDICT_trainFromBuffer_unsafe_legacy() :
|
||||
* Warning : `samplesBuffer` must be followed by noisy guard band !!!
|
||||
* Warning : `samplesBuffer` must be followed by noisy guard band.
|
||||
* @return : size of dictionary, or an error code which can be tested with ZDICT_isError()
|
||||
*/
|
||||
static size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
void* dictBuffer, size_t maxDictSize,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t params)
|
||||
@@ -1185,8 +1090,8 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.d = 8;
|
||||
params.steps = 4;
|
||||
/* Use default level since no compression level information is available */
|
||||
params.zParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
|
||||
/* Default to level 6 since no compression level information is available */
|
||||
params.zParams.compressionLevel = 3;
|
||||
#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
|
||||
params.zParams.notificationLevel = DEBUGLEVEL;
|
||||
#endif
|
||||
@@ -1204,5 +1109,3 @@ size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize
|
||||
samplesBuffer, samplesSizes, nbSamples,
|
||||
params);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
235
vendor/github.com/DataDog/zstd/zdict.h
generated
vendored
235
vendor/github.com/DataDog/zstd/zdict.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -37,145 +36,6 @@ extern "C" {
|
||||
# define ZDICTLIB_API ZDICTLIB_VISIBILITY
|
||||
#endif
|
||||
|
||||
/*******************************************************************************
|
||||
* Zstd dictionary builder
|
||||
*
|
||||
* FAQ
|
||||
* ===
|
||||
* Why should I use a dictionary?
|
||||
* ------------------------------
|
||||
*
|
||||
* Zstd can use dictionaries to improve compression ratio of small data.
|
||||
* Traditionally small files don't compress well because there is very little
|
||||
* repetition in a single sample, since it is small. But, if you are compressing
|
||||
* many similar files, like a bunch of JSON records that share the same
|
||||
* structure, you can train a dictionary on ahead of time on some samples of
|
||||
* these files. Then, zstd can use the dictionary to find repetitions that are
|
||||
* present across samples. This can vastly improve compression ratio.
|
||||
*
|
||||
* When is a dictionary useful?
|
||||
* ----------------------------
|
||||
*
|
||||
* Dictionaries are useful when compressing many small files that are similar.
|
||||
* The larger a file is, the less benefit a dictionary will have. Generally,
|
||||
* we don't expect dictionary compression to be effective past 100KB. And the
|
||||
* smaller a file is, the more we would expect the dictionary to help.
|
||||
*
|
||||
* How do I use a dictionary?
|
||||
* --------------------------
|
||||
*
|
||||
* Simply pass the dictionary to the zstd compressor with
|
||||
* `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to
|
||||
* the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other
|
||||
* more advanced functions that allow selecting some options, see zstd.h for
|
||||
* complete documentation.
|
||||
*
|
||||
* What is a zstd dictionary?
|
||||
* --------------------------
|
||||
*
|
||||
* A zstd dictionary has two pieces: Its header, and its content. The header
|
||||
* contains a magic number, the dictionary ID, and entropy tables. These
|
||||
* entropy tables allow zstd to save on header costs in the compressed file,
|
||||
* which really matters for small data. The content is just bytes, which are
|
||||
* repeated content that is common across many samples.
|
||||
*
|
||||
* What is a raw content dictionary?
|
||||
* ---------------------------------
|
||||
*
|
||||
* A raw content dictionary is just bytes. It doesn't have a zstd dictionary
|
||||
* header, a dictionary ID, or entropy tables. Any buffer is a valid raw
|
||||
* content dictionary.
|
||||
*
|
||||
* How do I train a dictionary?
|
||||
* ----------------------------
|
||||
*
|
||||
* Gather samples from your use case. These samples should be similar to each
|
||||
* other. If you have several use cases, you could try to train one dictionary
|
||||
* per use case.
|
||||
*
|
||||
* Pass those samples to `ZDICT_trainFromBuffer()` and that will train your
|
||||
* dictionary. There are a few advanced versions of this function, but this
|
||||
* is a great starting point. If you want to further tune your dictionary
|
||||
* you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow
|
||||
* you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`.
|
||||
*
|
||||
* If the dictionary training function fails, that is likely because you
|
||||
* either passed too few samples, or a dictionary would not be effective
|
||||
* for your data. Look at the messages that the dictionary trainer printed,
|
||||
* if it doesn't say too few samples, then a dictionary would not be effective.
|
||||
*
|
||||
* How large should my dictionary be?
|
||||
* ----------------------------------
|
||||
*
|
||||
* A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB.
|
||||
* The zstd CLI defaults to a 110KB dictionary. You likely don't need a
|
||||
* dictionary larger than that. But, most use cases can get away with a
|
||||
* smaller dictionary. The advanced dictionary builders can automatically
|
||||
* shrink the dictionary for you, and select a the smallest size that
|
||||
* doesn't hurt compression ratio too much. See the `shrinkDict` parameter.
|
||||
* A smaller dictionary can save memory, and potentially speed up
|
||||
* compression.
|
||||
*
|
||||
* How many samples should I provide to the dictionary builder?
|
||||
* ------------------------------------------------------------
|
||||
*
|
||||
* We generally recommend passing ~100x the size of the dictionary
|
||||
* in samples. A few thousand should suffice. Having too few samples
|
||||
* can hurt the dictionaries effectiveness. Having more samples will
|
||||
* only improve the dictionaries effectiveness. But having too many
|
||||
* samples can slow down the dictionary builder.
|
||||
*
|
||||
* How do I determine if a dictionary will be effective?
|
||||
* -----------------------------------------------------
|
||||
*
|
||||
* Simply train a dictionary and try it out. You can use zstd's built in
|
||||
* benchmarking tool to test the dictionary effectiveness.
|
||||
*
|
||||
* # Benchmark levels 1-3 without a dictionary
|
||||
* zstd -b1e3 -r /path/to/my/files
|
||||
* # Benchmark levels 1-3 with a dictionary
|
||||
* zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary
|
||||
*
|
||||
* When should I retrain a dictionary?
|
||||
* -----------------------------------
|
||||
*
|
||||
* You should retrain a dictionary when its effectiveness drops. Dictionary
|
||||
* effectiveness drops as the data you are compressing changes. Generally, we do
|
||||
* expect dictionaries to "decay" over time, as your data changes, but the rate
|
||||
* at which they decay depends on your use case. Internally, we regularly
|
||||
* retrain dictionaries, and if the new dictionary performs significantly
|
||||
* better than the old dictionary, we will ship the new dictionary.
|
||||
*
|
||||
* I have a raw content dictionary, how do I turn it into a zstd dictionary?
|
||||
* -------------------------------------------------------------------------
|
||||
*
|
||||
* If you have a raw content dictionary, e.g. by manually constructing it, or
|
||||
* using a third-party dictionary builder, you can turn it into a zstd
|
||||
* dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to
|
||||
* provide some samples of the data. It will add the zstd header to the
|
||||
* raw content, which contains a dictionary ID and entropy tables, which
|
||||
* will improve compression ratio, and allow zstd to write the dictionary ID
|
||||
* into the frame, if you so choose.
|
||||
*
|
||||
* Do I have to use zstd's dictionary builder?
|
||||
* -------------------------------------------
|
||||
*
|
||||
* No! You can construct dictionary content however you please, it is just
|
||||
* bytes. It will always be valid as a raw content dictionary. If you want
|
||||
* a zstd dictionary, which can improve compression ratio, use
|
||||
* `ZDICT_finalizeDictionary()`.
|
||||
*
|
||||
* What is the attack surface of a zstd dictionary?
|
||||
* ------------------------------------------------
|
||||
*
|
||||
* Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so
|
||||
* zstd should never crash, or access out-of-bounds memory no matter what
|
||||
* the dictionary is. However, if an attacker can control the dictionary
|
||||
* during decompression, they can cause zstd to generate arbitrary bytes,
|
||||
* just like if they controlled the compressed data.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
|
||||
/*! ZDICT_trainFromBuffer():
|
||||
* Train a dictionary from an array of samples.
|
||||
@@ -201,63 +61,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
|
||||
const void* samplesBuffer,
|
||||
const size_t* samplesSizes, unsigned nbSamples);
|
||||
|
||||
typedef struct {
|
||||
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
|
||||
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value)
|
||||
* NOTE: The zstd format reserves some dictionary IDs for future use.
|
||||
* You may use them in private settings, but be warned that they
|
||||
* may be used by zstd in a public dictionary registry in the future.
|
||||
* These dictionary IDs are:
|
||||
* - low range : <= 32767
|
||||
* - high range : >= (2^31)
|
||||
*/
|
||||
} ZDICT_params_t;
|
||||
|
||||
/*! ZDICT_finalizeDictionary():
|
||||
* Given a custom content as a basis for dictionary, and a set of samples,
|
||||
* finalize dictionary by adding headers and statistics according to the zstd
|
||||
* dictionary format.
|
||||
*
|
||||
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
||||
* supplied with an array of sizes `samplesSizes`, providing the size of each
|
||||
* sample in order. The samples are used to construct the statistics, so they
|
||||
* should be representative of what you will compress with this dictionary.
|
||||
*
|
||||
* The compression level can be set in `parameters`. You should pass the
|
||||
* compression level you expect to use in production. The statistics for each
|
||||
* compression level differ, so tuning the dictionary for the compression level
|
||||
* can help quite a bit.
|
||||
*
|
||||
* You can set an explicit dictionary ID in `parameters`, or allow us to pick
|
||||
* a random dictionary ID for you, but we can't guarantee no collisions.
|
||||
*
|
||||
* The dstDictBuffer and the dictContent may overlap, and the content will be
|
||||
* appended to the end of the header. If the header + the content doesn't fit in
|
||||
* maxDictSize the beginning of the content is truncated to make room, since it
|
||||
* is presumed that the most profitable content is at the end of the dictionary,
|
||||
* since that is the cheapest to reference.
|
||||
*
|
||||
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
|
||||
*
|
||||
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
|
||||
* or an error code, which can be tested by ZDICT_isError().
|
||||
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if
|
||||
* instructed to, using notificationLevel>0.
|
||||
* NOTE: This function currently may fail in several edge cases including:
|
||||
* * Not enough samples
|
||||
* * Samples are uncompressible
|
||||
* * Samples are all exactly the same
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
|
||||
const void* dictContent, size_t dictContentSize,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
|
||||
/*====== Helper functions ======*/
|
||||
ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
|
||||
ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
|
||||
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
|
||||
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
||||
|
||||
@@ -272,9 +78,11 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
#define ZDICT_DICTSIZE_MIN 256
|
||||
/* Deprecated: Remove in v1.6.0 */
|
||||
#define ZDICT_CONTENTSIZE_MIN 128
|
||||
typedef struct {
|
||||
int compressionLevel; /* optimize for a specific zstd compression level; 0 means default */
|
||||
unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||
unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
|
||||
} ZDICT_params_t;
|
||||
|
||||
/*! ZDICT_cover_params_t:
|
||||
* k and d are the only required parameters.
|
||||
@@ -390,6 +198,28 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
||||
const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_fastCover_params_t* parameters);
|
||||
|
||||
/*! ZDICT_finalizeDictionary():
|
||||
* Given a custom content as a basis for dictionary, and a set of samples,
|
||||
* finalize dictionary by adding headers and statistics.
|
||||
*
|
||||
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
||||
* supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
|
||||
*
|
||||
* dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
||||
* maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
|
||||
*
|
||||
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
|
||||
* or an error code, which can be tested by ZDICT_isError().
|
||||
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
|
||||
* Note 2: dictBuffer and dictContent can overlap
|
||||
*/
|
||||
#define ZDICT_CONTENTSIZE_MIN 128
|
||||
#define ZDICT_DICTSIZE_MIN 256
|
||||
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* dictContent, size_t dictContentSize,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
typedef struct {
|
||||
unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
|
||||
ZDICT_params_t zParams;
|
||||
@@ -411,11 +241,10 @@ typedef struct {
|
||||
* Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
||||
void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
void *dictBuffer, size_t dictBufferCapacity,
|
||||
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t parameters);
|
||||
|
||||
|
||||
/* Deprecation warnings */
|
||||
/* It is generally possible to disable deprecation warnings from compiler,
|
||||
for example with -Wno-deprecated-declarations for gcc
|
||||
@@ -427,7 +256,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
||||
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
|
||||
# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
|
||||
# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
|
||||
# elif (ZDICT_GCC_VERSION >= 301)
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
|
||||
@@ -451,5 +280,3 @@ size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize
|
||||
#endif
|
||||
|
||||
#endif /* DICTBUILDER_H_001 */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
133
vendor/github.com/DataDog/zstd/zstd.go
generated
vendored
133
vendor/github.com/DataDog/zstd/zstd.go
generated
vendored
@@ -1,18 +1,29 @@
|
||||
package zstd
|
||||
|
||||
/*
|
||||
// support decoding of "legacy" zstd payloads from versions [0.4, 0.8], matching the
|
||||
// default configuration of the zstd command line tool:
|
||||
// https://github.com/facebook/zstd/blob/dev/programs/README.md
|
||||
#cgo CFLAGS: -DZSTD_LEGACY_SUPPORT=4
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "stdint.h" // for uintptr_t
|
||||
|
||||
// The following *_wrapper function are used for removing superflouos
|
||||
// memory allocations when calling the wrapped functions from Go code.
|
||||
// See https://github.com/golang/go/issues/24450 for details.
|
||||
|
||||
static size_t ZSTD_compress_wrapper(uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize, int compressionLevel) {
|
||||
return ZSTD_compress((void*)dst, maxDstSize, (const void*)src, srcSize, compressionLevel);
|
||||
}
|
||||
|
||||
static size_t ZSTD_decompress_wrapper(uintptr_t dst, size_t maxDstSize, uintptr_t src, size_t srcSize) {
|
||||
return ZSTD_decompress((void*)dst, maxDstSize, (const void *)src, srcSize);
|
||||
}
|
||||
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"runtime"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
@@ -28,17 +39,6 @@ var (
|
||||
ErrEmptySlice = errors.New("Bytes slice is empty")
|
||||
)
|
||||
|
||||
const (
|
||||
// decompressSizeBufferLimit is the limit we set on creating a decompression buffer for the Decompress API
|
||||
// This is made to prevent DOS from maliciously-created payloads (aka zipbomb).
|
||||
// For large payloads with a compression ratio > 10, you can do your own allocation and pass it to the method:
|
||||
// dst := make([]byte, 1GB)
|
||||
// decompressed, err := zstd.Decompress(dst, src)
|
||||
decompressSizeBufferLimit = 1000 * 1000
|
||||
|
||||
zstdFrameHeaderSizeMax = 18 // From zstd.h. Since it's experimental API, hardcoding it
|
||||
)
|
||||
|
||||
// CompressBound returns the worst case size needed for a destination buffer,
|
||||
// which can be used to preallocate a destination buffer or select a previously
|
||||
// allocated buffer from a pool.
|
||||
@@ -57,30 +57,6 @@ func cCompressBound(srcSize int) int {
|
||||
return int(C.ZSTD_compressBound(C.size_t(srcSize)))
|
||||
}
|
||||
|
||||
// decompressSizeHint tries to give a hint on how much of the output buffer size we should have
|
||||
// based on zstd frame descriptors. To prevent DOS from maliciously-created payloads, limit the size
|
||||
func decompressSizeHint(src []byte) int {
|
||||
// 1 MB or 10x input size
|
||||
upperBound := 10 * len(src)
|
||||
if upperBound < decompressSizeBufferLimit {
|
||||
upperBound = decompressSizeBufferLimit
|
||||
}
|
||||
|
||||
hint := upperBound
|
||||
if len(src) >= zstdFrameHeaderSizeMax {
|
||||
hint = int(C.ZSTD_getFrameContentSize(unsafe.Pointer(&src[0]), C.size_t(len(src))))
|
||||
if hint < 0 { // On error, just use upperBound
|
||||
hint = upperBound
|
||||
}
|
||||
}
|
||||
|
||||
// Take the minimum of both
|
||||
if hint > upperBound {
|
||||
return upperBound
|
||||
}
|
||||
return hint
|
||||
}
|
||||
|
||||
// Compress src into dst. If you have a buffer to use, you can pass it to
|
||||
// prevent allocation. If it is too small, or if nil is passed, a new buffer
|
||||
// will be allocated and returned.
|
||||
@@ -97,26 +73,19 @@ func CompressLevel(dst, src []byte, level int) ([]byte, error) {
|
||||
dst = make([]byte, bound)
|
||||
}
|
||||
|
||||
// We need unsafe.Pointer(&src[0]) in the Cgo call to avoid "Go pointer to Go pointer" panics.
|
||||
// This means we need to special case empty input. See:
|
||||
// https://github.com/golang/go/issues/14210#issuecomment-346402945
|
||||
var cWritten C.size_t
|
||||
if len(src) == 0 {
|
||||
cWritten = C.ZSTD_compress(
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(nil),
|
||||
C.size_t(0),
|
||||
C.int(level))
|
||||
} else {
|
||||
cWritten = C.ZSTD_compress(
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src)),
|
||||
C.int(level))
|
||||
srcPtr := C.uintptr_t(uintptr(0)) // Do not point anywhere, if src is empty
|
||||
if len(src) > 0 {
|
||||
srcPtr = C.uintptr_t(uintptr(unsafe.Pointer(&src[0])))
|
||||
}
|
||||
|
||||
cWritten := C.ZSTD_compress_wrapper(
|
||||
C.uintptr_t(uintptr(unsafe.Pointer(&dst[0]))),
|
||||
C.size_t(len(dst)),
|
||||
srcPtr,
|
||||
C.size_t(len(src)),
|
||||
C.int(level))
|
||||
|
||||
runtime.KeepAlive(src)
|
||||
written := int(cWritten)
|
||||
// Check if the return is an Error code
|
||||
if err := getError(written); err != nil {
|
||||
@@ -132,25 +101,43 @@ func Decompress(dst, src []byte) ([]byte, error) {
|
||||
if len(src) == 0 {
|
||||
return []byte{}, ErrEmptySlice
|
||||
}
|
||||
decompress := func(dst, src []byte) ([]byte, error) {
|
||||
|
||||
bound := decompressSizeHint(src)
|
||||
if cap(dst) >= bound {
|
||||
dst = dst[0:cap(dst)]
|
||||
} else {
|
||||
dst = make([]byte, bound)
|
||||
}
|
||||
cWritten := C.ZSTD_decompress_wrapper(
|
||||
C.uintptr_t(uintptr(unsafe.Pointer(&dst[0]))),
|
||||
C.size_t(len(dst)),
|
||||
C.uintptr_t(uintptr(unsafe.Pointer(&src[0]))),
|
||||
C.size_t(len(src)))
|
||||
|
||||
written := int(C.ZSTD_decompress(
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src))))
|
||||
err := getError(written)
|
||||
if err == nil {
|
||||
runtime.KeepAlive(src)
|
||||
written := int(cWritten)
|
||||
// Check error
|
||||
if err := getError(written); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dst[:written], nil
|
||||
}
|
||||
if !IsDstSizeTooSmallError(err) {
|
||||
return nil, err
|
||||
|
||||
if len(dst) == 0 {
|
||||
// Attempt to use zStd to determine decompressed size (may result in error or 0)
|
||||
size := int(C.size_t(C.ZSTD_getDecompressedSize(unsafe.Pointer(&src[0]), C.size_t(len(src)))))
|
||||
|
||||
if err := getError(size); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if size > 0 {
|
||||
dst = make([]byte, size)
|
||||
} else {
|
||||
dst = make([]byte, len(src)*3) // starting guess
|
||||
}
|
||||
}
|
||||
for i := 0; i < 3; i++ { // 3 tries to allocate a bigger buffer
|
||||
result, err := decompress(dst, src)
|
||||
if !IsDstSizeTooSmallError(err) {
|
||||
return result, err
|
||||
}
|
||||
dst = make([]byte, len(dst)*2) // Grow buffer by 2
|
||||
}
|
||||
|
||||
// We failed getting a dst buffer of correct size, use stream API
|
||||
|
||||
991
vendor/github.com/DataDog/zstd/zstd.h
generated
vendored
991
vendor/github.com/DataDog/zstd/zstd.h
generated
vendored
File diff suppressed because it is too large
Load Diff
151
vendor/github.com/DataDog/zstd/zstd_bulk.go
generated
vendored
151
vendor/github.com/DataDog/zstd/zstd_bulk.go
generated
vendored
@@ -1,151 +0,0 @@
|
||||
package zstd
|
||||
|
||||
/*
|
||||
#include "zstd.h"
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"errors"
|
||||
"runtime"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrEmptyDictionary is returned when the given dictionary is empty
|
||||
ErrEmptyDictionary = errors.New("Dictionary is empty")
|
||||
// ErrBadDictionary is returned when cannot load the given dictionary
|
||||
ErrBadDictionary = errors.New("Cannot load dictionary")
|
||||
)
|
||||
|
||||
// BulkProcessor implements Bulk processing dictionary API.
|
||||
// When compressing multiple messages or blocks using the same dictionary,
|
||||
// it's recommended to digest the dictionary only once, since it's a costly operation.
|
||||
// NewBulkProcessor() will create a state from digesting a dictionary.
|
||||
// The resulting state can be used for future compression/decompression operations with very limited startup cost.
|
||||
// BulkProcessor can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
||||
// The state will be freed when gc cleans up BulkProcessor.
|
||||
type BulkProcessor struct {
|
||||
cDict *C.struct_ZSTD_CDict_s
|
||||
dDict *C.struct_ZSTD_DDict_s
|
||||
}
|
||||
|
||||
// NewBulkProcessor creates a new BulkProcessor with a pre-trained dictionary and compression level
|
||||
func NewBulkProcessor(dictionary []byte, compressionLevel int) (*BulkProcessor, error) {
|
||||
if len(dictionary) < 1 {
|
||||
return nil, ErrEmptyDictionary
|
||||
}
|
||||
|
||||
p := &BulkProcessor{}
|
||||
runtime.SetFinalizer(p, finalizeBulkProcessor)
|
||||
|
||||
p.cDict = C.ZSTD_createCDict(
|
||||
unsafe.Pointer(&dictionary[0]),
|
||||
C.size_t(len(dictionary)),
|
||||
C.int(compressionLevel),
|
||||
)
|
||||
if p.cDict == nil {
|
||||
return nil, ErrBadDictionary
|
||||
}
|
||||
p.dDict = C.ZSTD_createDDict(
|
||||
unsafe.Pointer(&dictionary[0]),
|
||||
C.size_t(len(dictionary)),
|
||||
)
|
||||
if p.dDict == nil {
|
||||
return nil, ErrBadDictionary
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Compress compresses `src` into `dst` with the dictionary given when creating the BulkProcessor.
|
||||
// If you have a buffer to use, you can pass it to prevent allocation.
|
||||
// If it is too small, or if nil is passed, a new buffer will be allocated and returned.
|
||||
func (p *BulkProcessor) Compress(dst, src []byte) ([]byte, error) {
|
||||
bound := CompressBound(len(src))
|
||||
if cap(dst) >= bound {
|
||||
dst = dst[0:bound]
|
||||
} else {
|
||||
dst = make([]byte, bound)
|
||||
}
|
||||
|
||||
cctx := C.ZSTD_createCCtx()
|
||||
// We need unsafe.Pointer(&src[0]) in the Cgo call to avoid "Go pointer to Go pointer" panics.
|
||||
// This means we need to special case empty input. See:
|
||||
// https://github.com/golang/go/issues/14210#issuecomment-346402945
|
||||
var cWritten C.size_t
|
||||
if len(src) == 0 {
|
||||
cWritten = C.ZSTD_compress_usingCDict(
|
||||
cctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(nil),
|
||||
C.size_t(len(src)),
|
||||
p.cDict,
|
||||
)
|
||||
} else {
|
||||
cWritten = C.ZSTD_compress_usingCDict(
|
||||
cctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src)),
|
||||
p.cDict,
|
||||
)
|
||||
}
|
||||
|
||||
C.ZSTD_freeCCtx(cctx)
|
||||
|
||||
written := int(cWritten)
|
||||
if err := getError(written); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dst[:written], nil
|
||||
}
|
||||
|
||||
// Decompress decompresses `src` into `dst` with the dictionary given when creating the BulkProcessor.
|
||||
// If you have a buffer to use, you can pass it to prevent allocation.
|
||||
// If it is too small, or if nil is passed, a new buffer will be allocated and returned.
|
||||
func (p *BulkProcessor) Decompress(dst, src []byte) ([]byte, error) {
|
||||
if len(src) == 0 {
|
||||
return nil, ErrEmptySlice
|
||||
}
|
||||
|
||||
contentSize := decompressSizeHint(src)
|
||||
if cap(dst) >= contentSize {
|
||||
dst = dst[0:contentSize]
|
||||
} else {
|
||||
dst = make([]byte, contentSize)
|
||||
}
|
||||
|
||||
if contentSize == 0 {
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
dctx := C.ZSTD_createDCtx()
|
||||
cWritten := C.ZSTD_decompress_usingDDict(
|
||||
dctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(contentSize),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src)),
|
||||
p.dDict,
|
||||
)
|
||||
C.ZSTD_freeDCtx(dctx)
|
||||
|
||||
written := int(cWritten)
|
||||
if err := getError(written); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return dst[:written], nil
|
||||
}
|
||||
|
||||
// finalizeBulkProcessor frees compression and decompression dictionaries from memory
|
||||
func finalizeBulkProcessor(p *BulkProcessor) {
|
||||
if p.cDict != nil {
|
||||
C.ZSTD_freeCDict(p.cDict)
|
||||
}
|
||||
if p.dDict != nil {
|
||||
C.ZSTD_freeDDict(p.dDict)
|
||||
}
|
||||
}
|
||||
23
vendor/github.com/DataDog/zstd/zstd_common.c
generated
vendored
23
vendor/github.com/DataDog/zstd/zstd_common.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -14,8 +13,8 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
|
||||
#include <stdlib.h> /* malloc, calloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h"
|
||||
|
||||
@@ -54,33 +53,31 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString
|
||||
/*=**************************************************************
|
||||
* Custom allocator
|
||||
****************************************************************/
|
||||
void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
|
||||
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc)
|
||||
return customMem.customAlloc(customMem.opaque, size);
|
||||
return ZSTD_malloc(size);
|
||||
return malloc(size);
|
||||
}
|
||||
|
||||
void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
|
||||
void* ZSTD_calloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc) {
|
||||
/* calloc implemented as malloc+memset;
|
||||
* not as efficient as calloc, but next best guess for custom malloc */
|
||||
void* const ptr = customMem.customAlloc(customMem.opaque, size);
|
||||
ZSTD_memset(ptr, 0, size);
|
||||
memset(ptr, 0, size);
|
||||
return ptr;
|
||||
}
|
||||
return ZSTD_calloc(1, size);
|
||||
return calloc(1, size);
|
||||
}
|
||||
|
||||
void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
|
||||
void ZSTD_free(void* ptr, ZSTD_customMem customMem)
|
||||
{
|
||||
if (ptr!=NULL) {
|
||||
if (customMem.customFree)
|
||||
customMem.customFree(customMem.opaque, ptr);
|
||||
else
|
||||
ZSTD_free(ptr);
|
||||
free(ptr);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
4313
vendor/github.com/DataDog/zstd/zstd_compress.c
generated
vendored
4313
vendor/github.com/DataDog/zstd/zstd_compress.c
generated
vendored
File diff suppressed because it is too large
Load Diff
618
vendor/github.com/DataDog/zstd/zstd_compress_internal.h
generated
vendored
618
vendor/github.com/DataDog/zstd/zstd_compress_internal.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -29,6 +28,7 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
@@ -64,7 +64,7 @@ typedef struct {
|
||||
} ZSTD_localDict;
|
||||
|
||||
typedef struct {
|
||||
HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)];
|
||||
U32 CTable[HUF_CTABLE_SIZE_U32(255)];
|
||||
HUF_repeat repeatMode;
|
||||
} ZSTD_hufCTables_t;
|
||||
|
||||
@@ -82,75 +82,11 @@ typedef struct {
|
||||
ZSTD_fseCTables_t fse;
|
||||
} ZSTD_entropyCTables_t;
|
||||
|
||||
/***********************************************
|
||||
* Entropy buffer statistics structs and funcs *
|
||||
***********************************************/
|
||||
/** ZSTD_hufCTablesMetadata_t :
|
||||
* Stores Literals Block Type for a super-block in hType, and
|
||||
* huffman tree description in hufDesBuffer.
|
||||
* hufDesSize refers to the size of huffman tree description in bytes.
|
||||
* This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
|
||||
typedef struct {
|
||||
symbolEncodingType_e hType;
|
||||
BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
|
||||
size_t hufDesSize;
|
||||
} ZSTD_hufCTablesMetadata_t;
|
||||
|
||||
/** ZSTD_fseCTablesMetadata_t :
|
||||
* Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
|
||||
* fse tables in fseTablesBuffer.
|
||||
* fseTablesSize refers to the size of fse tables in bytes.
|
||||
* This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
|
||||
typedef struct {
|
||||
symbolEncodingType_e llType;
|
||||
symbolEncodingType_e ofType;
|
||||
symbolEncodingType_e mlType;
|
||||
BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
|
||||
size_t fseTablesSize;
|
||||
size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
|
||||
} ZSTD_fseCTablesMetadata_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_hufCTablesMetadata_t hufMetadata;
|
||||
ZSTD_fseCTablesMetadata_t fseMetadata;
|
||||
} ZSTD_entropyCTablesMetadata_t;
|
||||
|
||||
/** ZSTD_buildBlockEntropyStats() :
|
||||
* Builds entropy for the block.
|
||||
* @return : 0 on success or error code */
|
||||
size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
|
||||
const ZSTD_entropyCTables_t* prevEntropy,
|
||||
ZSTD_entropyCTables_t* nextEntropy,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize);
|
||||
|
||||
/*********************************
|
||||
* Compression internals structs *
|
||||
*********************************/
|
||||
|
||||
typedef struct {
|
||||
U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */
|
||||
U32 len; /* Raw length of match */
|
||||
U32 off;
|
||||
U32 len;
|
||||
} ZSTD_match_t;
|
||||
|
||||
typedef struct {
|
||||
U32 offset; /* Offset of sequence */
|
||||
U32 litLength; /* Length of literals prior to match */
|
||||
U32 matchLength; /* Raw length of match */
|
||||
} rawSeq;
|
||||
|
||||
typedef struct {
|
||||
rawSeq* seq; /* The start of the sequences */
|
||||
size_t pos; /* The index in seq where reading stopped. pos <= size. */
|
||||
size_t posInSequence; /* The position within the sequence at seq[pos] where reading
|
||||
stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */
|
||||
size_t size; /* The number of sequences. <= capacity. */
|
||||
size_t capacity; /* The capacity starting from `seq` pointer */
|
||||
} rawSeqStore_t;
|
||||
|
||||
UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
|
||||
|
||||
typedef struct {
|
||||
int price;
|
||||
U32 off;
|
||||
@@ -180,7 +116,7 @@ typedef struct {
|
||||
U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
|
||||
ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
|
||||
const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
|
||||
ZSTD_paramSwitch_e literalCompressionMode;
|
||||
ZSTD_literalCompressionMode_e literalCompressionMode;
|
||||
} optState_t;
|
||||
|
||||
typedef struct {
|
||||
@@ -189,23 +125,14 @@ typedef struct {
|
||||
} ZSTD_compressedBlockState_t;
|
||||
|
||||
typedef struct {
|
||||
BYTE const* nextSrc; /* next block here to continue on current prefix */
|
||||
BYTE const* base; /* All regular indexes relative to this position */
|
||||
BYTE const* dictBase; /* extDict indexes relative to this position */
|
||||
U32 dictLimit; /* below that point, need extDict */
|
||||
U32 lowLimit; /* below that point, no more valid data */
|
||||
U32 nbOverflowCorrections; /* Number of times overflow correction has run since
|
||||
* ZSTD_window_init(). Useful for debugging coredumps
|
||||
* and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY.
|
||||
*/
|
||||
BYTE const* nextSrc; /* next block here to continue on current prefix */
|
||||
BYTE const* base; /* All regular indexes relative to this position */
|
||||
BYTE const* dictBase; /* extDict indexes relative to this position */
|
||||
U32 dictLimit; /* below that point, need extDict */
|
||||
U32 lowLimit; /* below that point, no more valid data */
|
||||
} ZSTD_window_t;
|
||||
|
||||
#define ZSTD_WINDOW_START_INDEX 2
|
||||
|
||||
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
|
||||
|
||||
#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
|
||||
|
||||
struct ZSTD_matchState_t {
|
||||
ZSTD_window_t window; /* State for window round buffer management */
|
||||
U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
|
||||
@@ -217,24 +144,12 @@ struct ZSTD_matchState_t {
|
||||
*/
|
||||
U32 nextToUpdate; /* index from which to continue table update */
|
||||
U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
|
||||
|
||||
U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
|
||||
U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
|
||||
U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
|
||||
|
||||
U32* hashTable;
|
||||
U32* hashTable3;
|
||||
U32* chainTable;
|
||||
|
||||
U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
|
||||
|
||||
int dedicatedDictSearch; /* Indicates whether this matchState is using the
|
||||
* dedicated dictionary search structure.
|
||||
*/
|
||||
optState_t opt; /* optimal parser state */
|
||||
const ZSTD_matchState_t* dictMatchState;
|
||||
ZSTD_compressionParameters cParams;
|
||||
const rawSeqStore_t* ldmSeqStore;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
@@ -248,26 +163,16 @@ typedef struct {
|
||||
U32 checksum;
|
||||
} ldmEntry_t;
|
||||
|
||||
typedef struct {
|
||||
BYTE const* split;
|
||||
U32 hash;
|
||||
U32 checksum;
|
||||
ldmEntry_t* bucket;
|
||||
} ldmMatchCandidate_t;
|
||||
|
||||
#define LDM_BATCH_SIZE 64
|
||||
|
||||
typedef struct {
|
||||
ZSTD_window_t window; /* State for the window round buffer management */
|
||||
ldmEntry_t* hashTable;
|
||||
U32 loadedDictEnd;
|
||||
BYTE* bucketOffsets; /* Next position in bucket to insert entry */
|
||||
size_t splitIndices[LDM_BATCH_SIZE];
|
||||
ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE];
|
||||
U64 hashPower; /* Used to compute the rolling hash.
|
||||
* Depends on ldmParams.minMatchLength */
|
||||
} ldmState_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
|
||||
U32 enableLdm; /* 1 if enable long distance matching */
|
||||
U32 hashLog; /* Log size of hashTable */
|
||||
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
|
||||
U32 minMatchLength; /* Minimum match length */
|
||||
@@ -275,6 +180,19 @@ typedef struct {
|
||||
U32 windowLog; /* Window log for the LDM */
|
||||
} ldmParams_t;
|
||||
|
||||
typedef struct {
|
||||
U32 offset;
|
||||
U32 litLength;
|
||||
U32 matchLength;
|
||||
} rawSeq;
|
||||
|
||||
typedef struct {
|
||||
rawSeq* seq; /* The start of the sequences */
|
||||
size_t pos; /* The position where reading stopped. <= size. */
|
||||
size_t size; /* The number of sequences. <= capacity. */
|
||||
size_t capacity; /* The capacity starting from `seq` pointer */
|
||||
} rawSeqStore_t;
|
||||
|
||||
typedef struct {
|
||||
int collectSequences;
|
||||
ZSTD_Sequence* seqStart;
|
||||
@@ -298,7 +216,7 @@ struct ZSTD_CCtx_params_s {
|
||||
* There is no guarantee that hint is close to actual source size */
|
||||
|
||||
ZSTD_dictAttachPref_e attachDictPref;
|
||||
ZSTD_paramSwitch_e literalCompressionMode;
|
||||
ZSTD_literalCompressionMode_e literalCompressionMode;
|
||||
|
||||
/* Multithreading: used to pass parameters to mtctx */
|
||||
int nbWorkers;
|
||||
@@ -309,68 +227,17 @@ struct ZSTD_CCtx_params_s {
|
||||
/* Long distance matching parameters */
|
||||
ldmParams_t ldmParams;
|
||||
|
||||
/* Dedicated dict search algorithm trigger */
|
||||
int enableDedicatedDictSearch;
|
||||
|
||||
/* Input/output buffer modes */
|
||||
ZSTD_bufferMode_e inBufferMode;
|
||||
ZSTD_bufferMode_e outBufferMode;
|
||||
|
||||
/* Sequence compression API */
|
||||
ZSTD_sequenceFormat_e blockDelimiters;
|
||||
int validateSequences;
|
||||
|
||||
/* Block splitting */
|
||||
ZSTD_paramSwitch_e useBlockSplitter;
|
||||
|
||||
/* Param for deciding whether to use row-based matchfinder */
|
||||
ZSTD_paramSwitch_e useRowMatchFinder;
|
||||
|
||||
/* Always load a dictionary in ext-dict mode (not prefix mode)? */
|
||||
int deterministicRefPrefix;
|
||||
|
||||
/* Internal use, for createCCtxParams() and freeCCtxParams() only */
|
||||
ZSTD_customMem customMem;
|
||||
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
|
||||
|
||||
#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
|
||||
#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
|
||||
|
||||
/**
|
||||
* Indicates whether this compression proceeds directly from user-provided
|
||||
* source buffer to user-provided destination buffer (ZSTDb_not_buffered), or
|
||||
* whether the context needs to buffer the input/output (ZSTDb_buffered).
|
||||
*/
|
||||
typedef enum {
|
||||
ZSTDb_not_buffered,
|
||||
ZSTDb_buffered
|
||||
} ZSTD_buffered_policy_e;
|
||||
|
||||
/**
|
||||
* Struct that contains all elements of block splitter that should be allocated
|
||||
* in a wksp.
|
||||
*/
|
||||
#define ZSTD_MAX_NB_BLOCK_SPLITS 196
|
||||
typedef struct {
|
||||
seqStore_t fullSeqStoreChunk;
|
||||
seqStore_t firstHalfSeqStore;
|
||||
seqStore_t secondHalfSeqStore;
|
||||
seqStore_t currSeqStore;
|
||||
seqStore_t nextSeqStore;
|
||||
|
||||
U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
|
||||
ZSTD_entropyCTablesMetadata_t entropyMetadata;
|
||||
} ZSTD_blockSplitCtx;
|
||||
|
||||
struct ZSTD_CCtx_s {
|
||||
ZSTD_compressionStage_e stage;
|
||||
int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
|
||||
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
|
||||
ZSTD_CCtx_params requestedParams;
|
||||
ZSTD_CCtx_params appliedParams;
|
||||
ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */
|
||||
U32 dictID;
|
||||
size_t dictContentSize;
|
||||
|
||||
ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
|
||||
size_t blockSize;
|
||||
@@ -379,11 +246,9 @@ struct ZSTD_CCtx_s {
|
||||
unsigned long long producedCSize;
|
||||
XXH64_state_t xxhState;
|
||||
ZSTD_customMem customMem;
|
||||
ZSTD_threadPool* pool;
|
||||
size_t staticSize;
|
||||
SeqCollector seqCollector;
|
||||
int isFirstBlock;
|
||||
int initialized;
|
||||
|
||||
seqStore_t seqStore; /* sequences storage ptrs */
|
||||
ldmState_t ldmState; /* long distance matching state */
|
||||
@@ -391,10 +256,7 @@ struct ZSTD_CCtx_s {
|
||||
size_t maxNbLdmSequences;
|
||||
rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
|
||||
ZSTD_blockState_t blockState;
|
||||
U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
|
||||
|
||||
/* Whether we are streaming or not */
|
||||
ZSTD_buffered_policy_e bufferedPolicy;
|
||||
U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
|
||||
|
||||
/* streaming */
|
||||
char* inBuff;
|
||||
@@ -409,10 +271,6 @@ struct ZSTD_CCtx_s {
|
||||
ZSTD_cStreamStage streamStage;
|
||||
U32 frameEnded;
|
||||
|
||||
/* Stable in/out buffer verification */
|
||||
ZSTD_inBuffer expectedInBuffer;
|
||||
size_t expectedOutBufferSize;
|
||||
|
||||
/* Dictionary */
|
||||
ZSTD_localDict localDict;
|
||||
const ZSTD_CDict* cdict;
|
||||
@@ -422,49 +280,17 @@ struct ZSTD_CCtx_s {
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
ZSTDMT_CCtx* mtctx;
|
||||
#endif
|
||||
|
||||
/* Tracing */
|
||||
#if ZSTD_TRACE
|
||||
ZSTD_TraceCtx traceCtx;
|
||||
#endif
|
||||
|
||||
/* Workspace for block splitter */
|
||||
ZSTD_blockSplitCtx blockSplitCtx;
|
||||
};
|
||||
|
||||
typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
|
||||
|
||||
typedef enum {
|
||||
ZSTD_noDict = 0,
|
||||
ZSTD_extDict = 1,
|
||||
ZSTD_dictMatchState = 2,
|
||||
ZSTD_dedicatedDictSearch = 3
|
||||
} ZSTD_dictMode_e;
|
||||
typedef enum { ZSTD_noDict = 0, ZSTD_extDict = 1, ZSTD_dictMatchState = 2 } ZSTD_dictMode_e;
|
||||
|
||||
typedef enum {
|
||||
ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict.
|
||||
* In this mode we use both the srcSize and the dictSize
|
||||
* when selecting and adjusting parameters.
|
||||
*/
|
||||
ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch.
|
||||
* In this mode we only take the srcSize into account when selecting
|
||||
* and adjusting parameters.
|
||||
*/
|
||||
ZSTD_cpm_createCDict = 2, /* Creating a CDict.
|
||||
* In this mode we take both the source size and the dictionary size
|
||||
* into account when selecting and adjusting the parameters.
|
||||
*/
|
||||
ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
|
||||
* We don't know what these parameters are for. We default to the legacy
|
||||
* behavior of taking both the source size and the dict size into account
|
||||
* when selecting and adjusting parameters.
|
||||
*/
|
||||
} ZSTD_cParamMode_e;
|
||||
|
||||
typedef size_t (*ZSTD_blockCompressor) (
|
||||
ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
|
||||
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode);
|
||||
|
||||
|
||||
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
|
||||
@@ -510,30 +336,6 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ZSTD_noCompressBlock() :
|
||||
* Writes uncompressed block to dst buffer from given src.
|
||||
* Returns the size of the block */
|
||||
MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
|
||||
{
|
||||
U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
|
||||
RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
|
||||
dstSize_tooSmall, "dst buf too small for uncompressed block");
|
||||
MEM_writeLE24(dst, cBlockHeader24);
|
||||
ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
|
||||
return ZSTD_blockHeaderSize + srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
|
||||
{
|
||||
BYTE* const op = (BYTE*)dst;
|
||||
U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);
|
||||
RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, "");
|
||||
MEM_writeLE24(op, cBlockHeader);
|
||||
op[3] = src;
|
||||
return 4;
|
||||
}
|
||||
|
||||
|
||||
/* ZSTD_minGain() :
|
||||
* minimum compression required
|
||||
* to generate a compress block or a compressed literals section.
|
||||
@@ -546,29 +348,12 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
|
||||
return (srcSize >> minlog) + 2;
|
||||
}
|
||||
|
||||
MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams)
|
||||
{
|
||||
switch (cctxParams->literalCompressionMode) {
|
||||
case ZSTD_ps_enable:
|
||||
return 0;
|
||||
case ZSTD_ps_disable:
|
||||
return 1;
|
||||
default:
|
||||
assert(0 /* impossible: pre-validated */);
|
||||
ZSTD_FALLTHROUGH;
|
||||
case ZSTD_ps_auto:
|
||||
return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTD_safecopyLiterals() :
|
||||
* memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
|
||||
* Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
|
||||
* large copies.
|
||||
*/
|
||||
static void
|
||||
ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w)
|
||||
{
|
||||
static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) {
|
||||
assert(iend > ilimit_w);
|
||||
if (ip <= ilimit_w) {
|
||||
ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap);
|
||||
@@ -578,30 +363,14 @@ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE con
|
||||
while (ip < iend) *op++ = *ip++;
|
||||
}
|
||||
|
||||
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
|
||||
#define STORE_REPCODE_1 STORE_REPCODE(1)
|
||||
#define STORE_REPCODE_2 STORE_REPCODE(2)
|
||||
#define STORE_REPCODE_3 STORE_REPCODE(3)
|
||||
#define STORE_REPCODE(r) (assert((r)>=1), assert((r)<=3), (r)-1)
|
||||
#define STORE_OFFSET(o) (assert((o)>0), o + ZSTD_REP_MOVE)
|
||||
#define STORED_IS_OFFSET(o) ((o) > ZSTD_REP_MOVE)
|
||||
#define STORED_IS_REPCODE(o) ((o) <= ZSTD_REP_MOVE)
|
||||
#define STORED_OFFSET(o) (assert(STORED_IS_OFFSET(o)), (o)-ZSTD_REP_MOVE)
|
||||
#define STORED_REPCODE(o) (assert(STORED_IS_REPCODE(o)), (o)+1) /* returns ID 1,2,3 */
|
||||
#define STORED_TO_OFFBASE(o) ((o)+1)
|
||||
#define OFFBASE_TO_STORED(o) ((o)-1)
|
||||
|
||||
/*! ZSTD_storeSeq() :
|
||||
* Store a sequence (litlen, litPtr, offCode and matchLength) into seqStore_t.
|
||||
* @offBase_minus1 : Users should use employ macros STORE_REPCODE_X and STORE_OFFSET().
|
||||
* @matchLength : must be >= MINMATCH
|
||||
* Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t.
|
||||
* `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes).
|
||||
* `mlBase` : matchLength - MINMATCH
|
||||
* Allowed to overread literals up to litLimit.
|
||||
*/
|
||||
HINT_INLINE UNUSED_ATTR void
|
||||
ZSTD_storeSeq(seqStore_t* seqStorePtr,
|
||||
size_t litLength, const BYTE* literals, const BYTE* litLimit,
|
||||
U32 offBase_minus1,
|
||||
size_t matchLength)
|
||||
HINT_INLINE UNUSED_ATTR
|
||||
void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase)
|
||||
{
|
||||
BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
|
||||
BYTE const* const litEnd = literals + litLength;
|
||||
@@ -610,7 +379,7 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
|
||||
if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
|
||||
{ U32 const pos = (U32)((const BYTE*)literals - g_start);
|
||||
DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u",
|
||||
pos, (U32)litLength, (U32)matchLength, (U32)offBase_minus1);
|
||||
pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode);
|
||||
}
|
||||
#endif
|
||||
assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
|
||||
@@ -634,66 +403,26 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
|
||||
|
||||
/* literal Length */
|
||||
if (litLength>0xFFFF) {
|
||||
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
|
||||
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthID = 1;
|
||||
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
}
|
||||
seqStorePtr->sequences[0].litLength = (U16)litLength;
|
||||
|
||||
/* match offset */
|
||||
seqStorePtr->sequences[0].offBase = STORED_TO_OFFBASE(offBase_minus1);
|
||||
seqStorePtr->sequences[0].offset = offCode + 1;
|
||||
|
||||
/* match Length */
|
||||
assert(matchLength >= MINMATCH);
|
||||
{ size_t const mlBase = matchLength - MINMATCH;
|
||||
if (mlBase>0xFFFF) {
|
||||
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
|
||||
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
}
|
||||
seqStorePtr->sequences[0].mlBase = (U16)mlBase;
|
||||
if (mlBase>0xFFFF) {
|
||||
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthID = 2;
|
||||
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
}
|
||||
seqStorePtr->sequences[0].matchLength = (U16)mlBase;
|
||||
|
||||
seqStorePtr->sequences++;
|
||||
}
|
||||
|
||||
/* ZSTD_updateRep() :
|
||||
* updates in-place @rep (array of repeat offsets)
|
||||
* @offBase_minus1 : sum-type, with same numeric representation as ZSTD_storeSeq()
|
||||
*/
|
||||
MEM_STATIC void
|
||||
ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0)
|
||||
{
|
||||
if (STORED_IS_OFFSET(offBase_minus1)) { /* full offset */
|
||||
rep[2] = rep[1];
|
||||
rep[1] = rep[0];
|
||||
rep[0] = STORED_OFFSET(offBase_minus1);
|
||||
} else { /* repcode */
|
||||
U32 const repCode = STORED_REPCODE(offBase_minus1) - 1 + ll0;
|
||||
if (repCode > 0) { /* note : if repCode==0, no change */
|
||||
U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
|
||||
rep[2] = (repCode >= 2) ? rep[1] : rep[2];
|
||||
rep[1] = rep[0];
|
||||
rep[0] = currentOffset;
|
||||
} else { /* repCode == 0 */
|
||||
/* nothing to do */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct repcodes_s {
|
||||
U32 rep[3];
|
||||
} repcodes_t;
|
||||
|
||||
MEM_STATIC repcodes_t
|
||||
ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0)
|
||||
{
|
||||
repcodes_t newReps;
|
||||
ZSTD_memcpy(&newReps, rep, sizeof(newReps));
|
||||
ZSTD_updateRep(newReps.rep, offBase_minus1, ll0);
|
||||
return newReps;
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Match length counter
|
||||
@@ -703,18 +432,9 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
# if STATIC_BMI2
|
||||
return _tzcnt_u64(val) >> 3;
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward64(&r, (U64)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
unsigned long r = 0;
|
||||
_BitScanForward64( &r, (U64)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return (__builtin_ctzll((U64)val) >> 3);
|
||||
# else
|
||||
@@ -730,14 +450,9 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, (U32)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
unsigned long r=0;
|
||||
_BitScanForward( &r, (U32)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_ctz((U32)val) >> 3);
|
||||
# else
|
||||
@@ -751,18 +466,9 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
||||
} else { /* Big Endian CPU */
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
# if STATIC_BMI2
|
||||
return _lzcnt_u64(val) >> 3;
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse64(&r, (U64)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse64( &r, val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return (__builtin_clzll(val) >> 3);
|
||||
# else
|
||||
@@ -775,14 +481,9 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, (unsigned long)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse( &r, (unsigned long)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_clz((U32)val) >> 3);
|
||||
# else
|
||||
@@ -863,8 +564,7 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
|
||||
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR
|
||||
size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
{
|
||||
switch(mls)
|
||||
{
|
||||
@@ -961,13 +661,6 @@ MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
|
||||
window->dictLimit = end;
|
||||
}
|
||||
|
||||
MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window)
|
||||
{
|
||||
return window.dictLimit == ZSTD_WINDOW_START_INDEX &&
|
||||
window.lowLimit == ZSTD_WINDOW_START_INDEX &&
|
||||
(window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_hasExtDict():
|
||||
* Returns non-zero if the window has a non-empty extDict.
|
||||
@@ -987,76 +680,20 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
|
||||
return ZSTD_window_hasExtDict(ms->window) ?
|
||||
ZSTD_extDict :
|
||||
ms->dictMatchState != NULL ?
|
||||
(ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) :
|
||||
ZSTD_dictMatchState :
|
||||
ZSTD_noDict;
|
||||
}
|
||||
|
||||
/* Defining this macro to non-zero tells zstd to run the overflow correction
|
||||
* code much more frequently. This is very inefficient, and should only be
|
||||
* used for tests and fuzzers.
|
||||
*/
|
||||
#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY
|
||||
# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
||||
# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1
|
||||
# else
|
||||
# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/**
|
||||
* ZSTD_window_canOverflowCorrect():
|
||||
* Returns non-zero if the indices are large enough for overflow correction
|
||||
* to work correctly without impacting compression ratio.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window,
|
||||
U32 cycleLog,
|
||||
U32 maxDist,
|
||||
U32 loadedDictEnd,
|
||||
void const* src)
|
||||
{
|
||||
U32 const cycleSize = 1u << cycleLog;
|
||||
U32 const curr = (U32)((BYTE const*)src - window.base);
|
||||
U32 const minIndexToOverflowCorrect = cycleSize
|
||||
+ MAX(maxDist, cycleSize)
|
||||
+ ZSTD_WINDOW_START_INDEX;
|
||||
|
||||
/* Adjust the min index to backoff the overflow correction frequency,
|
||||
* so we don't waste too much CPU in overflow correction. If this
|
||||
* computation overflows we don't really care, we just need to make
|
||||
* sure it is at least minIndexToOverflowCorrect.
|
||||
*/
|
||||
U32 const adjustment = window.nbOverflowCorrections + 1;
|
||||
U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment,
|
||||
minIndexToOverflowCorrect);
|
||||
U32 const indexLargeEnough = curr > adjustedIndex;
|
||||
|
||||
/* Only overflow correct early if the dictionary is invalidated already,
|
||||
* so we don't hurt compression ratio.
|
||||
*/
|
||||
U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd;
|
||||
|
||||
return indexLargeEnough && dictionaryInvalidated;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_needOverflowCorrection():
|
||||
* Returns non-zero if the indices are getting too large and need overflow
|
||||
* protection.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
|
||||
U32 cycleLog,
|
||||
U32 maxDist,
|
||||
U32 loadedDictEnd,
|
||||
void const* src,
|
||||
void const* srcEnd)
|
||||
{
|
||||
U32 const curr = (U32)((BYTE const*)srcEnd - window.base);
|
||||
if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
|
||||
if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return curr > ZSTD_CURRENT_MAX;
|
||||
U32 const current = (U32)((BYTE const*)srcEnd - window.base);
|
||||
return current > ZSTD_CURRENT_MAX;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1067,6 +704,7 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
|
||||
*
|
||||
* The least significant cycleLog bits of the indices must remain the same,
|
||||
* which may be 0. Every index up to maxDist in the past must be valid.
|
||||
* NOTE: (maxDist & cycleMask) must be zero.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
|
||||
U32 maxDist, void const* src)
|
||||
@@ -1090,51 +728,19 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
|
||||
* 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
|
||||
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
|
||||
*/
|
||||
U32 const cycleSize = 1u << cycleLog;
|
||||
U32 const cycleMask = cycleSize - 1;
|
||||
U32 const curr = (U32)((BYTE const*)src - window->base);
|
||||
U32 const currentCycle = curr & cycleMask;
|
||||
/* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */
|
||||
U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX
|
||||
? MAX(cycleSize, ZSTD_WINDOW_START_INDEX)
|
||||
: 0;
|
||||
U32 const newCurrent = currentCycle
|
||||
+ currentCycleCorrection
|
||||
+ MAX(maxDist, cycleSize);
|
||||
U32 const correction = curr - newCurrent;
|
||||
/* maxDist must be a power of two so that:
|
||||
* (newCurrent & cycleMask) == (curr & cycleMask)
|
||||
* This is required to not corrupt the chains / binary tree.
|
||||
*/
|
||||
assert((maxDist & (maxDist - 1)) == 0);
|
||||
assert((curr & cycleMask) == (newCurrent & cycleMask));
|
||||
assert(curr > newCurrent);
|
||||
if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
|
||||
/* Loose bound, should be around 1<<29 (see above) */
|
||||
assert(correction > 1<<28);
|
||||
}
|
||||
U32 const cycleMask = (1U << cycleLog) - 1;
|
||||
U32 const current = (U32)((BYTE const*)src - window->base);
|
||||
U32 const newCurrent = (current & cycleMask) + maxDist;
|
||||
U32 const correction = current - newCurrent;
|
||||
assert((maxDist & cycleMask) == 0);
|
||||
assert(current > newCurrent);
|
||||
/* Loose bound, should be around 1<<29 (see above) */
|
||||
assert(correction > 1<<28);
|
||||
|
||||
window->base += correction;
|
||||
window->dictBase += correction;
|
||||
if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) {
|
||||
window->lowLimit = ZSTD_WINDOW_START_INDEX;
|
||||
} else {
|
||||
window->lowLimit -= correction;
|
||||
}
|
||||
if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) {
|
||||
window->dictLimit = ZSTD_WINDOW_START_INDEX;
|
||||
} else {
|
||||
window->dictLimit -= correction;
|
||||
}
|
||||
|
||||
/* Ensure we can still reference the full window. */
|
||||
assert(newCurrent >= maxDist);
|
||||
assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX);
|
||||
/* Ensure that lowLimit and dictLimit didn't underflow. */
|
||||
assert(window->lowLimit <= newCurrent);
|
||||
assert(window->dictLimit <= newCurrent);
|
||||
|
||||
++window->nbOverflowCorrections;
|
||||
window->lowLimit -= correction;
|
||||
window->dictLimit -= correction;
|
||||
|
||||
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
|
||||
window->lowLimit);
|
||||
@@ -1238,17 +844,6 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
|
||||
} } }
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
|
||||
ZSTD_memset(window, 0, sizeof(*window));
|
||||
window->base = (BYTE const*)" ";
|
||||
window->dictBase = (BYTE const*)" ";
|
||||
ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */
|
||||
window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */
|
||||
window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */
|
||||
window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */
|
||||
window->nbOverflowCorrections = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* ZSTD_window_update():
|
||||
* Updates the window by appending [src, src + srcSize) to the window.
|
||||
@@ -1257,18 +852,13 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
|
||||
* Returns non-zero if the segment is contiguous.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
||||
void const* src, size_t srcSize,
|
||||
int forceNonContiguous)
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
BYTE const* const ip = (BYTE const*)src;
|
||||
U32 contiguous = 1;
|
||||
DEBUGLOG(5, "ZSTD_window_update");
|
||||
if (srcSize == 0)
|
||||
return contiguous;
|
||||
assert(window->base != NULL);
|
||||
assert(window->dictBase != NULL);
|
||||
/* Check if blocks follow each other */
|
||||
if (src != window->nextSrc || forceNonContiguous) {
|
||||
if (src != window->nextSrc) {
|
||||
/* not contiguous */
|
||||
size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
|
||||
DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
|
||||
@@ -1277,7 +867,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
||||
window->dictLimit = (U32)distanceFromBase;
|
||||
window->dictBase = window->base;
|
||||
window->base = ip - distanceFromBase;
|
||||
/* ms->nextToUpdate = window->dictLimit; */
|
||||
// ms->nextToUpdate = window->dictLimit;
|
||||
if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */
|
||||
contiguous = 0;
|
||||
}
|
||||
@@ -1293,35 +883,12 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
||||
return contiguous;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
|
||||
{
|
||||
U32 const maxDistance = 1U << windowLog;
|
||||
U32 const lowestValid = ms->window.lowLimit;
|
||||
U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
|
||||
U32 const isDictionary = (ms->loadedDictEnd != 0);
|
||||
/* When using a dictionary the entire dictionary is valid if a single byte of the dictionary
|
||||
* is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't
|
||||
* valid for the entire block. So this check is sufficient to find the lowest valid match index.
|
||||
*/
|
||||
U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
|
||||
return matchLowest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the lowest allowed match index in the prefix.
|
||||
*/
|
||||
MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
|
||||
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
|
||||
{
|
||||
U32 const maxDistance = 1U << windowLog;
|
||||
U32 const lowestValid = ms->window.dictLimit;
|
||||
U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
|
||||
U32 const lowestValid = ms->window.lowLimit;
|
||||
U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
|
||||
U32 const isDictionary = (ms->loadedDictEnd != 0);
|
||||
/* When computing the lowest prefix index we need to take the dictionary into account to handle
|
||||
* the edge case where the dictionary and the source are contiguous in memory.
|
||||
*/
|
||||
U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
|
||||
return matchLowest;
|
||||
}
|
||||
@@ -1364,20 +931,6 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ===============================================================
|
||||
* Shared internal declarations
|
||||
* These prototypes may be called from sources not in lib/compress
|
||||
* =============================================================== */
|
||||
|
||||
/* ZSTD_loadCEntropy() :
|
||||
* dict : must point at beginning of a valid zstd dictionary.
|
||||
* return : size of dictionary header (size of magic number + dict ID + entropy tables)
|
||||
* assumptions : magic number supposed already checked
|
||||
* and dictSize >= 8 */
|
||||
size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
|
||||
const void* const dict, size_t dictSize);
|
||||
|
||||
void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
|
||||
|
||||
/* ==============================================================
|
||||
* Private declarations
|
||||
@@ -1387,10 +940,9 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
|
||||
/* ZSTD_getCParamsFromCCtxParams() :
|
||||
* cParams are built depending on compressionLevel, src size hints,
|
||||
* LDM and manually set compression parameters.
|
||||
* Note: srcSizeHint == 0 means 0!
|
||||
*/
|
||||
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
|
||||
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
|
||||
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
|
||||
|
||||
/*! ZSTD_initCStream_internal() :
|
||||
* Private use only. Init streaming operation.
|
||||
@@ -1447,15 +999,5 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
|
||||
*/
|
||||
size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
|
||||
|
||||
/** ZSTD_cycleLog() :
|
||||
* condition for correct operation : hashLog > 1 */
|
||||
U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
|
||||
|
||||
/** ZSTD_CCtx_trace() :
|
||||
* Trace the end of a compression call.
|
||||
*/
|
||||
void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
|
||||
|
||||
#endif /* ZSTD_COMPRESS_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
40
vendor/github.com/DataDog/zstd/zstd_compress_literals.c
generated
vendored
40
vendor/github.com/DataDog/zstd/zstd_compress_literals.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -16,10 +15,10 @@
|
||||
|
||||
size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
||||
|
||||
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
|
||||
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall);
|
||||
|
||||
switch(flSize)
|
||||
{
|
||||
@@ -36,14 +35,13 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
|
||||
assert(0);
|
||||
}
|
||||
|
||||
ZSTD_memcpy(ostart + flSize, src, srcSize);
|
||||
DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
|
||||
memcpy(ostart + flSize, src, srcSize);
|
||||
return srcSize + flSize;
|
||||
}
|
||||
|
||||
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
||||
|
||||
(void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
|
||||
@@ -64,7 +62,6 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
|
||||
}
|
||||
|
||||
ostart[flSize] = *(const BYTE*)src;
|
||||
DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
|
||||
return flSize+1;
|
||||
}
|
||||
|
||||
@@ -74,8 +71,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
||||
const int bmi2,
|
||||
unsigned suspectUncompressible)
|
||||
const int bmi2)
|
||||
{
|
||||
size_t const minGain = ZSTD_minGain(srcSize, strategy);
|
||||
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
|
||||
@@ -84,11 +80,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
symbolEncodingType_e hType = set_compressed;
|
||||
size_t cLitSize;
|
||||
|
||||
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
|
||||
disableLiteralCompression, (U32)srcSize);
|
||||
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)",
|
||||
disableLiteralCompression);
|
||||
|
||||
/* Prepare nextEntropy assuming reusing the existing table */
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
|
||||
if (disableLiteralCompression)
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
@@ -106,25 +102,24 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
cLitSize = singleStream ?
|
||||
HUF_compress1X_repeat(
|
||||
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
||||
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) :
|
||||
255, 11, entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
|
||||
HUF_compress4X_repeat(
|
||||
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
||||
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible);
|
||||
255, 11, entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
|
||||
if (repeat != HUF_repeat_none) {
|
||||
/* reused the existing table */
|
||||
DEBUGLOG(5, "Reusing previous huffman table");
|
||||
hType = set_repeat;
|
||||
}
|
||||
}
|
||||
|
||||
if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
|
||||
memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
if (cLitSize==1) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
@@ -155,8 +150,5 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
default: /* not possible : lhSize is {3,4,5} */
|
||||
assert(0);
|
||||
}
|
||||
DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
|
||||
return lhSize+cLitSize;
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
9
vendor/github.com/DataDog/zstd/zstd_compress_literals.h
generated
vendored
9
vendor/github.com/DataDog/zstd/zstd_compress_literals.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -19,16 +18,12 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
|
||||
|
||||
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
|
||||
size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
ZSTD_hufCTables_t* nextHuf,
|
||||
ZSTD_strategy strategy, int disableLiteralCompression,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
||||
const int bmi2,
|
||||
unsigned suspectUncompressible);
|
||||
const int bmi2);
|
||||
|
||||
#endif /* ZSTD_COMPRESS_LITERALS_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
98
vendor/github.com/DataDog/zstd/zstd_compress_sequences.c
generated
vendored
98
vendor/github.com/DataDog/zstd/zstd_compress_sequences.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -51,19 +50,6 @@ static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
|
||||
return maxSymbolValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if we should use ncount=-1 else we should
|
||||
* use ncount=1 for low probability symbols instead.
|
||||
*/
|
||||
static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
|
||||
{
|
||||
/* Heuristic: This should cover most blocks <= 16K and
|
||||
* start to fade out after 16K to about 32K depending on
|
||||
* comprssibility.
|
||||
*/
|
||||
return nbSeq >= 2048;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the cost in bytes of encoding the normalized count header.
|
||||
* Returns an error if any of the helper functions return an error.
|
||||
@@ -74,7 +60,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
|
||||
BYTE wksp[FSE_NCOUNTBOUND];
|
||||
S16 norm[MaxSeq + 1];
|
||||
const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
|
||||
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), "");
|
||||
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
|
||||
return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
|
||||
}
|
||||
|
||||
@@ -86,8 +72,6 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
|
||||
{
|
||||
unsigned cost = 0;
|
||||
unsigned s;
|
||||
|
||||
assert(total > 0);
|
||||
for (s = 0; s <= max; ++s) {
|
||||
unsigned norm = (unsigned)((256 * count[s]) / total);
|
||||
if (count[s] != 0 && norm == 0)
|
||||
@@ -102,7 +86,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
|
||||
* Returns the cost in bits of encoding the distribution in count using ctable.
|
||||
* Returns an error if ctable cannot represent all the symbols in count.
|
||||
*/
|
||||
size_t ZSTD_fseBitCost(
|
||||
static size_t ZSTD_fseBitCost(
|
||||
FSE_CTable const* ctable,
|
||||
unsigned const* count,
|
||||
unsigned const max)
|
||||
@@ -112,22 +96,18 @@ size_t ZSTD_fseBitCost(
|
||||
unsigned s;
|
||||
FSE_CState_t cstate;
|
||||
FSE_initCState(&cstate, ctable);
|
||||
if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
|
||||
DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
|
||||
RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC,
|
||||
"Repeat FSE_CTable has maxSymbolValue %u < %u",
|
||||
ZSTD_getFSEMaxSymbolValue(ctable), max);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
for (s = 0; s <= max; ++s) {
|
||||
unsigned const tableLog = cstate.stateLog;
|
||||
unsigned const badCost = (tableLog + 1) << kAccuracyLog;
|
||||
unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
|
||||
if (count[s] == 0)
|
||||
continue;
|
||||
if (bitCost >= badCost) {
|
||||
DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
cost += (size_t)count[s] * bitCost;
|
||||
RETURN_ERROR_IF(bitCost >= badCost, GENERIC,
|
||||
"Repeat FSE_CTable has Prob[%u] == 0", s);
|
||||
cost += count[s] * bitCost;
|
||||
}
|
||||
return cost >> kAccuracyLog;
|
||||
}
|
||||
@@ -137,15 +117,15 @@ size_t ZSTD_fseBitCost(
|
||||
* table described by norm. The max symbol support by norm is assumed >= max.
|
||||
* norm must be valid for every symbol with non-zero probability in count.
|
||||
*/
|
||||
size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
|
||||
unsigned const* count, unsigned const max)
|
||||
static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
|
||||
unsigned const* count, unsigned const max)
|
||||
{
|
||||
unsigned const shift = 8 - accuracyLog;
|
||||
size_t cost = 0;
|
||||
unsigned s;
|
||||
assert(accuracyLog <= 8);
|
||||
for (s = 0; s <= max; ++s) {
|
||||
unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1;
|
||||
unsigned const normAcc = norm[s] != -1 ? norm[s] : 1;
|
||||
unsigned const norm256 = normAcc << shift;
|
||||
assert(norm256 > 0);
|
||||
assert(norm256 < 256);
|
||||
@@ -235,11 +215,6 @@ ZSTD_selectEncodingType(
|
||||
return set_compressed;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
S16 norm[MaxSeq + 1];
|
||||
U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)];
|
||||
} ZSTD_BuildCTableWksp;
|
||||
|
||||
size_t
|
||||
ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
||||
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
|
||||
@@ -255,18 +230,18 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
||||
|
||||
switch (type) {
|
||||
case set_rle:
|
||||
FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
|
||||
RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space");
|
||||
FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
|
||||
RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall);
|
||||
*op = codeTable[0];
|
||||
return 1;
|
||||
case set_repeat:
|
||||
ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize);
|
||||
memcpy(nextCTable, prevCTable, prevCTableSize);
|
||||
return 0;
|
||||
case set_basic:
|
||||
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
|
||||
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */
|
||||
return 0;
|
||||
case set_compressed: {
|
||||
ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace;
|
||||
S16 norm[MaxSeq + 1];
|
||||
size_t nbSeq_1 = nbSeq;
|
||||
const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
|
||||
if (count[codeTable[nbSeq-1]] > 1) {
|
||||
@@ -274,17 +249,14 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
||||
nbSeq_1--;
|
||||
}
|
||||
assert(nbSeq_1 > 1);
|
||||
assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
|
||||
(void)entropyWorkspaceSize;
|
||||
FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed");
|
||||
assert(oend >= op);
|
||||
{ size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */
|
||||
FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
|
||||
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed");
|
||||
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
|
||||
{ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
|
||||
FORWARD_IF_ERROR(NCountSize);
|
||||
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize));
|
||||
return NCountSize;
|
||||
}
|
||||
}
|
||||
default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach");
|
||||
default: assert(0); RETURN_ERROR(GENERIC);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,19 +286,19 @@ ZSTD_encodeSequences_body(
|
||||
FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
|
||||
if (MEM_32bits()) BIT_flushBits(&blockStream);
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]);
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
|
||||
if (MEM_32bits()) BIT_flushBits(&blockStream);
|
||||
if (longOffsets) {
|
||||
U32 const ofBits = ofCodeTable[nbSeq-1];
|
||||
unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
||||
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
||||
if (extraBits) {
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits);
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
|
||||
BIT_flushBits(&blockStream);
|
||||
}
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits,
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
|
||||
ofBits - extraBits);
|
||||
} else {
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]);
|
||||
BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
|
||||
}
|
||||
BIT_flushBits(&blockStream);
|
||||
|
||||
@@ -340,8 +312,8 @@ ZSTD_encodeSequences_body(
|
||||
U32 const mlBits = ML_bits[mlCode];
|
||||
DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
|
||||
(unsigned)sequences[n].litLength,
|
||||
(unsigned)sequences[n].mlBase + MINMATCH,
|
||||
(unsigned)sequences[n].offBase);
|
||||
(unsigned)sequences[n].matchLength + MINMATCH,
|
||||
(unsigned)sequences[n].offset);
|
||||
/* 32b*/ /* 64b*/
|
||||
/* (7)*/ /* (7)*/
|
||||
FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
|
||||
@@ -352,18 +324,18 @@ ZSTD_encodeSequences_body(
|
||||
BIT_flushBits(&blockStream); /* (7)*/
|
||||
BIT_addBits(&blockStream, sequences[n].litLength, llBits);
|
||||
if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
|
||||
BIT_addBits(&blockStream, sequences[n].mlBase, mlBits);
|
||||
BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
|
||||
if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
|
||||
if (longOffsets) {
|
||||
unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
||||
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
||||
if (extraBits) {
|
||||
BIT_addBits(&blockStream, sequences[n].offBase, extraBits);
|
||||
BIT_addBits(&blockStream, sequences[n].offset, extraBits);
|
||||
BIT_flushBits(&blockStream); /* (7)*/
|
||||
}
|
||||
BIT_addBits(&blockStream, sequences[n].offBase >> extraBits,
|
||||
BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
|
||||
ofBits - extraBits); /* 31 */
|
||||
} else {
|
||||
BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */
|
||||
BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
|
||||
}
|
||||
BIT_flushBits(&blockStream); /* (7)*/
|
||||
DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr));
|
||||
@@ -400,7 +372,7 @@ ZSTD_encodeSequences_default(
|
||||
|
||||
#if DYNAMIC_BMI2
|
||||
|
||||
static BMI2_TARGET_ATTRIBUTE size_t
|
||||
static TARGET_ATTRIBUTE("bmi2") size_t
|
||||
ZSTD_encodeSequences_bmi2(
|
||||
void* dst, size_t dstCapacity,
|
||||
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
|
||||
@@ -441,5 +413,3 @@ size_t ZSTD_encodeSequences(
|
||||
CTable_LitLength, llCodeTable,
|
||||
sequences, nbSeq, longOffsets);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
12
vendor/github.com/DataDog/zstd/zstd_compress_sequences.h
generated
vendored
12
vendor/github.com/DataDog/zstd/zstd_compress_sequences.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -45,13 +44,4 @@ size_t ZSTD_encodeSequences(
|
||||
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
|
||||
seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
|
||||
|
||||
size_t ZSTD_fseBitCost(
|
||||
FSE_CTable const* ctable,
|
||||
unsigned const* count,
|
||||
unsigned const max);
|
||||
|
||||
size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
|
||||
unsigned const* count, unsigned const max);
|
||||
#endif /* ZSTD_COMPRESS_SEQUENCES_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
576
vendor/github.com/DataDog/zstd/zstd_compress_superblock.c
generated
vendored
576
vendor/github.com/DataDog/zstd/zstd_compress_superblock.c
generated
vendored
@@ -1,576 +0,0 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "zstd_compress_superblock.h"
|
||||
|
||||
#include "zstd_internal.h" /* ZSTD_getSequenceLength */
|
||||
#include "hist.h" /* HIST_countFast_wksp */
|
||||
#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
|
||||
#include "zstd_compress_sequences.h"
|
||||
#include "zstd_compress_literals.h"
|
||||
|
||||
/** ZSTD_compressSubBlock_literal() :
|
||||
* Compresses literals section for a sub-block.
|
||||
* When we have to write the Huffman table we will sometimes choose a header
|
||||
* size larger than necessary. This is because we have to pick the header size
|
||||
* before we know the table size + compressed size, so we have a bound on the
|
||||
* table size. If we guessed incorrectly, we fall back to uncompressed literals.
|
||||
*
|
||||
* We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
|
||||
* in writing the header, otherwise it is set to 0.
|
||||
*
|
||||
* hufMetadata->hType has literals block type info.
|
||||
* If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block.
|
||||
* If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block.
|
||||
* If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block
|
||||
* If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
|
||||
* and the following sub-blocks' literals sections will be Treeless_Literals_Block.
|
||||
* @return : compressed size of literals section of a sub-block
|
||||
* Or 0 if it unable to compress.
|
||||
* Or error code */
|
||||
static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
||||
const ZSTD_hufCTablesMetadata_t* hufMetadata,
|
||||
const BYTE* literals, size_t litSize,
|
||||
void* dst, size_t dstSize,
|
||||
const int bmi2, int writeEntropy, int* entropyWritten)
|
||||
{
|
||||
size_t const header = writeEntropy ? 200 : 0;
|
||||
size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header));
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart + lhSize;
|
||||
U32 const singleStream = lhSize == 3;
|
||||
symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
|
||||
size_t cLitSize = 0;
|
||||
|
||||
(void)bmi2; /* TODO bmi2... */
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
|
||||
|
||||
*entropyWritten = 0;
|
||||
if (litSize == 0 || hufMetadata->hType == set_basic) {
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal");
|
||||
return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
|
||||
} else if (hufMetadata->hType == set_rle) {
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal");
|
||||
return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize);
|
||||
}
|
||||
|
||||
assert(litSize > 0);
|
||||
assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat);
|
||||
|
||||
if (writeEntropy && hufMetadata->hType == set_compressed) {
|
||||
ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
|
||||
op += hufMetadata->hufDesSize;
|
||||
cLitSize += hufMetadata->hufDesSize;
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
|
||||
}
|
||||
|
||||
/* TODO bmi2 */
|
||||
{ const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable)
|
||||
: HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable);
|
||||
op += cSize;
|
||||
cLitSize += cSize;
|
||||
if (cSize == 0 || ERR_isError(cSize)) {
|
||||
DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize));
|
||||
return 0;
|
||||
}
|
||||
/* If we expand and we aren't writing a header then emit uncompressed */
|
||||
if (!writeEntropy && cLitSize >= litSize) {
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible");
|
||||
return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
|
||||
}
|
||||
/* If we are writing headers then allow expansion that doesn't change our header size. */
|
||||
if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) {
|
||||
assert(cLitSize > litSize);
|
||||
DEBUGLOG(5, "Literals expanded beyond allowed header size");
|
||||
return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize);
|
||||
}
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize);
|
||||
}
|
||||
|
||||
/* Build header */
|
||||
switch(lhSize)
|
||||
{
|
||||
case 3: /* 2 - 2 - 10 - 10 */
|
||||
{ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14);
|
||||
MEM_writeLE24(ostart, lhc);
|
||||
break;
|
||||
}
|
||||
case 4: /* 2 - 2 - 14 - 14 */
|
||||
{ U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18);
|
||||
MEM_writeLE32(ostart, lhc);
|
||||
break;
|
||||
}
|
||||
case 5: /* 2 - 2 - 18 - 18 */
|
||||
{ U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22);
|
||||
MEM_writeLE32(ostart, lhc);
|
||||
ostart[4] = (BYTE)(cLitSize >> 10);
|
||||
break;
|
||||
}
|
||||
default: /* not possible : lhSize is {3,4,5} */
|
||||
assert(0);
|
||||
}
|
||||
*entropyWritten = 1;
|
||||
DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart));
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) {
|
||||
const seqDef* const sstart = sequences;
|
||||
const seqDef* const send = sequences + nbSeq;
|
||||
const seqDef* sp = sstart;
|
||||
size_t matchLengthSum = 0;
|
||||
size_t litLengthSum = 0;
|
||||
(void)(litLengthSum); /* suppress unused variable warning on some environments */
|
||||
while (send-sp > 0) {
|
||||
ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
|
||||
litLengthSum += seqLen.litLength;
|
||||
matchLengthSum += seqLen.matchLength;
|
||||
sp++;
|
||||
}
|
||||
assert(litLengthSum <= litSize);
|
||||
if (!lastSequence) {
|
||||
assert(litLengthSum == litSize);
|
||||
}
|
||||
return matchLengthSum + litSize;
|
||||
}
|
||||
|
||||
/** ZSTD_compressSubBlock_sequences() :
|
||||
* Compresses sequences section for a sub-block.
|
||||
* fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have
|
||||
* symbol compression modes for the super-block.
|
||||
* The first successfully compressed block will have these in its header.
|
||||
* We set entropyWritten=1 when we succeed in compressing the sequences.
|
||||
* The following sub-blocks will always have repeat mode.
|
||||
* @return : compressed size of sequences section of a sub-block
|
||||
* Or 0 if it is unable to compress
|
||||
* Or error code. */
|
||||
static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
|
||||
const ZSTD_fseCTablesMetadata_t* fseMetadata,
|
||||
const seqDef* sequences, size_t nbSeq,
|
||||
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
void* dst, size_t dstCapacity,
|
||||
const int bmi2, int writeEntropy, int* entropyWritten)
|
||||
{
|
||||
const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
BYTE* seqHead;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets);
|
||||
|
||||
*entropyWritten = 0;
|
||||
/* Sequences Header */
|
||||
RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
|
||||
dstSize_tooSmall, "");
|
||||
if (nbSeq < 0x7F)
|
||||
*op++ = (BYTE)nbSeq;
|
||||
else if (nbSeq < LONGNBSEQ)
|
||||
op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
|
||||
else
|
||||
op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
|
||||
if (nbSeq==0) {
|
||||
return op - ostart;
|
||||
}
|
||||
|
||||
/* seqHead : flags for FSE encoding type */
|
||||
seqHead = op++;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart));
|
||||
|
||||
if (writeEntropy) {
|
||||
const U32 LLtype = fseMetadata->llType;
|
||||
const U32 Offtype = fseMetadata->ofType;
|
||||
const U32 MLtype = fseMetadata->mlType;
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize);
|
||||
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
|
||||
ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
|
||||
op += fseMetadata->fseTablesSize;
|
||||
} else {
|
||||
const U32 repeat = set_repeat;
|
||||
*seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2));
|
||||
}
|
||||
|
||||
{ size_t const bitstreamSize = ZSTD_encodeSequences(
|
||||
op, oend - op,
|
||||
fseTables->matchlengthCTable, mlCode,
|
||||
fseTables->offcodeCTable, ofCode,
|
||||
fseTables->litlengthCTable, llCode,
|
||||
sequences, nbSeq,
|
||||
longOffsets, bmi2);
|
||||
FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
|
||||
op += bitstreamSize;
|
||||
/* zstd versions <= 1.3.4 mistakenly report corruption when
|
||||
* FSE_readNCount() receives a buffer < 4 bytes.
|
||||
* Fixed by https://github.com/facebook/zstd/pull/1146.
|
||||
* This can happen when the last set_compressed table present is 2
|
||||
* bytes and the bitstream is only one byte.
|
||||
* In this exceedingly rare case, we will simply emit an uncompressed
|
||||
* block, since it isn't worth optimizing.
|
||||
*/
|
||||
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
||||
if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) {
|
||||
/* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
|
||||
assert(fseMetadata->lastCountSize + bitstreamSize == 3);
|
||||
DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
|
||||
"emitting an uncompressed block.");
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize);
|
||||
}
|
||||
|
||||
/* zstd versions <= 1.4.0 mistakenly report error when
|
||||
* sequences section body size is less than 3 bytes.
|
||||
* Fixed by https://github.com/facebook/zstd/pull/1664.
|
||||
* This can happen when the previous sequences section block is compressed
|
||||
* with rle mode and the current block's sequences section is compressed
|
||||
* with repeat mode where sequences section body size can be 1 byte.
|
||||
*/
|
||||
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
||||
if (op-seqHead < 4) {
|
||||
DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting "
|
||||
"an uncompressed block when sequences are < 4 bytes");
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
*entropyWritten = 1;
|
||||
return op - ostart;
|
||||
}
|
||||
|
||||
/** ZSTD_compressSubBlock() :
|
||||
* Compresses a single sub-block.
|
||||
* @return : compressed size of the sub-block
|
||||
* Or 0 if it failed to compress. */
|
||||
static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
const seqDef* sequences, size_t nbSeq,
|
||||
const BYTE* literals, size_t litSize,
|
||||
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
void* dst, size_t dstCapacity,
|
||||
const int bmi2,
|
||||
int writeLitEntropy, int writeSeqEntropy,
|
||||
int* litEntropyWritten, int* seqEntropyWritten,
|
||||
U32 lastBlock)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart + ZSTD_blockHeaderSize;
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)",
|
||||
litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock);
|
||||
{ size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable,
|
||||
&entropyMetadata->hufMetadata, literals, litSize,
|
||||
op, oend-op, bmi2, writeLitEntropy, litEntropyWritten);
|
||||
FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed");
|
||||
if (cLitSize == 0) return 0;
|
||||
op += cLitSize;
|
||||
}
|
||||
{ size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse,
|
||||
&entropyMetadata->fseMetadata,
|
||||
sequences, nbSeq,
|
||||
llCode, mlCode, ofCode,
|
||||
cctxParams,
|
||||
op, oend-op,
|
||||
bmi2, writeSeqEntropy, seqEntropyWritten);
|
||||
FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed");
|
||||
if (cSeqSize == 0) return 0;
|
||||
op += cSeqSize;
|
||||
}
|
||||
/* Write block header */
|
||||
{ size_t cSize = (op-ostart)-ZSTD_blockHeaderSize;
|
||||
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
|
||||
MEM_writeLE24(ostart, cBlockHeader24);
|
||||
}
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize,
|
||||
const ZSTD_hufCTables_t* huf,
|
||||
const ZSTD_hufCTablesMetadata_t* hufMetadata,
|
||||
void* workspace, size_t wkspSize,
|
||||
int writeEntropy)
|
||||
{
|
||||
unsigned* const countWksp = (unsigned*)workspace;
|
||||
unsigned maxSymbolValue = 255;
|
||||
size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
|
||||
|
||||
if (hufMetadata->hType == set_basic) return litSize;
|
||||
else if (hufMetadata->hType == set_rle) return 1;
|
||||
else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) {
|
||||
size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize);
|
||||
if (ZSTD_isError(largest)) return litSize;
|
||||
{ size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue);
|
||||
if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize;
|
||||
return cLitSizeEstimate + literalSectionHeaderSize;
|
||||
} }
|
||||
assert(0); /* impossible */
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
|
||||
const BYTE* codeTable, unsigned maxCode,
|
||||
size_t nbSeq, const FSE_CTable* fseCTable,
|
||||
const U8* additionalBits,
|
||||
short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
unsigned* const countWksp = (unsigned*)workspace;
|
||||
const BYTE* ctp = codeTable;
|
||||
const BYTE* const ctStart = ctp;
|
||||
const BYTE* const ctEnd = ctStart + nbSeq;
|
||||
size_t cSymbolTypeSizeEstimateInBits = 0;
|
||||
unsigned max = maxCode;
|
||||
|
||||
HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
|
||||
if (type == set_basic) {
|
||||
/* We selected this encoding type, so it must be valid. */
|
||||
assert(max <= defaultMax);
|
||||
cSymbolTypeSizeEstimateInBits = max <= defaultMax
|
||||
? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max)
|
||||
: ERROR(GENERIC);
|
||||
} else if (type == set_rle) {
|
||||
cSymbolTypeSizeEstimateInBits = 0;
|
||||
} else if (type == set_compressed || type == set_repeat) {
|
||||
cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max);
|
||||
}
|
||||
if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10;
|
||||
while (ctp < ctEnd) {
|
||||
if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp];
|
||||
else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */
|
||||
ctp++;
|
||||
}
|
||||
return cSymbolTypeSizeEstimateInBits / 8;
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
|
||||
const BYTE* llCodeTable,
|
||||
const BYTE* mlCodeTable,
|
||||
size_t nbSeq,
|
||||
const ZSTD_fseCTables_t* fseTables,
|
||||
const ZSTD_fseCTablesMetadata_t* fseMetadata,
|
||||
void* workspace, size_t wkspSize,
|
||||
int writeEntropy)
|
||||
{
|
||||
size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
|
||||
size_t cSeqSizeEstimate = 0;
|
||||
if (nbSeq == 0) return sequencesSectionHeaderSize;
|
||||
cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff,
|
||||
nbSeq, fseTables->offcodeCTable, NULL,
|
||||
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
|
||||
workspace, wkspSize);
|
||||
cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL,
|
||||
nbSeq, fseTables->litlengthCTable, LL_bits,
|
||||
LL_defaultNorm, LL_defaultNormLog, MaxLL,
|
||||
workspace, wkspSize);
|
||||
cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML,
|
||||
nbSeq, fseTables->matchlengthCTable, ML_bits,
|
||||
ML_defaultNorm, ML_defaultNormLog, MaxML,
|
||||
workspace, wkspSize);
|
||||
if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
|
||||
return cSeqSizeEstimate + sequencesSectionHeaderSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
|
||||
const BYTE* ofCodeTable,
|
||||
const BYTE* llCodeTable,
|
||||
const BYTE* mlCodeTable,
|
||||
size_t nbSeq,
|
||||
const ZSTD_entropyCTables_t* entropy,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize,
|
||||
int writeLitEntropy, int writeSeqEntropy) {
|
||||
size_t cSizeEstimate = 0;
|
||||
cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize,
|
||||
&entropy->huf, &entropyMetadata->hufMetadata,
|
||||
workspace, wkspSize, writeLitEntropy);
|
||||
cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
|
||||
nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
|
||||
workspace, wkspSize, writeSeqEntropy);
|
||||
return cSizeEstimate + ZSTD_blockHeaderSize;
|
||||
}
|
||||
|
||||
static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata)
|
||||
{
|
||||
if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle)
|
||||
return 1;
|
||||
if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle)
|
||||
return 1;
|
||||
if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle)
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** ZSTD_compressSubBlock_multi() :
|
||||
* Breaks super-block into multiple sub-blocks and compresses them.
|
||||
* Entropy will be written to the first block.
|
||||
* The following blocks will use repeat mode to compress.
|
||||
* All sub-blocks are compressed blocks (no raw or rle blocks).
|
||||
* @return : compressed size of the super block (which is multiple ZSTD blocks)
|
||||
* Or 0 if it failed to compress. */
|
||||
static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
|
||||
const ZSTD_compressedBlockState_t* prevCBlock,
|
||||
ZSTD_compressedBlockState_t* nextCBlock,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const int bmi2, U32 lastBlock,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
const seqDef* const sstart = seqStorePtr->sequencesStart;
|
||||
const seqDef* const send = seqStorePtr->sequences;
|
||||
const seqDef* sp = sstart;
|
||||
const BYTE* const lstart = seqStorePtr->litStart;
|
||||
const BYTE* const lend = seqStorePtr->lit;
|
||||
const BYTE* lp = lstart;
|
||||
BYTE const* ip = (BYTE const*)src;
|
||||
BYTE const* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
const BYTE* llCodePtr = seqStorePtr->llCode;
|
||||
const BYTE* mlCodePtr = seqStorePtr->mlCode;
|
||||
const BYTE* ofCodePtr = seqStorePtr->ofCode;
|
||||
size_t targetCBlockSize = cctxParams->targetCBlockSize;
|
||||
size_t litSize, seqCount;
|
||||
int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed;
|
||||
int writeSeqEntropy = 1;
|
||||
int lastSequence = 0;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)",
|
||||
(unsigned)(lend-lp), (unsigned)(send-sstart));
|
||||
|
||||
litSize = 0;
|
||||
seqCount = 0;
|
||||
do {
|
||||
size_t cBlockSizeEstimate = 0;
|
||||
if (sstart == send) {
|
||||
lastSequence = 1;
|
||||
} else {
|
||||
const seqDef* const sequence = sp + seqCount;
|
||||
lastSequence = sequence == send - 1;
|
||||
litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength;
|
||||
seqCount++;
|
||||
}
|
||||
if (lastSequence) {
|
||||
assert(lp <= lend);
|
||||
assert(litSize <= (size_t)(lend - lp));
|
||||
litSize = (size_t)(lend - lp);
|
||||
}
|
||||
/* I think there is an optimization opportunity here.
|
||||
* Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful
|
||||
* since it recalculates estimate from scratch.
|
||||
* For example, it would recount literal distribution and symbol codes every time.
|
||||
*/
|
||||
cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount,
|
||||
&nextCBlock->entropy, entropyMetadata,
|
||||
workspace, wkspSize, writeLitEntropy, writeSeqEntropy);
|
||||
if (cBlockSizeEstimate > targetCBlockSize || lastSequence) {
|
||||
int litEntropyWritten = 0;
|
||||
int seqEntropyWritten = 0;
|
||||
const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence);
|
||||
const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
|
||||
sp, seqCount,
|
||||
lp, litSize,
|
||||
llCodePtr, mlCodePtr, ofCodePtr,
|
||||
cctxParams,
|
||||
op, oend-op,
|
||||
bmi2, writeLitEntropy, writeSeqEntropy,
|
||||
&litEntropyWritten, &seqEntropyWritten,
|
||||
lastBlock && lastSequence);
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
|
||||
if (cSize > 0 && cSize < decompressedSize) {
|
||||
DEBUGLOG(5, "Committed the sub-block");
|
||||
assert(ip + decompressedSize <= iend);
|
||||
ip += decompressedSize;
|
||||
sp += seqCount;
|
||||
lp += litSize;
|
||||
op += cSize;
|
||||
llCodePtr += seqCount;
|
||||
mlCodePtr += seqCount;
|
||||
ofCodePtr += seqCount;
|
||||
litSize = 0;
|
||||
seqCount = 0;
|
||||
/* Entropy only needs to be written once */
|
||||
if (litEntropyWritten) {
|
||||
writeLitEntropy = 0;
|
||||
}
|
||||
if (seqEntropyWritten) {
|
||||
writeSeqEntropy = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (!lastSequence);
|
||||
if (writeLitEntropy) {
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
|
||||
ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
|
||||
}
|
||||
if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
|
||||
/* If we haven't written our entropy tables, then we've violated our contract and
|
||||
* must emit an uncompressed block.
|
||||
*/
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten");
|
||||
return 0;
|
||||
}
|
||||
if (ip < iend) {
|
||||
size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock);
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip));
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
|
||||
assert(cSize != 0);
|
||||
op += cSize;
|
||||
/* We have to regenerate the repcodes because we've skipped some sequences */
|
||||
if (sp < send) {
|
||||
seqDef const* seq;
|
||||
repcodes_t rep;
|
||||
ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
|
||||
for (seq = sstart; seq < sp; ++seq) {
|
||||
ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
|
||||
}
|
||||
ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
|
||||
}
|
||||
}
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
||||
void* dst, size_t dstCapacity,
|
||||
void const* src, size_t srcSize,
|
||||
unsigned lastBlock) {
|
||||
ZSTD_entropyCTablesMetadata_t entropyMetadata;
|
||||
|
||||
FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
|
||||
&zc->blockState.prevCBlock->entropy,
|
||||
&zc->blockState.nextCBlock->entropy,
|
||||
&zc->appliedParams,
|
||||
&entropyMetadata,
|
||||
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
|
||||
|
||||
return ZSTD_compressSubBlock_multi(&zc->seqStore,
|
||||
zc->blockState.prevCBlock,
|
||||
zc->blockState.nextCBlock,
|
||||
&entropyMetadata,
|
||||
&zc->appliedParams,
|
||||
dst, dstCapacity,
|
||||
src, srcSize,
|
||||
zc->bmi2, lastBlock,
|
||||
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
35
vendor/github.com/DataDog/zstd/zstd_compress_superblock.h
generated
vendored
35
vendor/github.com/DataDog/zstd/zstd_compress_superblock.h
generated
vendored
@@ -1,35 +0,0 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_COMPRESS_ADVANCED_H
|
||||
#define ZSTD_COMPRESS_ADVANCED_H
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
|
||||
#include "zstd.h" /* ZSTD_CCtx */
|
||||
|
||||
/*-*************************************
|
||||
* Target Compressed Block Size
|
||||
***************************************/
|
||||
|
||||
/* ZSTD_compressSuperBlock() :
|
||||
* Used to compress a super block when targetCBlockSize is being used.
|
||||
* The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */
|
||||
size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
||||
void* dst, size_t dstCapacity,
|
||||
void const* src, size_t srcSize,
|
||||
unsigned lastBlock);
|
||||
|
||||
#endif /* ZSTD_COMPRESS_ADVANCED_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
131
vendor/github.com/DataDog/zstd/zstd_ctx.go
generated
vendored
131
vendor/github.com/DataDog/zstd/zstd_ctx.go
generated
vendored
@@ -1,131 +0,0 @@
|
||||
package zstd
|
||||
|
||||
/*
|
||||
#include "zstd.h"
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"runtime"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
type Ctx interface {
|
||||
// Compress src into dst. If you have a buffer to use, you can pass it to
|
||||
// prevent allocation. If it is too small, or if nil is passed, a new buffer
|
||||
// will be allocated and returned.
|
||||
Compress(dst, src []byte) ([]byte, error)
|
||||
|
||||
// CompressLevel is the same as Compress but you can pass a compression level
|
||||
CompressLevel(dst, src []byte, level int) ([]byte, error)
|
||||
|
||||
// Decompress src into dst. If you have a buffer to use, you can pass it to
|
||||
// prevent allocation. If it is too small, or if nil is passed, a new buffer
|
||||
// will be allocated and returned.
|
||||
Decompress(dst, src []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
type ctx struct {
|
||||
cctx *C.ZSTD_CCtx
|
||||
dctx *C.ZSTD_DCtx
|
||||
}
|
||||
|
||||
// Create a new ZStd Context.
|
||||
// When compressing/decompressing many times, it is recommended to allocate a
|
||||
// context just once, and re-use it for each successive compression operation.
|
||||
// This will make workload friendlier for system's memory.
|
||||
// Note : re-using context is just a speed / resource optimization.
|
||||
// It doesn't change the compression ratio, which remains identical.
|
||||
// Note 2 : In multi-threaded environments,
|
||||
// use one different context per thread for parallel execution.
|
||||
//
|
||||
func NewCtx() Ctx {
|
||||
c := &ctx{
|
||||
cctx: C.ZSTD_createCCtx(),
|
||||
dctx: C.ZSTD_createDCtx(),
|
||||
}
|
||||
|
||||
runtime.SetFinalizer(c, finalizeCtx)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *ctx) Compress(dst, src []byte) ([]byte, error) {
|
||||
return c.CompressLevel(dst, src, DefaultCompression)
|
||||
}
|
||||
|
||||
func (c *ctx) CompressLevel(dst, src []byte, level int) ([]byte, error) {
|
||||
bound := CompressBound(len(src))
|
||||
if cap(dst) >= bound {
|
||||
dst = dst[0:bound] // Reuse dst buffer
|
||||
} else {
|
||||
dst = make([]byte, bound)
|
||||
}
|
||||
|
||||
// We need unsafe.Pointer(&src[0]) in the Cgo call to avoid "Go pointer to Go pointer" panics.
|
||||
// This means we need to special case empty input. See:
|
||||
// https://github.com/golang/go/issues/14210#issuecomment-346402945
|
||||
var cWritten C.size_t
|
||||
if len(src) == 0 {
|
||||
cWritten = C.ZSTD_compressCCtx(
|
||||
c.cctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(nil),
|
||||
C.size_t(0),
|
||||
C.int(level))
|
||||
} else {
|
||||
cWritten = C.ZSTD_compressCCtx(
|
||||
c.cctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src)),
|
||||
C.int(level))
|
||||
}
|
||||
|
||||
written := int(cWritten)
|
||||
// Check if the return is an Error code
|
||||
if err := getError(written); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dst[:written], nil
|
||||
}
|
||||
|
||||
func (c *ctx) Decompress(dst, src []byte) ([]byte, error) {
|
||||
if len(src) == 0 {
|
||||
return []byte{}, ErrEmptySlice
|
||||
}
|
||||
|
||||
bound := decompressSizeHint(src)
|
||||
if cap(dst) >= bound {
|
||||
dst = dst[0:cap(dst)]
|
||||
} else {
|
||||
dst = make([]byte, bound)
|
||||
}
|
||||
|
||||
written := int(C.ZSTD_decompressDCtx(
|
||||
c.dctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src))))
|
||||
|
||||
err := getError(written)
|
||||
if err == nil {
|
||||
return dst[:written], nil
|
||||
}
|
||||
if !IsDstSizeTooSmallError(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// We failed getting a dst buffer of correct size, use stream API
|
||||
r := NewReader(bytes.NewReader(src))
|
||||
defer r.Close()
|
||||
return ioutil.ReadAll(r)
|
||||
}
|
||||
|
||||
func finalizeCtx(c *ctx) {
|
||||
C.ZSTD_freeCCtx(c.cctx)
|
||||
C.ZSTD_freeDCtx(c.dctx)
|
||||
}
|
||||
342
vendor/github.com/DataDog/zstd/zstd_cwksp.h
generated
vendored
342
vendor/github.com/DataDog/zstd/zstd_cwksp.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -25,6 +24,16 @@ extern "C" {
|
||||
* Constants
|
||||
***************************************/
|
||||
|
||||
/* define "workspace is too large" as this number of times larger than needed */
|
||||
#define ZSTD_WORKSPACETOOLARGE_FACTOR 3
|
||||
|
||||
/* when workspace is continuously too large
|
||||
* during at least this number of times,
|
||||
* context's memory usage is considered wasteful,
|
||||
* because it's sized to handle a worst case scenario which rarely happens.
|
||||
* In which case, resize it down to free some memory */
|
||||
#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
|
||||
|
||||
/* Since the workspace is effectively its own little malloc implementation /
|
||||
* arena, when we run under ASAN, we should similarly insert redzones between
|
||||
* each internal element of the workspace, so ASAN will catch overruns that
|
||||
@@ -36,10 +45,6 @@ extern "C" {
|
||||
#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
|
||||
#endif
|
||||
|
||||
|
||||
/* Set our tables and aligneds to align by 64 bytes */
|
||||
#define ZSTD_CWKSP_ALIGNMENT_BYTES 64
|
||||
|
||||
/*-*************************************
|
||||
* Structures
|
||||
***************************************/
|
||||
@@ -49,16 +54,6 @@ typedef enum {
|
||||
ZSTD_cwksp_alloc_aligned
|
||||
} ZSTD_cwksp_alloc_phase_e;
|
||||
|
||||
/**
|
||||
* Used to describe whether the workspace is statically allocated (and will not
|
||||
* necessarily ever be freed), or if it's dynamically allocated and we can
|
||||
* expect a well-formed caller to free this.
|
||||
*/
|
||||
typedef enum {
|
||||
ZSTD_cwksp_dynamic_alloc,
|
||||
ZSTD_cwksp_static_alloc
|
||||
} ZSTD_cwksp_static_alloc_e;
|
||||
|
||||
/**
|
||||
* Zstd fits all its internal datastructures into a single continuous buffer,
|
||||
* so that it only needs to perform a single OS allocation (or so that a buffer
|
||||
@@ -107,7 +102,7 @@ typedef enum {
|
||||
*
|
||||
* - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict,
|
||||
* so that literally everything fits in a single buffer. Note: if present,
|
||||
* this must be the first object in the workspace, since ZSTD_customFree{CCtx,
|
||||
* this must be the first object in the workspace, since ZSTD_free{CCtx,
|
||||
* CDict}() rely on a pointer comparison to see whether one or two frees are
|
||||
* required.
|
||||
*
|
||||
@@ -122,11 +117,10 @@ typedef enum {
|
||||
* - Tables: these are any of several different datastructures (hash tables,
|
||||
* chain tables, binary trees) that all respect a common format: they are
|
||||
* uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
|
||||
* Their sizes depend on the cparams. These tables are 64-byte aligned.
|
||||
* Their sizes depend on the cparams.
|
||||
*
|
||||
* - Aligned: these buffers are used for various purposes that require 4 byte
|
||||
* alignment, but don't require any initialization before they're used. These
|
||||
* buffers are each aligned to 64 bytes.
|
||||
* alignment, but don't require any initialization before they're used.
|
||||
*
|
||||
* - Buffers: these buffers are used for various purposes that don't require
|
||||
* any alignment or initialization before they're used. This means they can
|
||||
@@ -139,7 +133,8 @@ typedef enum {
|
||||
*
|
||||
* 1. Objects
|
||||
* 2. Buffers
|
||||
* 3. Aligned/Tables
|
||||
* 3. Aligned
|
||||
* 4. Tables
|
||||
*
|
||||
* Attempts to reserve objects of different types out of order will fail.
|
||||
*/
|
||||
@@ -152,10 +147,9 @@ typedef struct {
|
||||
void* tableValidEnd;
|
||||
void* allocStart;
|
||||
|
||||
BYTE allocFailed;
|
||||
int allocFailed;
|
||||
int workspaceOversizedDuration;
|
||||
ZSTD_cwksp_alloc_phase_e phase;
|
||||
ZSTD_cwksp_static_alloc_e isStatic;
|
||||
} ZSTD_cwksp;
|
||||
|
||||
/*-*************************************
|
||||
@@ -192,68 +186,63 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
|
||||
* Since tables aren't currently redzoned, you don't need to call through this
|
||||
* to figure out how much space you need for the matchState tables. Everything
|
||||
* else is though.
|
||||
*
|
||||
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size().
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
|
||||
if (size == 0)
|
||||
return 0;
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
#else
|
||||
return size;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes.
|
||||
* Used to determine the number of bytes required for a given "aligned".
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
|
||||
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES));
|
||||
MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
|
||||
ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) {
|
||||
assert(phase >= ws->phase);
|
||||
if (phase > ws->phase) {
|
||||
if (ws->phase < ZSTD_cwksp_alloc_buffers &&
|
||||
phase >= ZSTD_cwksp_alloc_buffers) {
|
||||
ws->tableValidEnd = ws->objectEnd;
|
||||
}
|
||||
if (ws->phase < ZSTD_cwksp_alloc_aligned &&
|
||||
phase >= ZSTD_cwksp_alloc_aligned) {
|
||||
/* If unaligned allocations down from a too-large top have left us
|
||||
* unaligned, we need to realign our alloc ptr. Technically, this
|
||||
* can consume space that is unaccounted for in the neededSpace
|
||||
* calculation. However, I believe this can only happen when the
|
||||
* workspace is too large, and specifically when it is too large
|
||||
* by a larger margin than the space that will be consumed. */
|
||||
/* TODO: cleaner, compiler warning friendly way to do this??? */
|
||||
ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1));
|
||||
if (ws->allocStart < ws->tableValidEnd) {
|
||||
ws->tableValidEnd = ws->allocStart;
|
||||
}
|
||||
}
|
||||
ws->phase = phase;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the amount of additional space the cwksp must allocate
|
||||
* for internal purposes (currently only alignment).
|
||||
* Returns whether this object/buffer/etc was allocated in this workspace.
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
|
||||
/* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes
|
||||
* to align the beginning of tables section, as well as another n_2=[0, 63] bytes
|
||||
* to align the beginning of the aligned section.
|
||||
*
|
||||
* n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and
|
||||
* aligneds being sized in multiples of 64 bytes.
|
||||
*/
|
||||
size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES;
|
||||
return slackSpace;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return the number of additional bytes required to align a pointer to the given number of bytes.
|
||||
* alignBytes must be a power of two.
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) {
|
||||
size_t const alignBytesMask = alignBytes - 1;
|
||||
size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
|
||||
assert((alignBytes & alignBytesMask) == 0);
|
||||
assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
return bytes;
|
||||
MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) {
|
||||
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal function. Do not use directly.
|
||||
* Reserves the given number of bytes within the aligned/buffer segment of the wksp,
|
||||
* which counts from the end of the wksp (as opposed to the object/table segment).
|
||||
*
|
||||
* Returns a pointer to the beginning of that space.
|
||||
*/
|
||||
MEM_STATIC void*
|
||||
ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes)
|
||||
{
|
||||
void* const alloc = (BYTE*)ws->allocStart - bytes;
|
||||
void* const bottom = ws->tableEnd;
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_internal(
|
||||
ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) {
|
||||
void* alloc;
|
||||
void* bottom = ws->tableEnd;
|
||||
ZSTD_cwksp_internal_advance_phase(ws, phase);
|
||||
alloc = (BYTE *)ws->allocStart - bytes;
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* over-reserve space */
|
||||
alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
#endif
|
||||
|
||||
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
|
||||
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
@@ -263,95 +252,16 @@ ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes)
|
||||
ws->allocFailed = 1;
|
||||
return NULL;
|
||||
}
|
||||
/* the area is reserved from the end of wksp.
|
||||
* If it overlaps with tableValidEnd, it voids guarantees on values' range */
|
||||
if (alloc < ws->tableValidEnd) {
|
||||
ws->tableValidEnd = alloc;
|
||||
}
|
||||
ws->allocStart = alloc;
|
||||
return alloc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves the cwksp to the next phase, and does any necessary allocations.
|
||||
* cwksp initialization must necessarily go through each phase in order.
|
||||
* Returns a 0 on success, or zstd error
|
||||
*/
|
||||
MEM_STATIC size_t
|
||||
ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase)
|
||||
{
|
||||
assert(phase >= ws->phase);
|
||||
if (phase > ws->phase) {
|
||||
/* Going from allocating objects to allocating buffers */
|
||||
if (ws->phase < ZSTD_cwksp_alloc_buffers &&
|
||||
phase >= ZSTD_cwksp_alloc_buffers) {
|
||||
ws->tableValidEnd = ws->objectEnd;
|
||||
}
|
||||
|
||||
/* Going from allocating buffers to allocating aligneds/tables */
|
||||
if (ws->phase < ZSTD_cwksp_alloc_aligned &&
|
||||
phase >= ZSTD_cwksp_alloc_aligned) {
|
||||
{ /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */
|
||||
size_t const bytesToAlign =
|
||||
ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign);
|
||||
ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */
|
||||
RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign),
|
||||
memory_allocation, "aligned phase - alignment initial allocation failed!");
|
||||
}
|
||||
{ /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */
|
||||
void* const alloc = ws->objectEnd;
|
||||
size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
void* const objectEnd = (BYTE*)alloc + bytesToAlign;
|
||||
DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign);
|
||||
RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation,
|
||||
"table phase - alignment initial allocation failed!");
|
||||
ws->objectEnd = objectEnd;
|
||||
ws->tableEnd = objectEnd; /* table area starts being empty */
|
||||
if (ws->tableValidEnd < ws->tableEnd) {
|
||||
ws->tableValidEnd = ws->tableEnd;
|
||||
} } }
|
||||
ws->phase = phase;
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether this object/buffer/etc was allocated in this workspace.
|
||||
*/
|
||||
MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr)
|
||||
{
|
||||
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal function. Do not use directly.
|
||||
*/
|
||||
MEM_STATIC void*
|
||||
ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase)
|
||||
{
|
||||
void* alloc;
|
||||
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* over-reserve space */
|
||||
bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
#endif
|
||||
|
||||
alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes);
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
|
||||
* either size. */
|
||||
if (alloc) {
|
||||
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
}
|
||||
}
|
||||
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
#endif
|
||||
|
||||
return alloc;
|
||||
@@ -360,44 +270,33 @@ ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase
|
||||
/**
|
||||
* Reserves and returns unaligned memory.
|
||||
*/
|
||||
MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes)
|
||||
{
|
||||
MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) {
|
||||
return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
|
||||
* Reserves and returns memory sized on and aligned on sizeof(unsigned).
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
|
||||
{
|
||||
void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
|
||||
ZSTD_cwksp_alloc_aligned);
|
||||
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
|
||||
return ptr;
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) {
|
||||
assert((bytes & (sizeof(U32)-1)) == 0);
|
||||
return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned);
|
||||
}
|
||||
|
||||
/**
|
||||
* Aligned on 64 bytes. These buffers have the special property that
|
||||
* Aligned on sizeof(unsigned). These buffers have the special property that
|
||||
* their values remain constrained, allowing us to re-use them without
|
||||
* memset()-ing them.
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
|
||||
{
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
|
||||
const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
|
||||
void* alloc;
|
||||
void* end;
|
||||
void* top;
|
||||
|
||||
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
|
||||
return NULL;
|
||||
}
|
||||
alloc = ws->tableEnd;
|
||||
end = (BYTE *)alloc + bytes;
|
||||
top = ws->allocStart;
|
||||
void* alloc = ws->tableEnd;
|
||||
void* end = (BYTE *)alloc + bytes;
|
||||
void* top = ws->allocStart;
|
||||
|
||||
DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
|
||||
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
|
||||
assert((bytes & (sizeof(U32)-1)) == 0);
|
||||
ZSTD_cwksp_internal_advance_phase(ws, phase);
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
assert(end <= top);
|
||||
if (end > top) {
|
||||
@@ -407,41 +306,35 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
|
||||
}
|
||||
ws->tableEnd = end;
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
}
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
#endif
|
||||
|
||||
assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
|
||||
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
|
||||
return alloc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aligned on sizeof(void*).
|
||||
* Note : should happen only once, at workspace first initialization
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes)
|
||||
{
|
||||
size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
|
||||
size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
|
||||
void* alloc = ws->objectEnd;
|
||||
void* end = (BYTE*)alloc + roundedBytes;
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* over-reserve space */
|
||||
end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
#endif
|
||||
|
||||
DEBUGLOG(4,
|
||||
DEBUGLOG(5,
|
||||
"cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining",
|
||||
alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
|
||||
assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0);
|
||||
assert(bytes % ZSTD_ALIGNOF(void*) == 0);
|
||||
assert(((size_t)alloc & (sizeof(void*)-1)) == 0);
|
||||
assert((bytes & (sizeof(void*)-1)) == 0);
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
/* we must be in the first phase, no advance is possible */
|
||||
if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) {
|
||||
DEBUGLOG(3, "cwksp: object alloc failed!");
|
||||
DEBUGLOG(4, "cwksp: object alloc failed!");
|
||||
ws->allocFailed = 1;
|
||||
return NULL;
|
||||
}
|
||||
@@ -449,23 +342,20 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes)
|
||||
ws->tableEnd = end;
|
||||
ws->tableValidEnd = end;
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
|
||||
* either size. */
|
||||
alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
}
|
||||
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
#endif
|
||||
|
||||
return alloc;
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws)
|
||||
{
|
||||
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) {
|
||||
DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty");
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
/* To validate that the table re-use logic is sound, and that we don't
|
||||
* access table space that we haven't cleaned, we re-"poison" the table
|
||||
* space every time we mark it dirty. */
|
||||
@@ -500,7 +390,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
|
||||
assert(ws->tableValidEnd >= ws->objectEnd);
|
||||
assert(ws->tableValidEnd <= ws->allocStart);
|
||||
if (ws->tableValidEnd < ws->tableEnd) {
|
||||
ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd);
|
||||
memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd);
|
||||
}
|
||||
ZSTD_cwksp_mark_tables_clean(ws);
|
||||
}
|
||||
@@ -512,12 +402,8 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
|
||||
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
|
||||
DEBUGLOG(4, "cwksp: clearing tables!");
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* We don't do this when the workspace is statically allocated, because
|
||||
* when that is the case, we have no capability to hook into the end of the
|
||||
* workspace's lifecycle to unpoison the memory.
|
||||
*/
|
||||
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
{
|
||||
size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
|
||||
__asan_poison_memory_region(ws->objectEnd, size);
|
||||
}
|
||||
@@ -534,7 +420,7 @@ MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
|
||||
MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
|
||||
DEBUGLOG(4, "cwksp: clearing!");
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
/* To validate that the context re-use logic is sound, and that we don't
|
||||
* access stuff that this compression hasn't initialized, we re-"poison"
|
||||
* the workspace (or at least the non-static, non-table parts of it)
|
||||
@@ -545,12 +431,8 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* We don't do this when the workspace is statically allocated, because
|
||||
* when that is the case, we have no capability to hook into the end of the
|
||||
* workspace's lifecycle to unpoison the memory.
|
||||
*/
|
||||
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
{
|
||||
size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
|
||||
__asan_poison_memory_region(ws->objectEnd, size);
|
||||
}
|
||||
@@ -570,7 +452,7 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
|
||||
* Any existing values in the workspace are ignored (the previously managed
|
||||
* buffer, if present, must be separately freed).
|
||||
*/
|
||||
MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) {
|
||||
MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) {
|
||||
DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size);
|
||||
assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */
|
||||
ws->workspace = start;
|
||||
@@ -578,45 +460,39 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_c
|
||||
ws->objectEnd = ws->workspace;
|
||||
ws->tableValidEnd = ws->objectEnd;
|
||||
ws->phase = ZSTD_cwksp_alloc_objects;
|
||||
ws->isStatic = isStatic;
|
||||
ZSTD_cwksp_clear(ws);
|
||||
ws->workspaceOversizedDuration = 0;
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) {
|
||||
void* workspace = ZSTD_customMalloc(size, customMem);
|
||||
void* workspace = ZSTD_malloc(size, customMem);
|
||||
DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size);
|
||||
RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!");
|
||||
ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc);
|
||||
RETURN_ERROR_IF(workspace == NULL, memory_allocation);
|
||||
ZSTD_cwksp_init(ws, workspace, size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
|
||||
void *ptr = ws->workspace;
|
||||
DEBUGLOG(4, "cwksp: freeing workspace");
|
||||
ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp));
|
||||
ZSTD_customFree(ptr, customMem);
|
||||
memset(ws, 0, sizeof(ZSTD_cwksp));
|
||||
ZSTD_free(ptr, customMem);
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves the management of a workspace from one cwksp to another. The src cwksp
|
||||
* is left in an invalid state (src must be re-init()'ed before it's used again).
|
||||
* is left in an invalid state (src must be re-init()'ed before its used again).
|
||||
*/
|
||||
MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
|
||||
*dst = *src;
|
||||
ZSTD_memset(src, 0, sizeof(ZSTD_cwksp));
|
||||
memset(src, 0, sizeof(ZSTD_cwksp));
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
|
||||
return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) {
|
||||
return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace)
|
||||
+ (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart);
|
||||
}
|
||||
|
||||
MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
|
||||
return ws->allocFailed;
|
||||
}
|
||||
@@ -625,24 +501,6 @@ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
|
||||
* Functions Checking Free Space
|
||||
***************************************/
|
||||
|
||||
/* ZSTD_alignmentSpaceWithinBounds() :
|
||||
* Returns if the estimated space needed for a wksp is within an acceptable limit of the
|
||||
* actual amount of space used.
|
||||
*/
|
||||
MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws,
|
||||
size_t const estimatedSpace, int resizedWorkspace) {
|
||||
if (resizedWorkspace) {
|
||||
/* Resized/newly allocated wksp should have exact bounds */
|
||||
return ZSTD_cwksp_used(ws) == estimatedSpace;
|
||||
} else {
|
||||
/* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes
|
||||
* than estimatedSpace. See the comments in zstd_cwksp.h for details.
|
||||
*/
|
||||
return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) {
|
||||
return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd);
|
||||
}
|
||||
@@ -675,5 +533,3 @@ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_CWKSP_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
31
vendor/github.com/DataDog/zstd/zstd_ddict.c
generated
vendored
31
vendor/github.com/DataDog/zstd/zstd_ddict.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -15,7 +14,7 @@
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
|
||||
#include <string.h> /* memcpy, memmove, memset */
|
||||
#include "cpu.h" /* bmi2 */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
@@ -26,7 +25,7 @@
|
||||
#include "zstd_ddict.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||
#include "zstd_legacy.h"
|
||||
# include "zstd_legacy.h"
|
||||
#endif
|
||||
|
||||
|
||||
@@ -66,10 +65,6 @@ void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
|
||||
dctx->virtualStart = ddict->dictContent;
|
||||
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
|
||||
dctx->previousDstEnd = dctx->dictEnd;
|
||||
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
||||
dctx->dictContentBeginForFuzzing = dctx->prefixStart;
|
||||
dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
|
||||
#endif
|
||||
if (ddict->entropyPresent) {
|
||||
dctx->litEntropy = 1;
|
||||
dctx->fseEntropy = 1;
|
||||
@@ -112,7 +107,7 @@ ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
|
||||
/* load entropy tables */
|
||||
RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
|
||||
&ddict->entropy, ddict->dictContent, ddict->dictSize)),
|
||||
dictionary_corrupted, "");
|
||||
dictionary_corrupted);
|
||||
ddict->entropyPresent = 1;
|
||||
return 0;
|
||||
}
|
||||
@@ -128,17 +123,17 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
|
||||
ddict->dictContent = dict;
|
||||
if (!dict) dictSize = 0;
|
||||
} else {
|
||||
void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
|
||||
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
|
||||
ddict->dictBuffer = internalBuffer;
|
||||
ddict->dictContent = internalBuffer;
|
||||
if (!internalBuffer) return ERROR(memory_allocation);
|
||||
ZSTD_memcpy(internalBuffer, dict, dictSize);
|
||||
memcpy(internalBuffer, dict, dictSize);
|
||||
}
|
||||
ddict->dictSize = dictSize;
|
||||
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
||||
|
||||
/* parse dictionary content */
|
||||
FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
|
||||
FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -148,9 +143,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
||||
ZSTD_dictContentType_e dictContentType,
|
||||
ZSTD_customMem customMem)
|
||||
{
|
||||
if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
|
||||
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
||||
|
||||
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
|
||||
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
|
||||
if (ddict == NULL) return NULL;
|
||||
ddict->cMem = customMem;
|
||||
{ size_t const initResult = ZSTD_initDDict_internal(ddict,
|
||||
@@ -199,7 +194,7 @@ const ZSTD_DDict* ZSTD_initStaticDDict(
|
||||
if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
|
||||
if (sBufferSize < neededSpace) return NULL;
|
||||
if (dictLoadMethod == ZSTD_dlm_byCopy) {
|
||||
ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */
|
||||
memcpy(ddict+1, dict, dictSize); /* local copy */
|
||||
dict = ddict+1;
|
||||
}
|
||||
if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
|
||||
@@ -214,8 +209,8 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
|
||||
{
|
||||
if (ddict==NULL) return 0; /* support free on NULL */
|
||||
{ ZSTD_customMem const cMem = ddict->cMem;
|
||||
ZSTD_customFree(ddict->dictBuffer, cMem);
|
||||
ZSTD_customFree(ddict, cMem);
|
||||
ZSTD_free(ddict->dictBuffer, cMem);
|
||||
ZSTD_free(ddict, cMem);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -243,5 +238,3 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
|
||||
if (ddict==NULL) return 0;
|
||||
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
7
vendor/github.com/DataDog/zstd/zstd_ddict.h
generated
vendored
7
vendor/github.com/DataDog/zstd/zstd_ddict.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -16,7 +15,7 @@
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd.h" /* ZSTD_DDict, and several public functions */
|
||||
|
||||
|
||||
@@ -43,5 +42,3 @@ void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
||||
|
||||
#endif /* ZSTD_DDICT_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
916
vendor/github.com/DataDog/zstd/zstd_decompress.c
generated
vendored
916
vendor/github.com/DataDog/zstd/zstd_decompress.c
generated
vendored
File diff suppressed because it is too large
Load Diff
1382
vendor/github.com/DataDog/zstd/zstd_decompress_block.c
generated
vendored
1382
vendor/github.com/DataDog/zstd/zstd_decompress_block.c
generated
vendored
File diff suppressed because it is too large
Load Diff
22
vendor/github.com/DataDog/zstd/zstd_decompress_block.h
generated
vendored
22
vendor/github.com/DataDog/zstd/zstd_decompress_block.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -16,7 +15,7 @@
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd.h" /* DCtx, and some public functions */
|
||||
#include "zstd_internal.h" /* blockProperties_t, and some public functions */
|
||||
#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
|
||||
@@ -34,12 +33,6 @@
|
||||
*/
|
||||
|
||||
|
||||
/* Streaming state is used to inform allocation of the literal buffer */
|
||||
typedef enum {
|
||||
not_streaming = 0,
|
||||
is_streaming = 1
|
||||
} streaming_operation;
|
||||
|
||||
/* ZSTD_decompressBlock_internal() :
|
||||
* decompress block, starting at `src`,
|
||||
* into destination buffer `dst`.
|
||||
@@ -48,24 +41,19 @@ typedef enum {
|
||||
*/
|
||||
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize, const int frame, const streaming_operation streaming);
|
||||
const void* src, size_t srcSize, const int frame);
|
||||
|
||||
/* ZSTD_buildFSETable() :
|
||||
* generate FSE decoding table for one symbol (ll, ml or off)
|
||||
* this function must be called with valid parameters only
|
||||
* (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
|
||||
* in which case it cannot fail.
|
||||
* The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is
|
||||
* defined in zstd_decompress_internal.h.
|
||||
* Internal use only.
|
||||
*/
|
||||
void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
|
||||
const short* normalizedCounter, unsigned maxSymbolValue,
|
||||
const U32* baseValue, const U8* nbAdditionalBits,
|
||||
unsigned tableLog, void* wksp, size_t wkspSize,
|
||||
int bmi2);
|
||||
const U32* baseValue, const U32* nbAdditionalBits,
|
||||
unsigned tableLog);
|
||||
|
||||
|
||||
#endif /* ZSTD_DEC_BLOCK_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
82
vendor/github.com/DataDog/zstd/zstd_decompress_internal.h
generated
vendored
82
vendor/github.com/DataDog/zstd/zstd_decompress_internal.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -21,33 +20,33 @@
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include "mem.h" /* BYTE, U16, U32 */
|
||||
#include "zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */
|
||||
#include "zstd_internal.h" /* ZSTD_seqSymbol */
|
||||
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Constants
|
||||
*********************************************************/
|
||||
static UNUSED_ATTR const U32 LL_base[MaxLL+1] = {
|
||||
static const U32 LL_base[MaxLL+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 18, 20, 22, 24, 28, 32, 40,
|
||||
48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
|
||||
0x2000, 0x4000, 0x8000, 0x10000 };
|
||||
|
||||
static UNUSED_ATTR const U32 OF_base[MaxOff+1] = {
|
||||
static const U32 OF_base[MaxOff+1] = {
|
||||
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
|
||||
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
|
||||
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
|
||||
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
|
||||
|
||||
static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = {
|
||||
static const U32 OF_bits[MaxOff+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31 };
|
||||
|
||||
static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
|
||||
static const U32 ML_base[MaxML+1] = {
|
||||
3, 4, 5, 6, 7, 8, 9, 10,
|
||||
11, 12, 13, 14, 15, 16, 17, 18,
|
||||
19, 20, 21, 22, 23, 24, 25, 26,
|
||||
@@ -74,16 +73,12 @@ static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
|
||||
|
||||
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
|
||||
|
||||
#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
|
||||
#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
|
||||
|
||||
typedef struct {
|
||||
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
|
||||
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
|
||||
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
|
||||
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
|
||||
} ZSTD_entropyDTables_t;
|
||||
|
||||
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
||||
@@ -100,29 +95,6 @@ typedef enum {
|
||||
ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
|
||||
} ZSTD_dictUses_e;
|
||||
|
||||
/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
|
||||
typedef struct {
|
||||
const ZSTD_DDict** ddictPtrTable;
|
||||
size_t ddictPtrTableSize;
|
||||
size_t ddictPtrCount;
|
||||
} ZSTD_DDictHashSet;
|
||||
|
||||
#ifndef ZSTD_DECODER_INTERNAL_BUFFER
|
||||
# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16)
|
||||
#endif
|
||||
|
||||
#define ZSTD_LBMIN 64
|
||||
#define ZSTD_LBMAX (128 << 10)
|
||||
|
||||
/* extra buffer, compensates when dst is not large enough to store litBuffer */
|
||||
#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX)
|
||||
|
||||
typedef enum {
|
||||
ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */
|
||||
ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */
|
||||
ZSTD_split = 2 /* Split between litExtraBuffer and dst */
|
||||
} ZSTD_litLocation_e;
|
||||
|
||||
struct ZSTD_DCtx_s
|
||||
{
|
||||
const ZSTD_seqSymbol* LLTptr;
|
||||
@@ -137,7 +109,6 @@ struct ZSTD_DCtx_s
|
||||
const void* dictEnd; /* end of previous segment */
|
||||
size_t expected;
|
||||
ZSTD_frameHeader fParams;
|
||||
U64 processedCSize;
|
||||
U64 decodedSize;
|
||||
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
|
||||
ZSTD_dStage stage;
|
||||
@@ -146,16 +117,12 @@ struct ZSTD_DCtx_s
|
||||
XXH64_state_t xxhState;
|
||||
size_t headerSize;
|
||||
ZSTD_format_e format;
|
||||
ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */
|
||||
U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */
|
||||
const BYTE* litPtr;
|
||||
ZSTD_customMem customMem;
|
||||
size_t litSize;
|
||||
size_t rleSize;
|
||||
size_t staticSize;
|
||||
#if DYNAMIC_BMI2 != 0
|
||||
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
|
||||
#endif
|
||||
|
||||
/* dictionary */
|
||||
ZSTD_DDict* ddictLocal;
|
||||
@@ -163,8 +130,6 @@ struct ZSTD_DCtx_s
|
||||
U32 dictID;
|
||||
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
|
||||
ZSTD_dictUses_e dictUses;
|
||||
ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */
|
||||
ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
|
||||
|
||||
/* streaming */
|
||||
ZSTD_dStreamStage streamStage;
|
||||
@@ -177,44 +142,17 @@ struct ZSTD_DCtx_s
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
size_t lhSize;
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||
void* legacyContext;
|
||||
U32 previousLegacyVersion;
|
||||
U32 legacyVersion;
|
||||
#endif
|
||||
U32 hostageByte;
|
||||
int noForwardProgress;
|
||||
ZSTD_bufferMode_e outBufferMode;
|
||||
ZSTD_outBuffer expectedOutBuffer;
|
||||
|
||||
/* workspace */
|
||||
BYTE* litBuffer;
|
||||
const BYTE* litBufferEnd;
|
||||
ZSTD_litLocation_e litBufferLocation;
|
||||
BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
|
||||
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
|
||||
size_t oversizedDuration;
|
||||
|
||||
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
||||
void const* dictContentBeginForFuzzing;
|
||||
void const* dictContentEndForFuzzing;
|
||||
#endif
|
||||
|
||||
/* Tracing */
|
||||
#if ZSTD_TRACE
|
||||
ZSTD_TraceCtx traceCtx;
|
||||
#endif
|
||||
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
|
||||
|
||||
MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
|
||||
#if DYNAMIC_BMI2 != 0
|
||||
return dctx->bmi2;
|
||||
#else
|
||||
(void)dctx;
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*-*******************************************************
|
||||
* Shared internal functions
|
||||
@@ -222,7 +160,7 @@ MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
|
||||
|
||||
/*! ZSTD_loadDEntropy() :
|
||||
* dict : must point at beginning of a valid zstd dictionary.
|
||||
* @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
|
||||
* @return : size of entropy tables read */
|
||||
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
||||
const void* const dict, size_t const dictSize);
|
||||
|
||||
@@ -231,9 +169,7 @@ size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
||||
* If yes, do nothing (continue on current segment).
|
||||
* If not, classify previous segment as "external dictionary", and start a new segment.
|
||||
* This function cannot fail. */
|
||||
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
|
||||
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst);
|
||||
|
||||
|
||||
#endif /* ZSTD_DECOMPRESS_INTERNAL_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
114
vendor/github.com/DataDog/zstd/zstd_deps.h
generated
vendored
114
vendor/github.com/DataDog/zstd/zstd_deps.h
generated
vendored
@@ -1,114 +0,0 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* This file provides common libc dependencies that zstd requires.
|
||||
* The purpose is to allow replacing this file with a custom implementation
|
||||
* to compile zstd without libc support.
|
||||
*/
|
||||
|
||||
/* Need:
|
||||
* NULL
|
||||
* INT_MAX
|
||||
* UINT_MAX
|
||||
* ZSTD_memcpy()
|
||||
* ZSTD_memset()
|
||||
* ZSTD_memmove()
|
||||
*/
|
||||
#ifndef ZSTD_DEPS_COMMON
|
||||
#define ZSTD_DEPS_COMMON
|
||||
|
||||
#include <limits.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
#if defined(__GNUC__) && __GNUC__ >= 4
|
||||
# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
|
||||
# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
|
||||
# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
|
||||
#else
|
||||
# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
|
||||
# define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
|
||||
# define ZSTD_memset(p,v,l) memset((p),(v),(l))
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_DEPS_COMMON */
|
||||
|
||||
/* Need:
|
||||
* ZSTD_malloc()
|
||||
* ZSTD_free()
|
||||
* ZSTD_calloc()
|
||||
*/
|
||||
#ifdef ZSTD_DEPS_NEED_MALLOC
|
||||
#ifndef ZSTD_DEPS_MALLOC
|
||||
#define ZSTD_DEPS_MALLOC
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#define ZSTD_malloc(s) malloc(s)
|
||||
#define ZSTD_calloc(n,s) calloc((n), (s))
|
||||
#define ZSTD_free(p) free((p))
|
||||
|
||||
#endif /* ZSTD_DEPS_MALLOC */
|
||||
#endif /* ZSTD_DEPS_NEED_MALLOC */
|
||||
|
||||
/*
|
||||
* Provides 64-bit math support.
|
||||
* Need:
|
||||
* U64 ZSTD_div64(U64 dividend, U32 divisor)
|
||||
*/
|
||||
#ifdef ZSTD_DEPS_NEED_MATH64
|
||||
#ifndef ZSTD_DEPS_MATH64
|
||||
#define ZSTD_DEPS_MATH64
|
||||
|
||||
#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
|
||||
|
||||
#endif /* ZSTD_DEPS_MATH64 */
|
||||
#endif /* ZSTD_DEPS_NEED_MATH64 */
|
||||
|
||||
/* Need:
|
||||
* assert()
|
||||
*/
|
||||
#ifdef ZSTD_DEPS_NEED_ASSERT
|
||||
#ifndef ZSTD_DEPS_ASSERT
|
||||
#define ZSTD_DEPS_ASSERT
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#endif /* ZSTD_DEPS_ASSERT */
|
||||
#endif /* ZSTD_DEPS_NEED_ASSERT */
|
||||
|
||||
/* Need:
|
||||
* ZSTD_DEBUG_PRINT()
|
||||
*/
|
||||
#ifdef ZSTD_DEPS_NEED_IO
|
||||
#ifndef ZSTD_DEPS_IO
|
||||
#define ZSTD_DEPS_IO
|
||||
|
||||
#include <stdio.h>
|
||||
#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
#endif /* ZSTD_DEPS_IO */
|
||||
#endif /* ZSTD_DEPS_NEED_IO */
|
||||
|
||||
/* Only requested when <stdint.h> is known to be present.
|
||||
* Need:
|
||||
* intptr_t
|
||||
*/
|
||||
#ifdef ZSTD_DEPS_NEED_STDINT
|
||||
#ifndef ZSTD_DEPS_STDINT
|
||||
#define ZSTD_DEPS_STDINT
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#endif /* ZSTD_DEPS_STDINT */
|
||||
#endif /* ZSTD_DEPS_NEED_STDINT */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
465
vendor/github.com/DataDog/zstd/zstd_double_fast.c
generated
vendored
465
vendor/github.com/DataDog/zstd/zstd_double_fast.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -32,15 +31,15 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
* is empty.
|
||||
*/
|
||||
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
|
||||
U32 const curr = (U32)(ip - base);
|
||||
U32 const current = (U32)(ip - base);
|
||||
U32 i;
|
||||
for (i = 0; i < fastHashFillStep; ++i) {
|
||||
size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls);
|
||||
size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8);
|
||||
if (i == 0)
|
||||
hashSmall[smHash] = curr + i;
|
||||
hashSmall[smHash] = current + i;
|
||||
if (i == 0 || hashLarge[lgHash] == 0)
|
||||
hashLarge[lgHash] = curr + i;
|
||||
hashLarge[lgHash] = current + i;
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
if (dtlm == ZSTD_dtlm_fast)
|
||||
break;
|
||||
@@ -49,216 +48,10 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize, U32 const mls /* template */)
|
||||
{
|
||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
||||
U32* const hashLong = ms->hashTable;
|
||||
const U32 hBitsL = cParams->hashLog;
|
||||
U32* const hashSmall = ms->chainTable;
|
||||
const U32 hBitsS = cParams->chainLog;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
||||
/* presumes that, if there is a dictionary, it must be using Attach mode */
|
||||
const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
|
||||
const BYTE* const prefixLowest = base + prefixLowestIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
U32 offsetSaved = 0;
|
||||
|
||||
size_t mLength;
|
||||
U32 offset;
|
||||
U32 curr;
|
||||
|
||||
/* how many positions to search before increasing step size */
|
||||
const size_t kStepIncr = 1 << kSearchStrength;
|
||||
/* the position at which to increment the step size if no match is found */
|
||||
const BYTE* nextStep;
|
||||
size_t step; /* the current step size */
|
||||
|
||||
size_t hl0; /* the long hash at ip */
|
||||
size_t hl1; /* the long hash at ip1 */
|
||||
|
||||
U32 idxl0; /* the long match index for ip */
|
||||
U32 idxl1; /* the long match index for ip1 */
|
||||
|
||||
const BYTE* matchl0; /* the long match for ip */
|
||||
const BYTE* matchs0; /* the short match for ip */
|
||||
const BYTE* matchl1; /* the long match for ip1 */
|
||||
|
||||
const BYTE* ip = istart; /* the current position */
|
||||
const BYTE* ip1; /* the next position */
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
|
||||
|
||||
/* init */
|
||||
ip += ((ip - prefixLowest) == 0);
|
||||
{
|
||||
U32 const current = (U32)(ip - base);
|
||||
U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog);
|
||||
U32 const maxRep = current - windowLow;
|
||||
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
|
||||
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
|
||||
}
|
||||
|
||||
/* Outer Loop: one iteration per match found and stored */
|
||||
while (1) {
|
||||
step = 1;
|
||||
nextStep = ip + kStepIncr;
|
||||
ip1 = ip + step;
|
||||
|
||||
if (ip1 > ilimit) {
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
hl0 = ZSTD_hashPtr(ip, hBitsL, 8);
|
||||
idxl0 = hashLong[hl0];
|
||||
matchl0 = base + idxl0;
|
||||
|
||||
/* Inner Loop: one iteration per search / position */
|
||||
do {
|
||||
const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls);
|
||||
const U32 idxs0 = hashSmall[hs0];
|
||||
curr = (U32)(ip-base);
|
||||
matchs0 = base + idxs0;
|
||||
|
||||
hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */
|
||||
|
||||
/* check noDict repcode */
|
||||
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
|
||||
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
|
||||
goto _match_stored;
|
||||
}
|
||||
|
||||
hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
|
||||
|
||||
if (idxl0 > prefixLowestIndex) {
|
||||
/* check prefix long match */
|
||||
if (MEM_read64(matchl0) == MEM_read64(ip)) {
|
||||
mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
|
||||
offset = (U32)(ip-matchl0);
|
||||
while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
}
|
||||
|
||||
idxl1 = hashLong[hl1];
|
||||
matchl1 = base + idxl1;
|
||||
|
||||
if (idxs0 > prefixLowestIndex) {
|
||||
/* check prefix short match */
|
||||
if (MEM_read32(matchs0) == MEM_read32(ip)) {
|
||||
goto _search_next_long;
|
||||
}
|
||||
}
|
||||
|
||||
if (ip1 >= nextStep) {
|
||||
PREFETCH_L1(ip1 + 64);
|
||||
PREFETCH_L1(ip1 + 128);
|
||||
step++;
|
||||
nextStep += kStepIncr;
|
||||
}
|
||||
ip = ip1;
|
||||
ip1 += step;
|
||||
|
||||
hl0 = hl1;
|
||||
idxl0 = idxl1;
|
||||
matchl0 = matchl1;
|
||||
#if defined(__aarch64__)
|
||||
PREFETCH_L1(ip+256);
|
||||
#endif
|
||||
} while (ip1 <= ilimit);
|
||||
|
||||
_cleanup:
|
||||
/* save reps for next block */
|
||||
rep[0] = offset_1 ? offset_1 : offsetSaved;
|
||||
rep[1] = offset_2 ? offset_2 : offsetSaved;
|
||||
|
||||
/* Return the last literals size */
|
||||
return (size_t)(iend - anchor);
|
||||
|
||||
_search_next_long:
|
||||
|
||||
/* check prefix long +1 match */
|
||||
if (idxl1 > prefixLowestIndex) {
|
||||
if (MEM_read64(matchl1) == MEM_read64(ip1)) {
|
||||
ip = ip1;
|
||||
mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8;
|
||||
offset = (U32)(ip-matchl1);
|
||||
while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
}
|
||||
|
||||
/* if no long +1 match, explore the short match we found */
|
||||
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
|
||||
offset = (U32)(ip - matchs0);
|
||||
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */
|
||||
|
||||
/* fall-through */
|
||||
|
||||
_match_found: /* requires ip, offset, mLength */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
|
||||
if (step < 4) {
|
||||
/* It is unsafe to write this value back to the hashtable when ip1 is
|
||||
* greater than or equal to the new ip we will have after we're done
|
||||
* processing this match. Rather than perform that test directly
|
||||
* (ip1 >= ip + mLength), which costs speed in practice, we do a simpler
|
||||
* more predictable test. The minmatch even if we take a short match is
|
||||
* 4 bytes, so as long as step, the distance between ip and ip1
|
||||
* (initially) is less than 4, we know ip1 < new ip. */
|
||||
hashLong[hl1] = (U32)(ip1 - base);
|
||||
}
|
||||
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
|
||||
_match_stored:
|
||||
/* match found */
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
|
||||
if (ip <= ilimit) {
|
||||
/* Complementary insertion */
|
||||
/* done after iLimit test, as candidates could be > iend-8 */
|
||||
{ U32 const indexToInsert = curr+2;
|
||||
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
|
||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
|
||||
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
|
||||
hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
|
||||
}
|
||||
|
||||
/* check immediate repcode */
|
||||
while ( (ip <= ilimit)
|
||||
&& ( (offset_2>0)
|
||||
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
|
||||
/* store sequence */
|
||||
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
||||
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, rLength);
|
||||
ip += rLength;
|
||||
anchor = ip;
|
||||
continue; /* faster when present ... (?) */
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
size_t ZSTD_compressBlock_doubleFast_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize,
|
||||
U32 const mls /* template */)
|
||||
U32 const mls /* template */, ZSTD_dictMode_e const dictMode)
|
||||
{
|
||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
||||
U32* const hashLong = ms->hashTable;
|
||||
@@ -270,8 +63,10 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
||||
const U32 lowestValid = ms->window.dictLimit;
|
||||
const U32 maxDistance = 1U << cParams->windowLog;
|
||||
/* presumes that, if there is a dictionary, it must be using Attach mode */
|
||||
const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
|
||||
const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid;
|
||||
const BYTE* const prefixLowest = base + prefixLowestIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
||||
@@ -279,30 +74,52 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
U32 offsetSaved = 0;
|
||||
|
||||
const ZSTD_matchState_t* const dms = ms->dictMatchState;
|
||||
const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
|
||||
const U32* const dictHashLong = dms->hashTable;
|
||||
const U32* const dictHashSmall = dms->chainTable;
|
||||
const U32 dictStartIndex = dms->window.dictLimit;
|
||||
const BYTE* const dictBase = dms->window.base;
|
||||
const BYTE* const dictStart = dictBase + dictStartIndex;
|
||||
const BYTE* const dictEnd = dms->window.nextSrc;
|
||||
const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase);
|
||||
const U32 dictHBitsL = dictCParams->hashLog;
|
||||
const U32 dictHBitsS = dictCParams->chainLog;
|
||||
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
|
||||
const ZSTD_compressionParameters* const dictCParams =
|
||||
dictMode == ZSTD_dictMatchState ?
|
||||
&dms->cParams : NULL;
|
||||
const U32* const dictHashLong = dictMode == ZSTD_dictMatchState ?
|
||||
dms->hashTable : NULL;
|
||||
const U32* const dictHashSmall = dictMode == ZSTD_dictMatchState ?
|
||||
dms->chainTable : NULL;
|
||||
const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.dictLimit : 0;
|
||||
const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.base : NULL;
|
||||
const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ?
|
||||
dictBase + dictStartIndex : NULL;
|
||||
const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ?
|
||||
dms->window.nextSrc : NULL;
|
||||
const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
|
||||
prefixLowestIndex - (U32)(dictEnd - dictBase) :
|
||||
0;
|
||||
const U32 dictHBitsL = dictMode == ZSTD_dictMatchState ?
|
||||
dictCParams->hashLog : hBitsL;
|
||||
const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ?
|
||||
dictCParams->chainLog : hBitsS;
|
||||
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart);
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic");
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic");
|
||||
|
||||
assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
|
||||
|
||||
/* if a dictionary is attached, it must be within window range */
|
||||
assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex);
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
assert(lowestValid + maxDistance >= endIndex);
|
||||
}
|
||||
|
||||
/* init */
|
||||
ip += (dictAndPrefixLength == 0);
|
||||
|
||||
/* dictMatchState repCode checks don't currently handle repCode == 0
|
||||
* disabling. */
|
||||
assert(offset_1 <= dictAndPrefixLength);
|
||||
assert(offset_2 <= dictAndPrefixLength);
|
||||
if (dictMode == ZSTD_noDict) {
|
||||
U32 const maxRep = (U32)(ip - prefixLowest);
|
||||
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
|
||||
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
|
||||
}
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
/* dictMatchState repCode checks don't currently handle repCode == 0
|
||||
* disabling. */
|
||||
assert(offset_1 <= dictAndPrefixLength);
|
||||
assert(offset_2 <= dictAndPrefixLength);
|
||||
}
|
||||
|
||||
/* Main Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
|
||||
@@ -312,24 +129,35 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
|
||||
size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8);
|
||||
size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls);
|
||||
U32 const curr = (U32)(ip-base);
|
||||
U32 const current = (U32)(ip-base);
|
||||
U32 const matchIndexL = hashLong[h2];
|
||||
U32 matchIndexS = hashSmall[h];
|
||||
const BYTE* matchLong = base + matchIndexL;
|
||||
const BYTE* match = base + matchIndexS;
|
||||
const U32 repIndex = curr + 1 - offset_1;
|
||||
const BYTE* repMatch = (repIndex < prefixLowestIndex) ?
|
||||
const U32 repIndex = current + 1 - offset_1;
|
||||
const BYTE* repMatch = (dictMode == ZSTD_dictMatchState
|
||||
&& repIndex < prefixLowestIndex) ?
|
||||
dictBase + (repIndex - dictIndexDelta) :
|
||||
base + repIndex;
|
||||
hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
|
||||
hashLong[h2] = hashSmall[h] = current; /* update hash tables */
|
||||
|
||||
/* check repcode */
|
||||
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
|
||||
/* check dictMatchState repcode */
|
||||
if (dictMode == ZSTD_dictMatchState
|
||||
&& ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
|
||||
goto _match_stored;
|
||||
}
|
||||
|
||||
/* check noDict repcode */
|
||||
if ( dictMode == ZSTD_noDict
|
||||
&& ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
|
||||
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
|
||||
goto _match_stored;
|
||||
}
|
||||
|
||||
@@ -341,7 +169,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
} else {
|
||||
} else if (dictMode == ZSTD_dictMatchState) {
|
||||
/* check dictMatchState long match */
|
||||
U32 const dictMatchIndexL = dictHashLong[dictHL];
|
||||
const BYTE* dictMatchL = dictBase + dictMatchIndexL;
|
||||
@@ -349,7 +177,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
|
||||
if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) {
|
||||
mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8;
|
||||
offset = (U32)(curr - dictMatchIndexL - dictIndexDelta);
|
||||
offset = (U32)(current - dictMatchIndexL - dictIndexDelta);
|
||||
while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
} }
|
||||
@@ -359,7 +187,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
if (MEM_read32(match) == MEM_read32(ip)) {
|
||||
goto _search_next_long;
|
||||
}
|
||||
} else {
|
||||
} else if (dictMode == ZSTD_dictMatchState) {
|
||||
/* check dictMatchState short match */
|
||||
U32 const dictMatchIndexS = dictHashSmall[dictHS];
|
||||
match = dictBase + dictMatchIndexS;
|
||||
@@ -370,9 +198,6 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
} }
|
||||
|
||||
ip += ((ip-anchor) >> kSearchStrength) + 1;
|
||||
#if defined(__aarch64__)
|
||||
PREFETCH_L1(ip+256);
|
||||
#endif
|
||||
continue;
|
||||
|
||||
_search_next_long:
|
||||
@@ -381,7 +206,7 @@ _search_next_long:
|
||||
size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
|
||||
U32 const matchIndexL3 = hashLong[hl3];
|
||||
const BYTE* matchL3 = base + matchIndexL3;
|
||||
hashLong[hl3] = curr + 1;
|
||||
hashLong[hl3] = current + 1;
|
||||
|
||||
/* check prefix long +1 match */
|
||||
if (matchIndexL3 > prefixLowestIndex) {
|
||||
@@ -392,7 +217,7 @@ _search_next_long:
|
||||
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
} else {
|
||||
} else if (dictMode == ZSTD_dictMatchState) {
|
||||
/* check dict long +1 match */
|
||||
U32 const dictMatchIndexL3 = dictHashLong[dictHLNext];
|
||||
const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
|
||||
@@ -400,15 +225,15 @@ _search_next_long:
|
||||
if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
|
||||
mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8;
|
||||
ip++;
|
||||
offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta);
|
||||
offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta);
|
||||
while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
} } }
|
||||
|
||||
/* if no long +1 match, explore the short match we found */
|
||||
if (matchIndexS < prefixLowestIndex) {
|
||||
if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) {
|
||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4;
|
||||
offset = (U32)(curr - matchIndexS);
|
||||
offset = (U32)(current - matchIndexS);
|
||||
while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
} else {
|
||||
mLength = ZSTD_count(ip+4, match+4, iend) + 4;
|
||||
@@ -416,11 +241,13 @@ _search_next_long:
|
||||
while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
}
|
||||
|
||||
/* fall-through */
|
||||
|
||||
_match_found:
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
|
||||
_match_stored:
|
||||
/* match found */
|
||||
@@ -430,7 +257,7 @@ _match_stored:
|
||||
if (ip <= ilimit) {
|
||||
/* Complementary insertion */
|
||||
/* done after iLimit test, as candidates could be > iend-8 */
|
||||
{ U32 const indexToInsert = curr+2;
|
||||
{ U32 const indexToInsert = current+2;
|
||||
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
|
||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
|
||||
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
|
||||
@@ -438,27 +265,43 @@ _match_stored:
|
||||
}
|
||||
|
||||
/* check immediate repcode */
|
||||
while (ip <= ilimit) {
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
|
||||
dictBase + repIndex2 - dictIndexDelta :
|
||||
base + repIndex2;
|
||||
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
||||
ip += repLength2;
|
||||
if (dictMode == ZSTD_dictMatchState) {
|
||||
while (ip <= ilimit) {
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState
|
||||
&& repIndex2 < prefixLowestIndex ?
|
||||
dictBase - dictIndexDelta + repIndex2 :
|
||||
base + repIndex2;
|
||||
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH);
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
||||
ip += repLength2;
|
||||
anchor = ip;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
} }
|
||||
|
||||
if (dictMode == ZSTD_noDict) {
|
||||
while ( (ip <= ilimit)
|
||||
&& ( (offset_2>0)
|
||||
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
|
||||
/* store sequence */
|
||||
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
||||
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH);
|
||||
ip += rLength;
|
||||
anchor = ip;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue; /* faster when present ... (?) */
|
||||
} } }
|
||||
} /* while (ip < ilimit) */
|
||||
|
||||
/* save reps for next block */
|
||||
@@ -469,24 +312,6 @@ _match_stored:
|
||||
return (size_t)(iend - anchor);
|
||||
}
|
||||
|
||||
#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
|
||||
static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
|
||||
void const* src, size_t srcSize) \
|
||||
{ \
|
||||
return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
|
||||
}
|
||||
|
||||
ZSTD_GEN_DFAST_FN(noDict, 4)
|
||||
ZSTD_GEN_DFAST_FN(noDict, 5)
|
||||
ZSTD_GEN_DFAST_FN(noDict, 6)
|
||||
ZSTD_GEN_DFAST_FN(noDict, 7)
|
||||
|
||||
ZSTD_GEN_DFAST_FN(dictMatchState, 4)
|
||||
ZSTD_GEN_DFAST_FN(dictMatchState, 5)
|
||||
ZSTD_GEN_DFAST_FN(dictMatchState, 6)
|
||||
ZSTD_GEN_DFAST_FN(dictMatchState, 7)
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
@@ -497,13 +322,13 @@ size_t ZSTD_compressBlock_doubleFast(
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -517,13 +342,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -559,7 +384,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
|
||||
/* if extDict is invalidated due to maxDistance, switch to "regular" variant */
|
||||
if (prefixStartIndex == dictStartIndex)
|
||||
return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict);
|
||||
|
||||
/* Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
|
||||
@@ -573,31 +398,31 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* matchLong = matchLongBase + matchLongIndex;
|
||||
|
||||
const U32 curr = (U32)(ip-base);
|
||||
const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
|
||||
const U32 current = (U32)(ip-base);
|
||||
const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */
|
||||
const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
size_t mLength;
|
||||
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
|
||||
hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */
|
||||
|
||||
if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
|
||||
& (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
|
||||
& (repIndex > dictStartIndex))
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
|
||||
} else {
|
||||
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
|
||||
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
U32 offset;
|
||||
mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8;
|
||||
offset = curr - matchLongIndex;
|
||||
offset = current - matchLongIndex;
|
||||
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
|
||||
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
|
||||
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
|
||||
@@ -605,24 +430,24 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* match3 = match3Base + matchIndex3;
|
||||
U32 offset;
|
||||
hashLong[h3] = curr + 1;
|
||||
hashLong[h3] = current + 1;
|
||||
if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
|
||||
const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart;
|
||||
mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8;
|
||||
ip++;
|
||||
offset = curr+1 - matchIndex3;
|
||||
offset = current+1 - matchIndex3;
|
||||
while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
|
||||
} else {
|
||||
const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
|
||||
offset = curr - matchIndex;
|
||||
offset = current - matchIndex;
|
||||
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
}
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
|
||||
} else {
|
||||
ip += ((ip-anchor) >> kSearchStrength) + 1;
|
||||
@@ -636,7 +461,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
if (ip <= ilimit) {
|
||||
/* Complementary insertion */
|
||||
/* done after iLimit test, as candidates could be > iend-8 */
|
||||
{ U32 const indexToInsert = curr+2;
|
||||
{ U32 const indexToInsert = current+2;
|
||||
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
|
||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
|
||||
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
|
||||
@@ -649,12 +474,12 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
|
||||
& (offset_2 <= current2 - dictStartIndex))
|
||||
& (repIndex2 > dictStartIndex))
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH);
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
||||
ip += repLength2;
|
||||
@@ -672,10 +497,6 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
return (size_t)(iend - anchor);
|
||||
}
|
||||
|
||||
ZSTD_GEN_DFAST_FN(extDict, 4)
|
||||
ZSTD_GEN_DFAST_FN(extDict, 5)
|
||||
ZSTD_GEN_DFAST_FN(extDict, 6)
|
||||
ZSTD_GEN_DFAST_FN(extDict, 7)
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
@@ -686,14 +507,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
5
vendor/github.com/DataDog/zstd/zstd_double_fast.h
generated
vendored
5
vendor/github.com/DataDog/zstd/zstd_double_fast.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -37,5 +36,3 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_DOUBLE_FAST_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
7
vendor/github.com/DataDog/zstd/zstd_errors.h
generated
vendored
7
vendor/github.com/DataDog/zstd/zstd_errors.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -77,8 +76,6 @@ typedef enum {
|
||||
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
|
||||
ZSTD_error_frameIndex_tooLarge = 100,
|
||||
ZSTD_error_seekableIO = 102,
|
||||
ZSTD_error_dstBuffer_wrong = 104,
|
||||
ZSTD_error_srcBuffer_wrong = 105,
|
||||
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
|
||||
} ZSTD_ErrorCode;
|
||||
|
||||
@@ -94,5 +91,3 @@ ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Sa
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_ERRORS_H_398273423 */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
472
vendor/github.com/DataDog/zstd/zstd_fast.c
generated
vendored
472
vendor/github.com/DataDog/zstd/zstd_fast.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -30,308 +29,148 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
|
||||
* Insert the other positions if their hash entry is empty.
|
||||
*/
|
||||
for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
|
||||
U32 const curr = (U32)(ip - base);
|
||||
U32 const current = (U32)(ip - base);
|
||||
size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls);
|
||||
hashTable[hash0] = curr;
|
||||
hashTable[hash0] = current;
|
||||
if (dtlm == ZSTD_dtlm_fast) continue;
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
{ U32 p;
|
||||
for (p = 1; p < fastHashFillStep; ++p) {
|
||||
size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls);
|
||||
if (hashTable[hash] == 0) { /* not yet filled */
|
||||
hashTable[hash] = curr + p;
|
||||
hashTable[hash] = current + p;
|
||||
} } } }
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* If you squint hard enough (and ignore repcodes), the search operation at any
|
||||
* given position is broken into 4 stages:
|
||||
*
|
||||
* 1. Hash (map position to hash value via input read)
|
||||
* 2. Lookup (map hash val to index via hashtable read)
|
||||
* 3. Load (map index to value at that position via input read)
|
||||
* 4. Compare
|
||||
*
|
||||
* Each of these steps involves a memory read at an address which is computed
|
||||
* from the previous step. This means these steps must be sequenced and their
|
||||
* latencies are cumulative.
|
||||
*
|
||||
* Rather than do 1->2->3->4 sequentially for a single position before moving
|
||||
* onto the next, this implementation interleaves these operations across the
|
||||
* next few positions:
|
||||
*
|
||||
* R = Repcode Read & Compare
|
||||
* H = Hash
|
||||
* T = Table Lookup
|
||||
* M = Match Read & Compare
|
||||
*
|
||||
* Pos | Time -->
|
||||
* ----+-------------------
|
||||
* N | ... M
|
||||
* N+1 | ... TM
|
||||
* N+2 | R H T M
|
||||
* N+3 | H TM
|
||||
* N+4 | R H T M
|
||||
* N+5 | H ...
|
||||
* N+6 | R ...
|
||||
*
|
||||
* This is very much analogous to the pipelining of execution in a CPU. And just
|
||||
* like a CPU, we have to dump the pipeline when we find a match (i.e., take a
|
||||
* branch).
|
||||
*
|
||||
* When this happens, we throw away our current state, and do the following prep
|
||||
* to re-enter the loop:
|
||||
*
|
||||
* Pos | Time -->
|
||||
* ----+-------------------
|
||||
* N | H T
|
||||
* N+1 | H
|
||||
*
|
||||
* This is also the work we do at the beginning to enter the loop initially.
|
||||
*/
|
||||
FORCE_INLINE_TEMPLATE size_t
|
||||
ZSTD_compressBlock_fast_noDict_generic(
|
||||
ZSTD_compressBlock_fast_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize,
|
||||
U32 const mls, U32 const hasStep)
|
||||
U32 const mls)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32* const hashTable = ms->hashTable;
|
||||
U32 const hlog = cParams->hashLog;
|
||||
/* support stepSize of 0 */
|
||||
size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2;
|
||||
size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
/* We check ip0 (ip + 0) and ip1 (ip + 1) each loop */
|
||||
const BYTE* ip0 = istart;
|
||||
const BYTE* ip1;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
||||
const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
|
||||
const U32 maxDistance = 1U << cParams->windowLog;
|
||||
const U32 validStartIndex = ms->window.dictLimit;
|
||||
const U32 prefixStartIndex = (endIndex - validStartIndex > maxDistance) ? endIndex - maxDistance : validStartIndex;
|
||||
const BYTE* const prefixStart = base + prefixStartIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
||||
|
||||
const BYTE* anchor = istart;
|
||||
const BYTE* ip0 = istart;
|
||||
const BYTE* ip1;
|
||||
const BYTE* ip2;
|
||||
const BYTE* ip3;
|
||||
U32 current0;
|
||||
|
||||
U32 rep_offset1 = rep[0];
|
||||
U32 rep_offset2 = rep[1];
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
U32 offsetSaved = 0;
|
||||
|
||||
size_t hash0; /* hash for ip0 */
|
||||
size_t hash1; /* hash for ip1 */
|
||||
U32 idx; /* match idx for ip0 */
|
||||
U32 mval; /* src value at match idx */
|
||||
|
||||
U32 offcode;
|
||||
const BYTE* match0;
|
||||
size_t mLength;
|
||||
|
||||
/* ip0 and ip1 are always adjacent. The targetLength skipping and
|
||||
* uncompressibility acceleration is applied to every other position,
|
||||
* matching the behavior of #1562. step therefore represents the gap
|
||||
* between pairs of positions, from ip0 to ip2 or ip1 to ip3. */
|
||||
size_t step;
|
||||
const BYTE* nextStep;
|
||||
const size_t kStepIncr = (1 << (kSearchStrength - 1));
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
|
||||
ip0 += (ip0 == prefixStart);
|
||||
{ U32 const curr = (U32)(ip0 - base);
|
||||
U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog);
|
||||
U32 const maxRep = curr - windowLow;
|
||||
if (rep_offset2 > maxRep) offsetSaved = rep_offset2, rep_offset2 = 0;
|
||||
if (rep_offset1 > maxRep) offsetSaved = rep_offset1, rep_offset1 = 0;
|
||||
}
|
||||
|
||||
/* start each op */
|
||||
_start: /* Requires: ip0 */
|
||||
|
||||
step = stepSize;
|
||||
nextStep = ip0 + kStepIncr;
|
||||
|
||||
/* calculate positions, ip0 - anchor == 0, so we skip step calc */
|
||||
ip1 = ip0 + 1;
|
||||
ip2 = ip0 + step;
|
||||
ip3 = ip2 + 1;
|
||||
|
||||
if (ip3 >= ilimit) {
|
||||
goto _cleanup;
|
||||
{ U32 const maxRep = (U32)(ip0 - prefixStart);
|
||||
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
|
||||
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
|
||||
}
|
||||
|
||||
hash0 = ZSTD_hashPtr(ip0, hlog, mls);
|
||||
hash1 = ZSTD_hashPtr(ip1, hlog, mls);
|
||||
/* Main Search Loop */
|
||||
while (ip1 < ilimit) { /* < instead of <=, because check at ip0+2 */
|
||||
size_t mLength;
|
||||
BYTE const* ip2 = ip0 + 2;
|
||||
size_t const h0 = ZSTD_hashPtr(ip0, hlog, mls);
|
||||
U32 const val0 = MEM_read32(ip0);
|
||||
size_t const h1 = ZSTD_hashPtr(ip1, hlog, mls);
|
||||
U32 const val1 = MEM_read32(ip1);
|
||||
U32 const current0 = (U32)(ip0-base);
|
||||
U32 const current1 = (U32)(ip1-base);
|
||||
U32 const matchIndex0 = hashTable[h0];
|
||||
U32 const matchIndex1 = hashTable[h1];
|
||||
BYTE const* repMatch = ip2-offset_1;
|
||||
const BYTE* match0 = base + matchIndex0;
|
||||
const BYTE* match1 = base + matchIndex1;
|
||||
U32 offcode;
|
||||
hashTable[h0] = current0; /* update hash table */
|
||||
hashTable[h1] = current1; /* update hash table */
|
||||
|
||||
idx = hashTable[hash0];
|
||||
assert(ip0 + 1 == ip1);
|
||||
|
||||
do {
|
||||
/* load repcode match for ip[2]*/
|
||||
const U32 rval = MEM_read32(ip2 - rep_offset1);
|
||||
|
||||
/* write back hash table entry */
|
||||
current0 = (U32)(ip0 - base);
|
||||
hashTable[hash0] = current0;
|
||||
|
||||
/* check repcode at ip[2] */
|
||||
if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) {
|
||||
ip0 = ip2;
|
||||
match0 = ip0 - rep_offset1;
|
||||
mLength = ip0[-1] == match0[-1];
|
||||
ip0 -= mLength;
|
||||
match0 -= mLength;
|
||||
offcode = STORE_REPCODE_1;
|
||||
mLength += 4;
|
||||
if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) {
|
||||
mLength = ip2[-1] == repMatch[-1] ? 1 : 0;
|
||||
ip0 = ip2 - mLength;
|
||||
match0 = repMatch - mLength;
|
||||
offcode = 0;
|
||||
goto _match;
|
||||
}
|
||||
|
||||
/* load match for ip[0] */
|
||||
if (idx >= prefixStartIndex) {
|
||||
mval = MEM_read32(base + idx);
|
||||
} else {
|
||||
mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
|
||||
}
|
||||
|
||||
/* check match at ip[0] */
|
||||
if (MEM_read32(ip0) == mval) {
|
||||
/* found a match! */
|
||||
if ((matchIndex0 > prefixStartIndex) && MEM_read32(match0) == val0) {
|
||||
/* found a regular match */
|
||||
goto _offset;
|
||||
}
|
||||
|
||||
/* lookup ip[1] */
|
||||
idx = hashTable[hash1];
|
||||
|
||||
/* hash ip[2] */
|
||||
hash0 = hash1;
|
||||
hash1 = ZSTD_hashPtr(ip2, hlog, mls);
|
||||
|
||||
/* advance to next positions */
|
||||
ip0 = ip1;
|
||||
ip1 = ip2;
|
||||
ip2 = ip3;
|
||||
|
||||
/* write back hash table entry */
|
||||
current0 = (U32)(ip0 - base);
|
||||
hashTable[hash0] = current0;
|
||||
|
||||
/* load match for ip[0] */
|
||||
if (idx >= prefixStartIndex) {
|
||||
mval = MEM_read32(base + idx);
|
||||
} else {
|
||||
mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
|
||||
}
|
||||
|
||||
/* check match at ip[0] */
|
||||
if (MEM_read32(ip0) == mval) {
|
||||
/* found a match! */
|
||||
if ((matchIndex1 > prefixStartIndex) && MEM_read32(match1) == val1) {
|
||||
/* found a regular match after one literal */
|
||||
ip0 = ip1;
|
||||
match0 = match1;
|
||||
goto _offset;
|
||||
}
|
||||
|
||||
/* lookup ip[1] */
|
||||
idx = hashTable[hash1];
|
||||
|
||||
/* hash ip[2] */
|
||||
hash0 = hash1;
|
||||
hash1 = ZSTD_hashPtr(ip2, hlog, mls);
|
||||
|
||||
/* advance to next positions */
|
||||
ip0 = ip1;
|
||||
ip1 = ip2;
|
||||
ip2 = ip0 + step;
|
||||
ip3 = ip1 + step;
|
||||
|
||||
/* calculate step */
|
||||
if (ip2 >= nextStep) {
|
||||
step++;
|
||||
PREFETCH_L1(ip1 + 64);
|
||||
PREFETCH_L1(ip1 + 128);
|
||||
nextStep += kStepIncr;
|
||||
{ size_t const step = ((size_t)(ip0-anchor) >> (kSearchStrength - 1)) + stepSize;
|
||||
assert(step >= 2);
|
||||
ip0 += step;
|
||||
ip1 += step;
|
||||
continue;
|
||||
}
|
||||
} while (ip3 < ilimit);
|
||||
_offset: /* Requires: ip0, match0 */
|
||||
/* Compute the offset code */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = (U32)(ip0-match0);
|
||||
offcode = offset_1 + ZSTD_REP_MOVE;
|
||||
mLength = 0;
|
||||
/* Count the backwards match length */
|
||||
while (((ip0>anchor) & (match0>prefixStart))
|
||||
&& (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */
|
||||
|
||||
_cleanup:
|
||||
/* Note that there are probably still a couple positions we could search.
|
||||
* However, it seems to be a meaningful performance hit to try to search
|
||||
* them. So let's not. */
|
||||
_match: /* Requires: ip0, match0, offcode */
|
||||
/* Count the forward length */
|
||||
mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH);
|
||||
/* match found */
|
||||
ip0 += mLength;
|
||||
anchor = ip0;
|
||||
ip1 = ip0 + 1;
|
||||
|
||||
if (ip0 <= ilimit) {
|
||||
/* Fill Table */
|
||||
assert(base+current0+2 > istart); /* check base overflow */
|
||||
hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
|
||||
hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
|
||||
|
||||
while ( ((ip0 <= ilimit) & (offset_2>0)) /* offset_2==0 means offset_2 is invalidated */
|
||||
&& (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) {
|
||||
/* store sequence */
|
||||
size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4;
|
||||
{ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */
|
||||
hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
|
||||
ip0 += rLength;
|
||||
ip1 = ip0 + 1;
|
||||
ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH);
|
||||
anchor = ip0;
|
||||
continue; /* faster when present (confirmed on gcc-8) ... (?) */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* save reps for next block */
|
||||
rep[0] = rep_offset1 ? rep_offset1 : offsetSaved;
|
||||
rep[1] = rep_offset2 ? rep_offset2 : offsetSaved;
|
||||
rep[0] = offset_1 ? offset_1 : offsetSaved;
|
||||
rep[1] = offset_2 ? offset_2 : offsetSaved;
|
||||
|
||||
/* Return the last literals size */
|
||||
return (size_t)(iend - anchor);
|
||||
|
||||
_offset: /* Requires: ip0, idx */
|
||||
|
||||
/* Compute the offset code. */
|
||||
match0 = base + idx;
|
||||
rep_offset2 = rep_offset1;
|
||||
rep_offset1 = (U32)(ip0-match0);
|
||||
offcode = STORE_OFFSET(rep_offset1);
|
||||
mLength = 4;
|
||||
|
||||
/* Count the backwards match length. */
|
||||
while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) {
|
||||
ip0--;
|
||||
match0--;
|
||||
mLength++;
|
||||
}
|
||||
|
||||
_match: /* Requires: ip0, match0, offcode */
|
||||
|
||||
/* Count the forward length. */
|
||||
mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend);
|
||||
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
|
||||
|
||||
ip0 += mLength;
|
||||
anchor = ip0;
|
||||
|
||||
/* write next hash table entry */
|
||||
if (ip1 < ip0) {
|
||||
hashTable[hash1] = (U32)(ip1 - base);
|
||||
}
|
||||
|
||||
/* Fill table and check for immediate repcode. */
|
||||
if (ip0 <= ilimit) {
|
||||
/* Fill Table */
|
||||
assert(base+current0+2 > istart); /* check base overflow */
|
||||
hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
|
||||
hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
|
||||
|
||||
if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */
|
||||
while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) {
|
||||
/* store sequence */
|
||||
size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4;
|
||||
{ U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */
|
||||
hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
|
||||
ip0 += rLength;
|
||||
ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, STORE_REPCODE_1, rLength);
|
||||
anchor = ip0;
|
||||
continue; /* faster when present (confirmed on gcc-8) ... (?) */
|
||||
} } }
|
||||
|
||||
goto _start;
|
||||
}
|
||||
|
||||
#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \
|
||||
static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
|
||||
void const* src, size_t srcSize) \
|
||||
{ \
|
||||
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \
|
||||
}
|
||||
|
||||
ZSTD_GEN_FAST_FN(noDict, 4, 1)
|
||||
ZSTD_GEN_FAST_FN(noDict, 5, 1)
|
||||
ZSTD_GEN_FAST_FN(noDict, 6, 1)
|
||||
ZSTD_GEN_FAST_FN(noDict, 7, 1)
|
||||
|
||||
ZSTD_GEN_FAST_FN(noDict, 4, 0)
|
||||
ZSTD_GEN_FAST_FN(noDict, 5, 0)
|
||||
ZSTD_GEN_FAST_FN(noDict, 6, 0)
|
||||
ZSTD_GEN_FAST_FN(noDict, 7, 0)
|
||||
|
||||
size_t ZSTD_compressBlock_fast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
@@ -339,40 +178,24 @@ size_t ZSTD_compressBlock_fast(
|
||||
{
|
||||
U32 const mls = ms->cParams.minMatch;
|
||||
assert(ms->dictMatchState == NULL);
|
||||
if (ms->cParams.targetLength > 1) {
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize);
|
||||
}
|
||||
} else {
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize);
|
||||
}
|
||||
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7);
|
||||
}
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
|
||||
void const* src, size_t srcSize, U32 const mls)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32* const hashTable = ms->hashTable;
|
||||
@@ -408,9 +231,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
assert(endIndex - prefixStartIndex <= maxDistance);
|
||||
(void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */
|
||||
|
||||
(void)hasStep; /* not currently specialized on whether it's accelerated */
|
||||
|
||||
/* ensure there will be no underflow
|
||||
/* ensure there will be no no underflow
|
||||
* when translating a dict index into a local index */
|
||||
assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
|
||||
|
||||
@@ -426,21 +247,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
|
||||
size_t mLength;
|
||||
size_t const h = ZSTD_hashPtr(ip, hlog, mls);
|
||||
U32 const curr = (U32)(ip-base);
|
||||
U32 const current = (U32)(ip-base);
|
||||
U32 const matchIndex = hashTable[h];
|
||||
const BYTE* match = base + matchIndex;
|
||||
const U32 repIndex = curr + 1 - offset_1;
|
||||
const U32 repIndex = current + 1 - offset_1;
|
||||
const BYTE* repMatch = (repIndex < prefixStartIndex) ?
|
||||
dictBase + (repIndex - dictIndexDelta) :
|
||||
base + repIndex;
|
||||
hashTable[h] = curr; /* update hash table */
|
||||
hashTable[h] = current; /* update hash table */
|
||||
|
||||
if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH);
|
||||
} else if ( (matchIndex <= prefixStartIndex) ) {
|
||||
size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
|
||||
U32 const dictMatchIndex = dictHashTable[dictHash];
|
||||
@@ -452,7 +273,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
continue;
|
||||
} else {
|
||||
/* found a dict match */
|
||||
U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta);
|
||||
U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta);
|
||||
mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4;
|
||||
while (((ip>anchor) & (dictMatch>dictStart))
|
||||
&& (ip[-1] == dictMatch[-1])) {
|
||||
@@ -460,7 +281,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
} /* catch up */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
}
|
||||
} else if (MEM_read32(match) != MEM_read32(ip)) {
|
||||
/* it's not a match, and we're not going to check the dictionary */
|
||||
@@ -475,7 +296,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
&& (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
}
|
||||
|
||||
/* match found */
|
||||
@@ -484,8 +305,8 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
|
||||
if (ip <= ilimit) {
|
||||
/* Fill Table */
|
||||
assert(base+curr+2 > istart); /* check base overflow */
|
||||
hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */
|
||||
assert(base+current+2 > istart); /* check base overflow */
|
||||
hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; /* here because current+2 could be > iend-8 */
|
||||
hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
|
||||
|
||||
/* check immediate repcode */
|
||||
@@ -500,7 +321,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH);
|
||||
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
|
||||
ip += repLength2;
|
||||
anchor = ip;
|
||||
@@ -519,12 +340,6 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
|
||||
return (size_t)(iend - anchor);
|
||||
}
|
||||
|
||||
|
||||
ZSTD_GEN_FAST_FN(dictMatchState, 4, 0)
|
||||
ZSTD_GEN_FAST_FN(dictMatchState, 5, 0)
|
||||
ZSTD_GEN_FAST_FN(dictMatchState, 6, 0)
|
||||
ZSTD_GEN_FAST_FN(dictMatchState, 7, 0)
|
||||
|
||||
size_t ZSTD_compressBlock_fast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
@@ -535,20 +350,20 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
|
||||
void const* src, size_t srcSize, U32 const mls)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32* const hashTable = ms->hashTable;
|
||||
@@ -572,13 +387,11 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
|
||||
(void)hasStep; /* not currently specialized on whether it's accelerated */
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1);
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic");
|
||||
|
||||
/* switch to "regular" variant if extDict is invalidated due to maxDistance */
|
||||
if (prefixStartIndex == dictStartIndex)
|
||||
return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls);
|
||||
|
||||
/* Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
|
||||
@@ -586,20 +399,19 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
const U32 matchIndex = hashTable[h];
|
||||
const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* match = matchBase + matchIndex;
|
||||
const U32 curr = (U32)(ip-base);
|
||||
const U32 repIndex = curr + 1 - offset_1;
|
||||
const U32 current = (U32)(ip-base);
|
||||
const U32 repIndex = current + 1 - offset_1;
|
||||
const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
hashTable[h] = curr; /* update hash table */
|
||||
DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr);
|
||||
hashTable[h] = current; /* update hash table */
|
||||
assert(offset_1 <= current +1); /* check repIndex */
|
||||
|
||||
if ( ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */
|
||||
& (offset_1 <= curr+1 - dictStartIndex) ) /* note: we are searching at curr+1 */
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex))
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, rLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH);
|
||||
ip += rLength;
|
||||
anchor = ip;
|
||||
} else {
|
||||
@@ -611,30 +423,30 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
}
|
||||
{ const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
|
||||
const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
|
||||
U32 const offset = curr - matchIndex;
|
||||
U32 const offset = current - matchIndex;
|
||||
size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
|
||||
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
||||
offset_2 = offset_1; offset_1 = offset; /* update offset history */
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
} }
|
||||
|
||||
if (ip <= ilimit) {
|
||||
/* Fill Table */
|
||||
hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2;
|
||||
hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2;
|
||||
hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
|
||||
/* check immediate repcode */
|
||||
while (ip <= ilimit) {
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 <= curr - dictStartIndex)) /* intentional overflow */
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
{ U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, STORE_REPCODE_1, repLength2);
|
||||
ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH);
|
||||
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
|
||||
ip += repLength2;
|
||||
anchor = ip;
|
||||
@@ -651,10 +463,6 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
|
||||
return (size_t)(iend - anchor);
|
||||
}
|
||||
|
||||
ZSTD_GEN_FAST_FN(extDict, 4, 0)
|
||||
ZSTD_GEN_FAST_FN(extDict, 5, 0)
|
||||
ZSTD_GEN_FAST_FN(extDict, 6, 0)
|
||||
ZSTD_GEN_FAST_FN(extDict, 7, 0)
|
||||
|
||||
size_t ZSTD_compressBlock_fast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
@@ -665,14 +473,12 @@ size_t ZSTD_compressBlock_fast_extDict(
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
case 4 :
|
||||
return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
|
||||
case 5 :
|
||||
return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
|
||||
case 6 :
|
||||
return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
|
||||
case 7 :
|
||||
return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize);
|
||||
return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
5
vendor/github.com/DataDog/zstd/zstd_fast.h
generated
vendored
5
vendor/github.com/DataDog/zstd/zstd_fast.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -36,5 +35,3 @@ size_t ZSTD_compressBlock_fast_extDict(
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_FAST_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
360
vendor/github.com/DataDog/zstd/zstd_internal.h
generated
vendored
360
vendor/github.com/DataDog/zstd/zstd_internal.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -21,7 +20,6 @@
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "compiler.h"
|
||||
#include "cpu.h"
|
||||
#include "mem.h"
|
||||
#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
|
||||
#include "error_private.h"
|
||||
@@ -35,11 +33,6 @@
|
||||
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#endif
|
||||
#include "xxhash.h" /* XXH_reset, update, digest */
|
||||
#ifndef ZSTD_NO_TRACE
|
||||
# include "zstd_trace.h"
|
||||
#else
|
||||
# define ZSTD_TRACE 0
|
||||
#endif
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
@@ -59,7 +52,50 @@ extern "C" {
|
||||
#undef MAX
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
#define BOUNDED(min,val,max) (MAX(min,MIN(val,max)))
|
||||
|
||||
/**
|
||||
* Return the specified error if the condition evaluates to true.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
* In order to do that (particularly, printing the conditional that failed),
|
||||
* this can't just wrap RETURN_ERROR().
|
||||
*/
|
||||
#define RETURN_ERROR_IF(cond, err, ...) \
|
||||
if (cond) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
}
|
||||
|
||||
/**
|
||||
* Unconditionally return the specified error.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
*/
|
||||
#define RETURN_ERROR(err, ...) \
|
||||
do { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
} while(0);
|
||||
|
||||
/**
|
||||
* If the provided expression evaluates to an error code, returns that error code.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
*/
|
||||
#define FORWARD_IF_ERROR(err, ...) \
|
||||
do { \
|
||||
size_t const err_code = (err); \
|
||||
if (ERR_isError(err_code)) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return err_code; \
|
||||
} \
|
||||
} while(0);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@@ -68,7 +104,8 @@ extern "C" {
|
||||
#define ZSTD_OPT_NUM (1<<12)
|
||||
|
||||
#define ZSTD_REP_NUM 3 /* number of repcodes */
|
||||
static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
|
||||
static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
@@ -82,17 +119,15 @@ static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||
static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
#define ZSTD_FRAMEIDSIZE 4 /* magic number size */
|
||||
|
||||
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||
static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||
typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
|
||||
|
||||
#define ZSTD_FRAMECHECKSUMSIZE 4
|
||||
|
||||
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
||||
|
||||
@@ -115,88 +150,51 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
|
||||
#define OffFSELog 8
|
||||
#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
|
||||
|
||||
#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
|
||||
/* Each table cannot take more than #symbols * FSELog bits */
|
||||
#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8)
|
||||
|
||||
static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16
|
||||
};
|
||||
static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = {
|
||||
4, 3, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 3, 2, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1
|
||||
};
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16 };
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 3, 2, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1 };
|
||||
#define LL_DEFAULTNORMLOG 6 /* for static allocation */
|
||||
static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
|
||||
static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
|
||||
|
||||
static UNUSED_ATTR const U8 ML_bits[MaxML+1] = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16
|
||||
};
|
||||
static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = {
|
||||
1, 4, 3, 2, 2, 2, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1,-1,-1,
|
||||
-1,-1,-1,-1,-1
|
||||
};
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16 };
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1,-1,-1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
#define ML_DEFAULTNORMLOG 6 /* for static allocation */
|
||||
static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
|
||||
static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
|
||||
|
||||
static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = {
|
||||
1, 1, 1, 1, 1, 1, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1,-1
|
||||
};
|
||||
static const S16 OF_defaultNorm[DefaultMaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
#define OF_DEFAULTNORMLOG 5 /* for static allocation */
|
||||
static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
|
||||
static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Shared functions to include for inlining
|
||||
*********************************************/
|
||||
static void ZSTD_copy8(void* dst, const void* src) {
|
||||
#if defined(ZSTD_ARCH_ARM_NEON)
|
||||
vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
|
||||
#else
|
||||
ZSTD_memcpy(dst, src, 8);
|
||||
#endif
|
||||
}
|
||||
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
||||
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
||||
|
||||
/* Need to use memmove here since the literal buffer can now be located within
|
||||
the dst buffer. In circumstances where the op "catches up" to where the
|
||||
literal buffer is, there can be partial overlaps in this call on the final
|
||||
copy if the literal is being shifted by less than 16 bytes. */
|
||||
static void ZSTD_copy16(void* dst, const void* src) {
|
||||
#if defined(ZSTD_ARCH_ARM_NEON)
|
||||
vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
|
||||
#elif defined(ZSTD_ARCH_X86_SSE2)
|
||||
_mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
|
||||
#elif defined(__clang__)
|
||||
ZSTD_memmove(dst, src, 16);
|
||||
#else
|
||||
/* ZSTD_memmove is not inlined properly by gcc */
|
||||
BYTE copy16_buf[16];
|
||||
ZSTD_memcpy(copy16_buf, src, 16);
|
||||
ZSTD_memcpy(dst, copy16_buf, 16);
|
||||
#endif
|
||||
}
|
||||
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
||||
static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); }
|
||||
#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
|
||||
|
||||
#define WILDCOPY_OVERLENGTH 32
|
||||
@@ -209,13 +207,13 @@ typedef enum {
|
||||
} ZSTD_overlap_e;
|
||||
|
||||
/*! ZSTD_wildcopy() :
|
||||
* Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
|
||||
* Custom version of memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
|
||||
* @param ovtype controls the overlap detection
|
||||
* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
|
||||
* - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
|
||||
* The src buffer must be before the dst buffer.
|
||||
*/
|
||||
MEM_STATIC FORCE_INLINE_ATTR
|
||||
MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
|
||||
void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
|
||||
{
|
||||
ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
|
||||
@@ -223,6 +221,8 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + length;
|
||||
|
||||
assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN));
|
||||
|
||||
if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
|
||||
/* Handle short offset copies. */
|
||||
do {
|
||||
@@ -230,117 +230,47 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
|
||||
} while (op < oend);
|
||||
} else {
|
||||
assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
|
||||
/* Separate out the first COPY16() call because the copy length is
|
||||
/* Separate out the first two COPY16() calls because the copy length is
|
||||
* almost certain to be short, so the branches have different
|
||||
* probabilities. Since it is almost certain to be short, only do
|
||||
* one COPY16() in the first call. Then, do two calls per loop since
|
||||
* at that point it is more likely to have a high trip count.
|
||||
* probabilities.
|
||||
* On gcc-9 unrolling once is +1.6%, twice is +2%, thrice is +1.8%.
|
||||
* On clang-8 unrolling once is +1.4%, twice is +3.3%, thrice is +3%.
|
||||
*/
|
||||
#ifdef __aarch64__
|
||||
do {
|
||||
COPY16(op, ip);
|
||||
}
|
||||
while (op < oend);
|
||||
#else
|
||||
ZSTD_copy16(op, ip);
|
||||
if (16 >= length) return;
|
||||
op += 16;
|
||||
ip += 16;
|
||||
COPY16(op, ip);
|
||||
COPY16(op, ip);
|
||||
if (op >= oend) return;
|
||||
do {
|
||||
COPY16(op, ip);
|
||||
COPY16(op, ip);
|
||||
}
|
||||
while (op < oend);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const length = MIN(dstCapacity, srcSize);
|
||||
if (length > 0) {
|
||||
ZSTD_memcpy(dst, src, length);
|
||||
}
|
||||
return length;
|
||||
}
|
||||
|
||||
/* define "workspace is too large" as this number of times larger than needed */
|
||||
#define ZSTD_WORKSPACETOOLARGE_FACTOR 3
|
||||
|
||||
/* when workspace is continuously too large
|
||||
* during at least this number of times,
|
||||
* context's memory usage is considered wasteful,
|
||||
* because it's sized to handle a worst case scenario which rarely happens.
|
||||
* In which case, resize it down to free some memory */
|
||||
#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
|
||||
|
||||
/* Controls whether the input/output buffer is buffered or stable. */
|
||||
typedef enum {
|
||||
ZSTD_bm_buffered = 0, /* Buffer the input/output */
|
||||
ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
|
||||
} ZSTD_bufferMode_e;
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private declarations
|
||||
*********************************************/
|
||||
typedef struct seqDef_s {
|
||||
U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
|
||||
U32 offset;
|
||||
U16 litLength;
|
||||
U16 mlBase; /* mlBase == matchLength - MINMATCH */
|
||||
U16 matchLength;
|
||||
} seqDef;
|
||||
|
||||
/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
|
||||
typedef enum {
|
||||
ZSTD_llt_none = 0, /* no longLengthType */
|
||||
ZSTD_llt_literalLength = 1, /* represents a long literal */
|
||||
ZSTD_llt_matchLength = 2 /* represents a long match */
|
||||
} ZSTD_longLengthType_e;
|
||||
|
||||
typedef struct {
|
||||
seqDef* sequencesStart;
|
||||
seqDef* sequences; /* ptr to end of sequences */
|
||||
seqDef* sequences;
|
||||
BYTE* litStart;
|
||||
BYTE* lit; /* ptr to end of literals */
|
||||
BYTE* lit;
|
||||
BYTE* llCode;
|
||||
BYTE* mlCode;
|
||||
BYTE* ofCode;
|
||||
size_t maxNbSeq;
|
||||
size_t maxNbLit;
|
||||
|
||||
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
|
||||
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
|
||||
* the existing value of the litLength or matchLength by 0x10000.
|
||||
*/
|
||||
ZSTD_longLengthType_e longLengthType;
|
||||
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
|
||||
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
|
||||
U32 longLengthPos;
|
||||
} seqStore_t;
|
||||
|
||||
typedef struct {
|
||||
U32 litLength;
|
||||
U32 matchLength;
|
||||
} ZSTD_sequenceLength;
|
||||
|
||||
/**
|
||||
* Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
|
||||
* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
|
||||
*/
|
||||
MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
|
||||
{
|
||||
ZSTD_sequenceLength seqLen;
|
||||
seqLen.litLength = seq->litLength;
|
||||
seqLen.matchLength = seq->mlBase + MINMATCH;
|
||||
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
|
||||
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
|
||||
seqLen.litLength += 0xFFFF;
|
||||
}
|
||||
if (seqStore->longLengthType == ZSTD_llt_matchLength) {
|
||||
seqLen.matchLength += 0xFFFF;
|
||||
}
|
||||
}
|
||||
return seqLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Contains the compressed frame size and an upper-bound for the decompressed frame size.
|
||||
* Note: before using `compressedSize`, check for errors using ZSTD_isError().
|
||||
@@ -356,9 +286,9 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBu
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
|
||||
|
||||
/* custom memory allocation functions */
|
||||
void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
|
||||
void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
|
||||
void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
|
||||
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
|
||||
void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
|
||||
void ZSTD_free(void* ptr, ZSTD_customMem customMem);
|
||||
|
||||
|
||||
MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
|
||||
@@ -366,18 +296,9 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
# if STATIC_BMI2 == 1
|
||||
return _lzcnt_u32(val)^31;
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
unsigned long r=0;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return __builtin_clz (val) ^ 31;
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
@@ -395,63 +316,6 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the number of trailing zeros of a `size_t`.
|
||||
* Most compilers should support CTZ as a builtin. A backup
|
||||
* implementation is provided if the builtin isn't supported, but
|
||||
* it may not be terribly efficient.
|
||||
*/
|
||||
MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val)
|
||||
{
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
# if STATIC_BMI2
|
||||
return _tzcnt_u64(val);
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward64(&r, (U64)val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return __builtin_ctzll((U64)val);
|
||||
# else
|
||||
static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19,
|
||||
4, 25, 14, 28, 9, 34, 20, 56,
|
||||
5, 17, 26, 54, 15, 41, 29, 43,
|
||||
10, 31, 38, 35, 21, 45, 49, 57,
|
||||
63, 6, 12, 18, 24, 27, 33, 55,
|
||||
16, 53, 40, 42, 30, 37, 44, 48,
|
||||
62, 11, 23, 32, 52, 39, 36, 47,
|
||||
61, 22, 51, 46, 60, 50, 59, 58 };
|
||||
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, (U32)val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return __builtin_ctz((U32)val);
|
||||
# else
|
||||
static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3,
|
||||
30, 22, 20, 15, 25, 17, 4, 8,
|
||||
31, 27, 13, 23, 21, 19, 16, 7,
|
||||
26, 12, 18, 6, 11, 5, 10, 9 };
|
||||
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ZSTD_invalidateRepCodes() :
|
||||
* ensures next compression will not use repcodes from previous block.
|
||||
@@ -478,19 +342,9 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
||||
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/**
|
||||
* @returns true iff the CPU supports dynamic BMI2 dispatch.
|
||||
*/
|
||||
MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
|
||||
{
|
||||
ZSTD_cpuid_t cpuid = ZSTD_cpuid();
|
||||
return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
1448
vendor/github.com/DataDog/zstd/zstd_lazy.c
generated
vendored
1448
vendor/github.com/DataDog/zstd/zstd_lazy.c
generated
vendored
File diff suppressed because it is too large
Load Diff
63
vendor/github.com/DataDog/zstd/zstd_lazy.h
generated
vendored
63
vendor/github.com/DataDog/zstd/zstd_lazy.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -18,18 +17,7 @@ extern "C" {
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
|
||||
/**
|
||||
* Dedicated Dictionary Search Structure bucket log. In the
|
||||
* ZSTD_dedicatedDictSearch mode, the hashTable has
|
||||
* 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just
|
||||
* one.
|
||||
*/
|
||||
#define ZSTD_LAZY_DDSS_BUCKET_LOG 2
|
||||
|
||||
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
|
||||
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
|
||||
|
||||
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
|
||||
|
||||
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
|
||||
|
||||
@@ -45,15 +33,6 @@ size_t ZSTD_compressBlock_lazy(
|
||||
size_t ZSTD_compressBlock_greedy(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
@@ -67,34 +46,6 @@ size_t ZSTD_compressBlock_lazy_dictMatchState(
|
||||
size_t ZSTD_compressBlock_greedy_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_greedy_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
@@ -105,24 +56,12 @@ size_t ZSTD_compressBlock_lazy_extDict(
|
||||
size_t ZSTD_compressBlock_lazy2_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_extDict_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_extDict_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_extDict_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btlazy2_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_LAZY_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
558
vendor/github.com/DataDog/zstd/zstd_ldm.c
generated
vendored
558
vendor/github.com/DataDog/zstd/zstd_ldm.c
generated
vendored
@@ -1,137 +1,22 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "zstd_ldm.h"
|
||||
|
||||
#include "debug.h"
|
||||
#include "xxhash.h"
|
||||
#include "zstd_fast.h" /* ZSTD_fillHashTable() */
|
||||
#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
|
||||
#include "zstd_ldm_geartab.h"
|
||||
|
||||
#define LDM_BUCKET_SIZE_LOG 3
|
||||
#define LDM_MIN_MATCH_LENGTH 64
|
||||
#define LDM_HASH_RLOG 7
|
||||
|
||||
typedef struct {
|
||||
U64 rolling;
|
||||
U64 stopMask;
|
||||
} ldmRollingHashState_t;
|
||||
|
||||
/** ZSTD_ldm_gear_init():
|
||||
*
|
||||
* Initializes the rolling hash state such that it will honor the
|
||||
* settings in params. */
|
||||
static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params)
|
||||
{
|
||||
unsigned maxBitsInMask = MIN(params->minMatchLength, 64);
|
||||
unsigned hashRateLog = params->hashRateLog;
|
||||
|
||||
state->rolling = ~(U32)0;
|
||||
|
||||
/* The choice of the splitting criterion is subject to two conditions:
|
||||
* 1. it has to trigger on average every 2^(hashRateLog) bytes;
|
||||
* 2. ideally, it has to depend on a window of minMatchLength bytes.
|
||||
*
|
||||
* In the gear hash algorithm, bit n depends on the last n bytes;
|
||||
* so in order to obtain a good quality splitting criterion it is
|
||||
* preferable to use bits with high weight.
|
||||
*
|
||||
* To match condition 1 we use a mask with hashRateLog bits set
|
||||
* and, because of the previous remark, we make sure these bits
|
||||
* have the highest possible weight while still respecting
|
||||
* condition 2.
|
||||
*/
|
||||
if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) {
|
||||
state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog);
|
||||
} else {
|
||||
/* In this degenerate case we simply honor the hash rate. */
|
||||
state->stopMask = ((U64)1 << hashRateLog) - 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_gear_reset()
|
||||
* Feeds [data, data + minMatchLength) into the hash without registering any
|
||||
* splits. This effectively resets the hash state. This is used when skipping
|
||||
* over data, either at the beginning of a block, or skipping sections.
|
||||
*/
|
||||
static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state,
|
||||
BYTE const* data, size_t minMatchLength)
|
||||
{
|
||||
U64 hash = state->rolling;
|
||||
size_t n = 0;
|
||||
|
||||
#define GEAR_ITER_ONCE() do { \
|
||||
hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
|
||||
n += 1; \
|
||||
} while (0)
|
||||
while (n + 3 < minMatchLength) {
|
||||
GEAR_ITER_ONCE();
|
||||
GEAR_ITER_ONCE();
|
||||
GEAR_ITER_ONCE();
|
||||
GEAR_ITER_ONCE();
|
||||
}
|
||||
while (n < minMatchLength) {
|
||||
GEAR_ITER_ONCE();
|
||||
}
|
||||
#undef GEAR_ITER_ONCE
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_gear_feed():
|
||||
*
|
||||
* Registers in the splits array all the split points found in the first
|
||||
* size bytes following the data pointer. This function terminates when
|
||||
* either all the data has been processed or LDM_BATCH_SIZE splits are
|
||||
* present in the splits array.
|
||||
*
|
||||
* Precondition: The splits array must not be full.
|
||||
* Returns: The number of bytes processed. */
|
||||
static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state,
|
||||
BYTE const* data, size_t size,
|
||||
size_t* splits, unsigned* numSplits)
|
||||
{
|
||||
size_t n;
|
||||
U64 hash, mask;
|
||||
|
||||
hash = state->rolling;
|
||||
mask = state->stopMask;
|
||||
n = 0;
|
||||
|
||||
#define GEAR_ITER_ONCE() do { \
|
||||
hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
|
||||
n += 1; \
|
||||
if (UNLIKELY((hash & mask) == 0)) { \
|
||||
splits[*numSplits] = n; \
|
||||
*numSplits += 1; \
|
||||
if (*numSplits == LDM_BATCH_SIZE) \
|
||||
goto done; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
while (n + 3 < size) {
|
||||
GEAR_ITER_ONCE();
|
||||
GEAR_ITER_ONCE();
|
||||
GEAR_ITER_ONCE();
|
||||
GEAR_ITER_ONCE();
|
||||
}
|
||||
while (n < size) {
|
||||
GEAR_ITER_ONCE();
|
||||
}
|
||||
|
||||
#undef GEAR_ITER_ONCE
|
||||
|
||||
done:
|
||||
state->rolling = hash;
|
||||
return n;
|
||||
}
|
||||
#define LDM_HASH_CHAR_OFFSET 10
|
||||
|
||||
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
ZSTD_compressionParameters const* cParams)
|
||||
@@ -141,6 +26,13 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
|
||||
if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
|
||||
if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
|
||||
if (cParams->strategy >= ZSTD_btopt) {
|
||||
/* Get out of the way of the optimal parser */
|
||||
U32 const minMatch = MAX(cParams->targetLength, params->minMatchLength);
|
||||
assert(minMatch >= ZSTD_LDM_MINMATCH_MIN);
|
||||
assert(minMatch <= ZSTD_LDM_MINMATCH_MAX);
|
||||
params->minMatchLength = minMatch;
|
||||
}
|
||||
if (params->hashLog == 0) {
|
||||
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
|
||||
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
|
||||
@@ -160,12 +52,47 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params)
|
||||
size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
|
||||
size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize)
|
||||
+ ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t));
|
||||
return params.enableLdm == ZSTD_ps_enable ? totalSize : 0;
|
||||
return params.enableLdm ? totalSize : 0;
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
|
||||
{
|
||||
return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0;
|
||||
return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getSmallHash() :
|
||||
* numBits should be <= 32
|
||||
* If numBits==0, returns 0.
|
||||
* @return : the most significant numBits of value. */
|
||||
static U32 ZSTD_ldm_getSmallHash(U64 value, U32 numBits)
|
||||
{
|
||||
assert(numBits <= 32);
|
||||
return numBits == 0 ? 0 : (U32)(value >> (64 - numBits));
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getChecksum() :
|
||||
* numBitsToDiscard should be <= 32
|
||||
* @return : the next most significant 32 bits after numBitsToDiscard */
|
||||
static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard)
|
||||
{
|
||||
assert(numBitsToDiscard <= 32);
|
||||
return (hash >> (64 - 32 - numBitsToDiscard)) & 0xFFFFFFFF;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getTag() ;
|
||||
* Given the hash, returns the most significant numTagBits bits
|
||||
* after (32 + hbits) bits.
|
||||
*
|
||||
* If there are not enough bits remaining, return the last
|
||||
* numTagBits bits. */
|
||||
static U32 ZSTD_ldm_getTag(U64 hash, U32 hbits, U32 numTagBits)
|
||||
{
|
||||
assert(numTagBits < 32 && hbits <= 32);
|
||||
if (32 - hbits < numTagBits) {
|
||||
return hash & (((U32)1 << numTagBits) - 1);
|
||||
} else {
|
||||
return (hash >> (32 - hbits - numTagBits)) & (((U32)1 << numTagBits) - 1);
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getBucket() :
|
||||
@@ -182,12 +109,38 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
|
||||
size_t const hash, const ldmEntry_t entry,
|
||||
ldmParams_t const ldmParams)
|
||||
{
|
||||
BYTE* const pOffset = ldmState->bucketOffsets + hash;
|
||||
unsigned const offset = *pOffset;
|
||||
|
||||
*(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
|
||||
*pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
|
||||
BYTE* const bucketOffsets = ldmState->bucketOffsets;
|
||||
*(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry;
|
||||
bucketOffsets[hash]++;
|
||||
bucketOffsets[hash] &= ((U32)1 << ldmParams.bucketSizeLog) - 1;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_makeEntryAndInsertByTag() :
|
||||
*
|
||||
* Gets the small hash, checksum, and tag from the rollingHash.
|
||||
*
|
||||
* If the tag matches (1 << ldmParams.hashRateLog)-1, then
|
||||
* creates an ldmEntry from the offset, and inserts it into the hash table.
|
||||
*
|
||||
* hBits is the length of the small hash, which is the most significant hBits
|
||||
* of rollingHash. The checksum is the next 32 most significant bits, followed
|
||||
* by ldmParams.hashRateLog bits that make up the tag. */
|
||||
static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
|
||||
U64 const rollingHash,
|
||||
U32 const hBits,
|
||||
U32 const offset,
|
||||
ldmParams_t const ldmParams)
|
||||
{
|
||||
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
|
||||
U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
|
||||
if (tag == tagMask) {
|
||||
U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
|
||||
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
|
||||
ldmEntry_t entry;
|
||||
entry.offset = offset;
|
||||
entry.checksum = checksum;
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, entry, ldmParams);
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_countBackwardsMatch() :
|
||||
@@ -196,10 +149,10 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
|
||||
* We count only bytes where pMatch >= pBase and pIn >= pAnchor. */
|
||||
static size_t ZSTD_ldm_countBackwardsMatch(
|
||||
const BYTE* pIn, const BYTE* pAnchor,
|
||||
const BYTE* pMatch, const BYTE* pMatchBase)
|
||||
const BYTE* pMatch, const BYTE* pBase)
|
||||
{
|
||||
size_t matchLength = 0;
|
||||
while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) {
|
||||
while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
|
||||
pIn--;
|
||||
pMatch--;
|
||||
matchLength++;
|
||||
@@ -207,27 +160,6 @@ static size_t ZSTD_ldm_countBackwardsMatch(
|
||||
return matchLength;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_countBackwardsMatch_2segments() :
|
||||
* Returns the number of bytes that match backwards from pMatch,
|
||||
* even with the backwards match spanning 2 different segments.
|
||||
*
|
||||
* On reaching `pMatchBase`, start counting from mEnd */
|
||||
static size_t ZSTD_ldm_countBackwardsMatch_2segments(
|
||||
const BYTE* pIn, const BYTE* pAnchor,
|
||||
const BYTE* pMatch, const BYTE* pMatchBase,
|
||||
const BYTE* pExtDictStart, const BYTE* pExtDictEnd)
|
||||
{
|
||||
size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase);
|
||||
if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) {
|
||||
/* If backwards match is entirely in the extDict or prefix, immediately return */
|
||||
return matchLength;
|
||||
}
|
||||
DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength);
|
||||
matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart);
|
||||
DEBUGLOG(7, "final backwards match length = %zu", matchLength);
|
||||
return matchLength;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_fillFastTables() :
|
||||
*
|
||||
* Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies.
|
||||
@@ -265,43 +197,30 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ZSTD_ldm_fillHashTable(
|
||||
ldmState_t* ldmState, const BYTE* ip,
|
||||
const BYTE* iend, ldmParams_t const* params)
|
||||
/** ZSTD_ldm_fillLdmHashTable() :
|
||||
*
|
||||
* Fills hashTable from (lastHashed + 1) to iend (non-inclusive).
|
||||
* lastHash is the rolling hash that corresponds to lastHashed.
|
||||
*
|
||||
* Returns the rolling hash corresponding to position iend-1. */
|
||||
static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
|
||||
U64 lastHash, const BYTE* lastHashed,
|
||||
const BYTE* iend, const BYTE* base,
|
||||
U32 hBits, ldmParams_t const ldmParams)
|
||||
{
|
||||
U32 const minMatchLength = params->minMatchLength;
|
||||
U32 const hBits = params->hashLog - params->bucketSizeLog;
|
||||
BYTE const* const base = ldmState->window.base;
|
||||
BYTE const* const istart = ip;
|
||||
ldmRollingHashState_t hashState;
|
||||
size_t* const splits = ldmState->splitIndices;
|
||||
unsigned numSplits;
|
||||
U64 rollingHash = lastHash;
|
||||
const BYTE* cur = lastHashed + 1;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_ldm_fillHashTable");
|
||||
|
||||
ZSTD_ldm_gear_init(&hashState, params);
|
||||
while (ip < iend) {
|
||||
size_t hashed;
|
||||
unsigned n;
|
||||
|
||||
numSplits = 0;
|
||||
hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
|
||||
|
||||
for (n = 0; n < numSplits; n++) {
|
||||
if (ip + splits[n] >= istart + minMatchLength) {
|
||||
BYTE const* const split = ip + splits[n] - minMatchLength;
|
||||
U64 const xxhash = XXH64(split, minMatchLength, 0);
|
||||
U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
|
||||
ldmEntry_t entry;
|
||||
|
||||
entry.offset = (U32)(split - base);
|
||||
entry.checksum = (U32)(xxhash >> 32);
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
|
||||
}
|
||||
}
|
||||
|
||||
ip += hashed;
|
||||
while (cur < iend) {
|
||||
rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1],
|
||||
cur[ldmParams.minMatchLength-1],
|
||||
state->hashPower);
|
||||
ZSTD_ldm_makeEntryAndInsertByTag(state,
|
||||
rollingHash, hBits,
|
||||
(U32)(cur - base), ldmParams);
|
||||
++cur;
|
||||
}
|
||||
return rollingHash;
|
||||
}
|
||||
|
||||
|
||||
@@ -312,10 +231,10 @@ void ZSTD_ldm_fillHashTable(
|
||||
* (after a long match, only update tables a limited amount). */
|
||||
static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
|
||||
{
|
||||
U32 const curr = (U32)(anchor - ms->window.base);
|
||||
if (curr > ms->nextToUpdate + 1024) {
|
||||
U32 const current = (U32)(anchor - ms->window.base);
|
||||
if (current > ms->nextToUpdate + 1024) {
|
||||
ms->nextToUpdate =
|
||||
curr - MIN(512, curr - ms->nextToUpdate - 1024);
|
||||
current - MIN(512, current - ms->nextToUpdate - 1024);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,8 +245,11 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
/* LDM parameters */
|
||||
int const extDict = ZSTD_window_hasExtDict(ldmState->window);
|
||||
U32 const minMatchLength = params->minMatchLength;
|
||||
U32 const entsPerBucket = 1U << params->bucketSizeLog;
|
||||
U64 const hashPower = ldmState->hashPower;
|
||||
U32 const hBits = params->hashLog - params->bucketSizeLog;
|
||||
U32 const ldmBucketSize = 1U << params->bucketSizeLog;
|
||||
U32 const hashRateLog = params->hashRateLog;
|
||||
U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
|
||||
/* Prefix and extDict parameters */
|
||||
U32 const dictLimit = ldmState->window.dictLimit;
|
||||
U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
|
||||
@@ -339,69 +261,45 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
/* Input bounds */
|
||||
BYTE const* const istart = (BYTE const*)src;
|
||||
BYTE const* const iend = istart + srcSize;
|
||||
BYTE const* const ilimit = iend - HASH_READ_SIZE;
|
||||
BYTE const* const ilimit = iend - MAX(minMatchLength, HASH_READ_SIZE);
|
||||
/* Input positions */
|
||||
BYTE const* anchor = istart;
|
||||
BYTE const* ip = istart;
|
||||
/* Rolling hash state */
|
||||
ldmRollingHashState_t hashState;
|
||||
/* Arrays for staged-processing */
|
||||
size_t* const splits = ldmState->splitIndices;
|
||||
ldmMatchCandidate_t* const candidates = ldmState->matchCandidates;
|
||||
unsigned numSplits;
|
||||
/* Rolling hash */
|
||||
BYTE const* lastHashed = NULL;
|
||||
U64 rollingHash = 0;
|
||||
|
||||
if (srcSize < minMatchLength)
|
||||
return iend - anchor;
|
||||
while (ip <= ilimit) {
|
||||
size_t mLength;
|
||||
U32 const current = (U32)(ip - base);
|
||||
size_t forwardMatchLength = 0, backwardMatchLength = 0;
|
||||
ldmEntry_t* bestEntry = NULL;
|
||||
if (ip != istart) {
|
||||
rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
|
||||
lastHashed[minMatchLength],
|
||||
hashPower);
|
||||
} else {
|
||||
rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength);
|
||||
}
|
||||
lastHashed = ip;
|
||||
|
||||
/* Initialize the rolling hash state with the first minMatchLength bytes */
|
||||
ZSTD_ldm_gear_init(&hashState, params);
|
||||
ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength);
|
||||
ip += minMatchLength;
|
||||
|
||||
while (ip < ilimit) {
|
||||
size_t hashed;
|
||||
unsigned n;
|
||||
|
||||
numSplits = 0;
|
||||
hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip,
|
||||
splits, &numSplits);
|
||||
|
||||
for (n = 0; n < numSplits; n++) {
|
||||
BYTE const* const split = ip + splits[n] - minMatchLength;
|
||||
U64 const xxhash = XXH64(split, minMatchLength, 0);
|
||||
U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
|
||||
|
||||
candidates[n].split = split;
|
||||
candidates[n].hash = hash;
|
||||
candidates[n].checksum = (U32)(xxhash >> 32);
|
||||
candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
|
||||
PREFETCH_L1(candidates[n].bucket);
|
||||
/* Do not insert and do not look for a match */
|
||||
if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
|
||||
ip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (n = 0; n < numSplits; n++) {
|
||||
size_t forwardMatchLength = 0, backwardMatchLength = 0,
|
||||
bestMatchLength = 0, mLength;
|
||||
U32 offset;
|
||||
BYTE const* const split = candidates[n].split;
|
||||
U32 const checksum = candidates[n].checksum;
|
||||
U32 const hash = candidates[n].hash;
|
||||
ldmEntry_t* const bucket = candidates[n].bucket;
|
||||
ldmEntry_t const* cur;
|
||||
ldmEntry_t const* bestEntry = NULL;
|
||||
ldmEntry_t newEntry;
|
||||
/* Get the best entry and compute the match lengths */
|
||||
{
|
||||
ldmEntry_t* const bucket =
|
||||
ZSTD_ldm_getBucket(ldmState,
|
||||
ZSTD_ldm_getSmallHash(rollingHash, hBits),
|
||||
*params);
|
||||
ldmEntry_t* cur;
|
||||
size_t bestMatchLength = 0;
|
||||
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
|
||||
|
||||
newEntry.offset = (U32)(split - base);
|
||||
newEntry.checksum = checksum;
|
||||
|
||||
/* If a split point would generate a sequence overlapping with
|
||||
* the previous one, we merely register it in the hash table and
|
||||
* move on */
|
||||
if (split < anchor) {
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (cur = bucket; cur < bucket + entsPerBucket; cur++) {
|
||||
for (cur = bucket; cur < bucket + ldmBucketSize; ++cur) {
|
||||
size_t curForwardMatchLength, curBackwardMatchLength,
|
||||
curTotalMatchLength;
|
||||
if (cur->checksum != checksum || cur->offset <= lowestIndex) {
|
||||
@@ -415,23 +313,30 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
cur->offset < dictLimit ? dictEnd : iend;
|
||||
BYTE const* const lowMatchPtr =
|
||||
cur->offset < dictLimit ? dictStart : lowPrefixPtr;
|
||||
curForwardMatchLength =
|
||||
ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr);
|
||||
if (curForwardMatchLength < minMatchLength) {
|
||||
continue;
|
||||
}
|
||||
curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments(
|
||||
split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd);
|
||||
} else { /* !extDict */
|
||||
BYTE const* const pMatch = base + cur->offset;
|
||||
curForwardMatchLength = ZSTD_count(split, pMatch, iend);
|
||||
|
||||
curForwardMatchLength = ZSTD_count_2segments(
|
||||
ip, pMatch, iend,
|
||||
matchEnd, lowPrefixPtr);
|
||||
if (curForwardMatchLength < minMatchLength) {
|
||||
continue;
|
||||
}
|
||||
curBackwardMatchLength =
|
||||
ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr);
|
||||
ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch,
|
||||
lowMatchPtr);
|
||||
curTotalMatchLength = curForwardMatchLength +
|
||||
curBackwardMatchLength;
|
||||
} else { /* !extDict */
|
||||
BYTE const* const pMatch = base + cur->offset;
|
||||
curForwardMatchLength = ZSTD_count(ip, pMatch, iend);
|
||||
if (curForwardMatchLength < minMatchLength) {
|
||||
continue;
|
||||
}
|
||||
curBackwardMatchLength =
|
||||
ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch,
|
||||
lowPrefixPtr);
|
||||
curTotalMatchLength = curForwardMatchLength +
|
||||
curBackwardMatchLength;
|
||||
}
|
||||
curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength;
|
||||
|
||||
if (curTotalMatchLength > bestMatchLength) {
|
||||
bestMatchLength = curTotalMatchLength;
|
||||
@@ -440,54 +345,57 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
bestEntry = cur;
|
||||
}
|
||||
}
|
||||
|
||||
/* No match found -- insert an entry into the hash table
|
||||
* and process the next candidate match */
|
||||
if (bestEntry == NULL) {
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Match found */
|
||||
offset = (U32)(split - base) - bestEntry->offset;
|
||||
mLength = forwardMatchLength + backwardMatchLength;
|
||||
{
|
||||
rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
|
||||
|
||||
/* Out of sequence storage */
|
||||
if (rawSeqStore->size == rawSeqStore->capacity)
|
||||
return ERROR(dstSize_tooSmall);
|
||||
seq->litLength = (U32)(split - backwardMatchLength - anchor);
|
||||
seq->matchLength = (U32)mLength;
|
||||
seq->offset = offset;
|
||||
rawSeqStore->size++;
|
||||
}
|
||||
|
||||
/* Insert the current entry into the hash table --- it must be
|
||||
* done after the previous block to avoid clobbering bestEntry */
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
|
||||
|
||||
anchor = split + forwardMatchLength;
|
||||
|
||||
/* If we find a match that ends after the data that we've hashed
|
||||
* then we have a repeating, overlapping, pattern. E.g. all zeros.
|
||||
* If one repetition of the pattern matches our `stopMask` then all
|
||||
* repetitions will. We don't need to insert them all into out table,
|
||||
* only the first one. So skip over overlapping matches.
|
||||
* This is a major speed boost (20x) for compressing a single byte
|
||||
* repeated, when that byte ends up in the table.
|
||||
*/
|
||||
if (anchor > ip + hashed) {
|
||||
ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength);
|
||||
/* Continue the outer loop at anchor (ip + hashed == anchor). */
|
||||
ip = anchor - hashed;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ip += hashed;
|
||||
}
|
||||
/* No match found -- continue searching */
|
||||
if (bestEntry == NULL) {
|
||||
ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash,
|
||||
hBits, current,
|
||||
*params);
|
||||
ip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Match found */
|
||||
mLength = forwardMatchLength + backwardMatchLength;
|
||||
ip -= backwardMatchLength;
|
||||
|
||||
{
|
||||
/* Store the sequence:
|
||||
* ip = current - backwardMatchLength
|
||||
* The match is at (bestEntry->offset - backwardMatchLength)
|
||||
*/
|
||||
U32 const matchIndex = bestEntry->offset;
|
||||
U32 const offset = current - matchIndex;
|
||||
rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
|
||||
|
||||
/* Out of sequence storage */
|
||||
if (rawSeqStore->size == rawSeqStore->capacity)
|
||||
return ERROR(dstSize_tooSmall);
|
||||
seq->litLength = (U32)(ip - anchor);
|
||||
seq->matchLength = (U32)mLength;
|
||||
seq->offset = offset;
|
||||
rawSeqStore->size++;
|
||||
}
|
||||
|
||||
/* Insert the current entry into the hash table */
|
||||
ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits,
|
||||
(U32)(lastHashed - base),
|
||||
*params);
|
||||
|
||||
assert(ip + backwardMatchLength == lastHashed);
|
||||
|
||||
/* Fill the hash table from lastHashed+1 to ip+mLength*/
|
||||
/* Heuristic: don't need to fill the entire table at end of block */
|
||||
if (ip + mLength <= ilimit) {
|
||||
rollingHash = ZSTD_ldm_fillLdmHashTable(
|
||||
ldmState, rollingHash, lastHashed,
|
||||
ip + mLength, base, hBits, *params);
|
||||
lastHashed = ip + mLength - 1;
|
||||
}
|
||||
ip += mLength;
|
||||
anchor = ip;
|
||||
}
|
||||
return iend - anchor;
|
||||
}
|
||||
|
||||
@@ -536,13 +444,11 @@ size_t ZSTD_ldm_generateSequences(
|
||||
|
||||
assert(chunkStart < iend);
|
||||
/* 1. Perform overflow correction if necessary. */
|
||||
if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) {
|
||||
if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) {
|
||||
U32 const ldmHSize = 1U << params->hashLog;
|
||||
U32 const correction = ZSTD_window_correctOverflow(
|
||||
&ldmState->window, /* cycleLog */ 0, maxDist, chunkStart);
|
||||
ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction);
|
||||
/* invalidate dictionaries on overflow correction */
|
||||
ldmState->loadedDictEnd = 0;
|
||||
}
|
||||
/* 2. We enforce the maximum offset allowed.
|
||||
*
|
||||
@@ -551,14 +457,8 @@ size_t ZSTD_ldm_generateSequences(
|
||||
* TODO: * Test the chunk size.
|
||||
* * Try invalidation after the sequence generation and test the
|
||||
* the offset against maxDist directly.
|
||||
*
|
||||
* NOTE: Because of dictionaries + sequence splitting we MUST make sure
|
||||
* that any offset used is valid at the END of the sequence, since it may
|
||||
* be split into two sequences. This condition holds when using
|
||||
* ZSTD_window_enforceMaxDist(), but if we move to checking offsets
|
||||
* against maxDist directly, we'll have to carefully handle that case.
|
||||
*/
|
||||
ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL);
|
||||
ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, NULL, NULL);
|
||||
/* 3. Generate the sequences for the chunk, and get newLeftoverSize. */
|
||||
newLeftoverSize = ZSTD_ldm_generateSequences_internal(
|
||||
ldmState, sequences, params, chunkStart, chunkSize);
|
||||
@@ -580,9 +480,7 @@ size_t ZSTD_ldm_generateSequences(
|
||||
return 0;
|
||||
}
|
||||
|
||||
void
|
||||
ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
|
||||
{
|
||||
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) {
|
||||
while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
|
||||
rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
|
||||
if (srcSize <= seq->litLength) {
|
||||
@@ -641,32 +539,14 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
|
||||
return sequence;
|
||||
}
|
||||
|
||||
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
|
||||
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
|
||||
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
|
||||
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
|
||||
if (currPos >= currSeq.litLength + currSeq.matchLength) {
|
||||
currPos -= currSeq.litLength + currSeq.matchLength;
|
||||
rawSeqStore->pos++;
|
||||
} else {
|
||||
rawSeqStore->posInSequence = currPos;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
|
||||
rawSeqStore->posInSequence = 0;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_paramSwitch_e useRowMatchFinder,
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
unsigned const minMatch = cParams->minMatch;
|
||||
ZSTD_blockCompressor const blockCompressor =
|
||||
ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
|
||||
ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
|
||||
/* Input bounds */
|
||||
BYTE const* const istart = (BYTE const*)src;
|
||||
BYTE const* const iend = istart + srcSize;
|
||||
@@ -674,18 +554,9 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
BYTE const* ip = istart;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
|
||||
/* If using opt parser, use LDMs only as candidates rather than always accepting them */
|
||||
if (cParams->strategy >= ZSTD_btopt) {
|
||||
size_t lastLLSize;
|
||||
ms->ldmSeqStore = rawSeqStore;
|
||||
lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize);
|
||||
ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
|
||||
return lastLLSize;
|
||||
}
|
||||
|
||||
assert(rawSeqStore->pos <= rawSeqStore->size);
|
||||
assert(rawSeqStore->size <= rawSeqStore->capacity);
|
||||
/* Loop through each sequence and apply the block compressor to the literals */
|
||||
/* Loop through each sequence and apply the block compressor to the lits */
|
||||
while (rawSeqStore->pos < rawSeqStore->size && ip < iend) {
|
||||
/* maybeSplitSequence updates rawSeqStore->pos */
|
||||
rawSeq const sequence = maybeSplitSequence(rawSeqStore,
|
||||
@@ -695,13 +566,14 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
if (sequence.offset == 0)
|
||||
break;
|
||||
|
||||
assert(sequence.offset <= (1U << cParams->windowLog));
|
||||
assert(ip + sequence.litLength + sequence.matchLength <= iend);
|
||||
|
||||
/* Fill tables for block compressor */
|
||||
ZSTD_ldm_limitTableUpdate(ms, ip);
|
||||
ZSTD_ldm_fillFastTables(ms, ip);
|
||||
/* Run the block compressor */
|
||||
DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength);
|
||||
DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength);
|
||||
{
|
||||
size_t const newLitLength =
|
||||
blockCompressor(ms, seqStore, rep, ip, sequence.litLength);
|
||||
@@ -712,8 +584,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
rep[0] = sequence.offset;
|
||||
/* Store the sequence */
|
||||
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
|
||||
STORE_OFFSET(sequence.offset),
|
||||
sequence.matchLength);
|
||||
sequence.offset + ZSTD_REP_MOVE,
|
||||
sequence.matchLength - MINMATCH);
|
||||
ip += sequence.matchLength;
|
||||
}
|
||||
}
|
||||
@@ -723,5 +595,3 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
/* Compress the last literals */
|
||||
return blockCompressor(ms, seqStore, rep, ip, iend - ip);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
19
vendor/github.com/DataDog/zstd/zstd_ldm.h
generated
vendored
19
vendor/github.com/DataDog/zstd/zstd_ldm.h
generated
vendored
@@ -1,12 +1,10 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_LDM_H
|
||||
@@ -25,10 +23,6 @@ extern "C" {
|
||||
|
||||
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
|
||||
|
||||
void ZSTD_ldm_fillHashTable(
|
||||
ldmState_t* state, const BYTE* ip,
|
||||
const BYTE* iend, ldmParams_t const* params);
|
||||
|
||||
/**
|
||||
* ZSTD_ldm_generateSequences():
|
||||
*
|
||||
@@ -67,7 +61,6 @@ size_t ZSTD_ldm_generateSequences(
|
||||
*/
|
||||
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_paramSwitch_e useRowMatchFinder,
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
/**
|
||||
@@ -75,17 +68,11 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
*
|
||||
* Skip past `srcSize` bytes worth of sequences in `rawSeqStore`.
|
||||
* Avoids emitting matches less than `minMatch` bytes.
|
||||
* Must be called for data that is not passed to ZSTD_ldm_blockCompress().
|
||||
* Must be called for data with is not passed to ZSTD_ldm_blockCompress().
|
||||
*/
|
||||
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
|
||||
U32 const minMatch);
|
||||
|
||||
/* ZSTD_ldm_skipRawSeqStoreBytes():
|
||||
* Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'.
|
||||
* Not to be used in conjunction with ZSTD_ldm_skipSequences().
|
||||
* Must be called for data with is not passed to ZSTD_ldm_blockCompress().
|
||||
*/
|
||||
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes);
|
||||
|
||||
/** ZSTD_ldm_getTableSize() :
|
||||
* Estimate the space needed for long distance matching tables or 0 if LDM is
|
||||
@@ -116,5 +103,3 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_FAST_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
109
vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h
generated
vendored
109
vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h
generated
vendored
@@ -1,109 +0,0 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_LDM_GEARTAB_H
|
||||
#define ZSTD_LDM_GEARTAB_H
|
||||
|
||||
#include "compiler.h" /* UNUSED_ATTR */
|
||||
#include "mem.h" /* U64 */
|
||||
|
||||
static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = {
|
||||
0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc,
|
||||
0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05,
|
||||
0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e,
|
||||
0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889,
|
||||
0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e,
|
||||
0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140,
|
||||
0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e,
|
||||
0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f,
|
||||
0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391,
|
||||
0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210,
|
||||
0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be,
|
||||
0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a,
|
||||
0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b,
|
||||
0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4,
|
||||
0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb,
|
||||
0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312,
|
||||
0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01,
|
||||
0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc,
|
||||
0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967,
|
||||
0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553,
|
||||
0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f,
|
||||
0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2,
|
||||
0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d,
|
||||
0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a,
|
||||
0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74,
|
||||
0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3,
|
||||
0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1,
|
||||
0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b,
|
||||
0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568,
|
||||
0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a,
|
||||
0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1,
|
||||
0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9,
|
||||
0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463,
|
||||
0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba,
|
||||
0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9,
|
||||
0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61,
|
||||
0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec,
|
||||
0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6,
|
||||
0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab,
|
||||
0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5,
|
||||
0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59,
|
||||
0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7,
|
||||
0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc,
|
||||
0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb,
|
||||
0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be,
|
||||
0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312,
|
||||
0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1,
|
||||
0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc,
|
||||
0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d,
|
||||
0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445,
|
||||
0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c,
|
||||
0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5,
|
||||
0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5,
|
||||
0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28,
|
||||
0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a,
|
||||
0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9,
|
||||
0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15,
|
||||
0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef,
|
||||
0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2,
|
||||
0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375,
|
||||
0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3,
|
||||
0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595,
|
||||
0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389,
|
||||
0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4,
|
||||
0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756,
|
||||
0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc,
|
||||
0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45,
|
||||
0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea,
|
||||
0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f,
|
||||
0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc,
|
||||
0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c,
|
||||
0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a,
|
||||
0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17,
|
||||
0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3,
|
||||
0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4,
|
||||
0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91,
|
||||
0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40,
|
||||
0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741,
|
||||
0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f,
|
||||
0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4,
|
||||
0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad,
|
||||
0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047,
|
||||
0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2,
|
||||
0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e,
|
||||
0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b,
|
||||
0x2b4da14f2613d8f4
|
||||
};
|
||||
|
||||
#endif /* ZSTD_LDM_GEARTAB_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
5
vendor/github.com/DataDog/zstd/zstd_legacy.h
generated
vendored
5
vendor/github.com/DataDog/zstd/zstd_legacy.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -414,5 +413,3 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_LEGACY_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
687
vendor/github.com/DataDog/zstd/zstd_opt.c
generated
vendored
687
vendor/github.com/DataDog/zstd/zstd_opt.c
generated
vendored
File diff suppressed because it is too large
Load Diff
5
vendor/github.com/DataDog/zstd/zstd_opt.h
generated
vendored
5
vendor/github.com/DataDog/zstd/zstd_opt.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -55,5 +54,3 @@ size_t ZSTD_compressBlock_btultra_extDict(
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_OPT_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
220
vendor/github.com/DataDog/zstd/zstd_stream.go
generated
vendored
220
vendor/github.com/DataDog/zstd/zstd_stream.go
generated
vendored
@@ -1,6 +1,8 @@
|
||||
package zstd
|
||||
|
||||
/*
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "stdint.h" // for uintptr_t
|
||||
#include "zstd.h"
|
||||
|
||||
typedef struct compressStream2_result_s {
|
||||
@@ -9,10 +11,9 @@ typedef struct compressStream2_result_s {
|
||||
size_t bytes_written;
|
||||
} compressStream2_result;
|
||||
|
||||
static void ZSTD_compressStream2_wrapper(compressStream2_result* result, ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
|
||||
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
|
||||
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
|
||||
static void ZSTD_compressStream2_wrapper(compressStream2_result* result, ZSTD_CCtx* ctx, uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize) {
|
||||
ZSTD_outBuffer outBuffer = { (void*)dst, maxDstSize, 0 };
|
||||
ZSTD_inBuffer inBuffer = { (void*)src, srcSize, 0 };
|
||||
size_t retCode = ZSTD_compressStream2(ctx, &outBuffer, &inBuffer, ZSTD_e_continue);
|
||||
|
||||
result->return_code = retCode;
|
||||
@@ -20,21 +21,9 @@ static void ZSTD_compressStream2_wrapper(compressStream2_result* result, ZSTD_CC
|
||||
result->bytes_written = outBuffer.pos;
|
||||
}
|
||||
|
||||
static void ZSTD_compressStream2_flush(compressStream2_result* result, ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
|
||||
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
|
||||
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
|
||||
size_t retCode = ZSTD_compressStream2(ctx, &outBuffer, &inBuffer, ZSTD_e_flush);
|
||||
|
||||
result->return_code = retCode;
|
||||
result->bytes_consumed = inBuffer.pos;
|
||||
result->bytes_written = outBuffer.pos;
|
||||
}
|
||||
|
||||
static void ZSTD_compressStream2_finish(compressStream2_result* result, ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
|
||||
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
|
||||
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
|
||||
static void ZSTD_compressStream2_finish(compressStream2_result* result, ZSTD_CCtx* ctx, uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize) {
|
||||
ZSTD_outBuffer outBuffer = { (void*)dst, maxDstSize, 0 };
|
||||
ZSTD_inBuffer inBuffer = { (void*)src, srcSize, 0 };
|
||||
size_t retCode = ZSTD_compressStream2(ctx, &outBuffer, &inBuffer, ZSTD_e_end);
|
||||
|
||||
result->return_code = retCode;
|
||||
@@ -49,10 +38,9 @@ typedef struct decompressStream2_result_s {
|
||||
size_t bytes_written;
|
||||
} decompressStream2_result;
|
||||
|
||||
static void ZSTD_decompressStream_wrapper(decompressStream2_result* result, ZSTD_DCtx* ctx,
|
||||
void* dst, size_t maxDstSize, const void* src, size_t srcSize) {
|
||||
ZSTD_outBuffer outBuffer = { dst, maxDstSize, 0 };
|
||||
ZSTD_inBuffer inBuffer = { src, srcSize, 0 };
|
||||
static void ZSTD_decompressStream_wrapper(decompressStream2_result* result, ZSTD_DCtx* ctx, uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize) {
|
||||
ZSTD_outBuffer outBuffer = { (void*)dst, maxDstSize, 0 };
|
||||
ZSTD_inBuffer inBuffer = { (void*)src, srcSize, 0 };
|
||||
size_t retCode = ZSTD_decompressStream(ctx, &outBuffer, &inBuffer);
|
||||
|
||||
result->return_code = retCode;
|
||||
@@ -167,19 +155,20 @@ func (w *Writer) Write(p []byte) (int, error) {
|
||||
srcData = w.srcBuffer
|
||||
}
|
||||
|
||||
if len(srcData) == 0 {
|
||||
// this is technically unnecessary: srcData is p or w.srcBuffer, and len() > 0 checked above
|
||||
// but this ensures the code can change without dereferencing an srcData[0]
|
||||
return 0, nil
|
||||
srcPtr := C.uintptr_t(uintptr(0)) // Do not point anywhere, if src is empty
|
||||
if len(srcData) > 0 {
|
||||
srcPtr = C.uintptr_t(uintptr(unsafe.Pointer(&srcData[0])))
|
||||
}
|
||||
|
||||
C.ZSTD_compressStream2_wrapper(
|
||||
w.resultBuffer,
|
||||
w.ctx,
|
||||
unsafe.Pointer(&w.dstBuffer[0]),
|
||||
C.uintptr_t(uintptr(unsafe.Pointer(&w.dstBuffer[0]))),
|
||||
C.size_t(len(w.dstBuffer)),
|
||||
unsafe.Pointer(&srcData[0]),
|
||||
srcPtr,
|
||||
C.size_t(len(srcData)),
|
||||
)
|
||||
runtime.KeepAlive(p) // Ensure p is kept until here so pointer doesn't disappear during C call
|
||||
ret := int(w.resultBuffer.return_code)
|
||||
if err := getError(ret); err != nil {
|
||||
return 0, err
|
||||
@@ -214,49 +203,6 @@ func (w *Writer) Write(p []byte) (int, error) {
|
||||
return len(p), err
|
||||
}
|
||||
|
||||
// Flush writes any unwritten data to the underlying io.Writer.
|
||||
func (w *Writer) Flush() error {
|
||||
if w.firstError != nil {
|
||||
return w.firstError
|
||||
}
|
||||
|
||||
ret := 1 // So we loop at least once
|
||||
for ret > 0 {
|
||||
var srcPtr *byte // Do not point anywhere, if src is empty
|
||||
if len(w.srcBuffer) > 0 {
|
||||
srcPtr = &w.srcBuffer[0]
|
||||
}
|
||||
|
||||
C.ZSTD_compressStream2_flush(
|
||||
w.resultBuffer,
|
||||
w.ctx,
|
||||
unsafe.Pointer(&w.dstBuffer[0]),
|
||||
C.size_t(len(w.dstBuffer)),
|
||||
unsafe.Pointer(srcPtr),
|
||||
C.size_t(len(w.srcBuffer)),
|
||||
)
|
||||
ret = int(w.resultBuffer.return_code)
|
||||
if err := getError(ret); err != nil {
|
||||
return err
|
||||
}
|
||||
w.srcBuffer = w.srcBuffer[w.resultBuffer.bytes_consumed:]
|
||||
written := int(w.resultBuffer.bytes_written)
|
||||
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ret > 0 { // We have a hint if we need to resize the dstBuffer
|
||||
w.dstBuffer = w.dstBuffer[:cap(w.dstBuffer)]
|
||||
if len(w.dstBuffer) < ret {
|
||||
w.dstBuffer = make([]byte, ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the Writer, flushing any unwritten data to the underlying
|
||||
// io.Writer and freeing objects, but does not close the underlying io.Writer.
|
||||
func (w *Writer) Close() error {
|
||||
@@ -266,17 +212,17 @@ func (w *Writer) Close() error {
|
||||
|
||||
ret := 1 // So we loop at least once
|
||||
for ret > 0 {
|
||||
var srcPtr *byte // Do not point anywhere, if src is empty
|
||||
srcPtr := C.uintptr_t(uintptr(0)) // Do not point anywhere, if src is empty
|
||||
if len(w.srcBuffer) > 0 {
|
||||
srcPtr = &w.srcBuffer[0]
|
||||
srcPtr = C.uintptr_t(uintptr(unsafe.Pointer(&w.srcBuffer[0])))
|
||||
}
|
||||
|
||||
C.ZSTD_compressStream2_finish(
|
||||
w.resultBuffer,
|
||||
w.ctx,
|
||||
unsafe.Pointer(&w.dstBuffer[0]),
|
||||
C.uintptr_t(uintptr(unsafe.Pointer(&w.dstBuffer[0]))),
|
||||
C.size_t(len(w.dstBuffer)),
|
||||
unsafe.Pointer(srcPtr),
|
||||
srcPtr,
|
||||
C.size_t(len(w.srcBuffer)),
|
||||
)
|
||||
ret = int(w.resultBuffer.return_code)
|
||||
@@ -285,11 +231,7 @@ func (w *Writer) Close() error {
|
||||
}
|
||||
w.srcBuffer = w.srcBuffer[w.resultBuffer.bytes_consumed:]
|
||||
written := int(w.resultBuffer.bytes_written)
|
||||
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
|
||||
if err != nil {
|
||||
C.ZSTD_freeCStream(w.ctx)
|
||||
return err
|
||||
}
|
||||
w.underlyingWriter.Write(w.dstBuffer[:written])
|
||||
|
||||
if ret > 0 { // We have a hint if we need to resize the dstBuffer
|
||||
w.dstBuffer = w.dstBuffer[:cap(w.dstBuffer)]
|
||||
@@ -421,86 +363,49 @@ func (r *reader) Read(p []byte) (int, error) {
|
||||
return 0, r.firstError
|
||||
}
|
||||
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
// If we already have enough bytes, return
|
||||
if r.decompSize-r.decompOff >= len(p) {
|
||||
copy(p, r.decompressionBuffer[r.decompOff:])
|
||||
r.decompOff += len(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// If we already have some uncompressed bytes, return without blocking
|
||||
if r.decompSize > r.decompOff {
|
||||
if r.decompSize-r.decompOff > len(p) {
|
||||
copy(p, r.decompressionBuffer[r.decompOff:])
|
||||
r.decompOff += len(p)
|
||||
return len(p), nil
|
||||
}
|
||||
// From https://golang.org/pkg/io/#Reader
|
||||
// > Read conventionally returns what is available instead of waiting for more.
|
||||
copy(p, r.decompressionBuffer[r.decompOff:r.decompSize])
|
||||
got := r.decompSize - r.decompOff
|
||||
r.decompOff = r.decompSize
|
||||
return got, nil
|
||||
}
|
||||
copy(p, r.decompressionBuffer[r.decompOff:r.decompSize])
|
||||
got := r.decompSize - r.decompOff
|
||||
r.decompSize = 0
|
||||
r.decompOff = 0
|
||||
|
||||
// Repeatedly read from the underlying reader until we get
|
||||
// at least one zstd block, so that we don't block if the
|
||||
// other end has flushed a block.
|
||||
for {
|
||||
// - If the last decompression didn't entirely fill the decompression buffer,
|
||||
// zstd flushed all it could, and needs new data. In that case, do 1 Read.
|
||||
// - If the last decompression did entirely fill the decompression buffer,
|
||||
// it might have needed more room to decompress the input. In that case,
|
||||
// don't do any unnecessary Read that might block.
|
||||
needsData := r.decompSize < len(r.decompressionBuffer)
|
||||
|
||||
var src []byte
|
||||
if !needsData {
|
||||
src = r.compressionBuffer[:r.compressionLeft]
|
||||
} else {
|
||||
src = r.compressionBuffer
|
||||
var n int
|
||||
var err error
|
||||
// Read until data arrives or an error occurs.
|
||||
for n == 0 && err == nil {
|
||||
n, err = r.underlyingReader.Read(src[r.compressionLeft:])
|
||||
}
|
||||
if err != nil && err != io.EOF { // Handle underlying reader errors first
|
||||
return 0, fmt.Errorf("failed to read from underlying reader: %s", err)
|
||||
}
|
||||
if n == 0 {
|
||||
// Ideally, we'd return with ErrUnexpectedEOF in all cases where the stream was unexpectedly EOF'd
|
||||
// during a block or frame, i.e. when there are incomplete, pending compression data.
|
||||
// However, it's hard to detect those cases with zstd. Namely, there is no way to know the size of
|
||||
// the current buffered compression data in the zstd stream internal buffers.
|
||||
// Best effort: throw ErrUnexpectedEOF if we still have some pending buffered compression data that
|
||||
// zstd doesn't want to accept.
|
||||
// If we don't have any buffered compression data but zstd still has some in its internal buffers,
|
||||
// we will return with EOF instead.
|
||||
if r.compressionLeft > 0 {
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
src = src[:r.compressionLeft+n]
|
||||
for got < len(p) {
|
||||
// Populate src
|
||||
src := r.compressionBuffer
|
||||
reader := r.underlyingReader
|
||||
n, err := TryReadFull(reader, src[r.compressionLeft:])
|
||||
if err != nil && err != errShortRead { // Handle underlying reader errors first
|
||||
return 0, fmt.Errorf("failed to read from underlying reader: %s", err)
|
||||
} else if n == 0 && r.compressionLeft == 0 {
|
||||
return got, io.EOF
|
||||
}
|
||||
src = src[:r.compressionLeft+n]
|
||||
|
||||
// C code
|
||||
var srcPtr *byte // Do not point anywhere, if src is empty
|
||||
srcPtr := C.uintptr_t(uintptr(0)) // Do not point anywhere, if src is empty
|
||||
if len(src) > 0 {
|
||||
srcPtr = &src[0]
|
||||
srcPtr = C.uintptr_t(uintptr(unsafe.Pointer(&src[0])))
|
||||
}
|
||||
|
||||
C.ZSTD_decompressStream_wrapper(
|
||||
r.resultBuffer,
|
||||
r.ctx,
|
||||
unsafe.Pointer(&r.decompressionBuffer[0]),
|
||||
C.uintptr_t(uintptr(unsafe.Pointer(&r.decompressionBuffer[0]))),
|
||||
C.size_t(len(r.decompressionBuffer)),
|
||||
unsafe.Pointer(srcPtr),
|
||||
srcPtr,
|
||||
C.size_t(len(src)),
|
||||
)
|
||||
retCode := int(r.resultBuffer.return_code)
|
||||
|
||||
// Keep src here even though we reuse later, the code might be deleted at some point
|
||||
// Keep src here eventhough we reuse later, the code might be deleted at some point
|
||||
runtime.KeepAlive(src)
|
||||
if err := getError(retCode); err != nil {
|
||||
if err = getError(retCode); err != nil {
|
||||
return 0, fmt.Errorf("failed to decompress: %s", err)
|
||||
}
|
||||
|
||||
@@ -510,9 +415,10 @@ func (r *reader) Read(p []byte) (int, error) {
|
||||
left := src[bytesConsumed:]
|
||||
copy(r.compressionBuffer, left)
|
||||
}
|
||||
r.compressionLeft = len(src) - bytesConsumed
|
||||
r.compressionLeft = len(src) - int(bytesConsumed)
|
||||
r.decompSize = int(r.resultBuffer.bytes_written)
|
||||
r.decompOff = copy(p, r.decompressionBuffer[:r.decompSize])
|
||||
r.decompOff = copy(p[got:], r.decompressionBuffer[:r.decompSize])
|
||||
got += r.decompOff
|
||||
|
||||
// Resize buffers
|
||||
nsize := retCode // Hint for next src buffer size
|
||||
@@ -524,9 +430,25 @@ func (r *reader) Read(p []byte) (int, error) {
|
||||
nsize = r.compressionLeft
|
||||
}
|
||||
r.compressionBuffer = resize(r.compressionBuffer, nsize)
|
||||
|
||||
if r.decompOff > 0 {
|
||||
return r.decompOff, nil
|
||||
}
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
|
||||
// TryReadFull reads buffer just as ReadFull does
|
||||
// Here we expect that buffer may end and we do not return ErrUnexpectedEOF as ReadAtLeast does.
|
||||
// We return errShortRead instead to distinguish short reads and failures.
|
||||
// We cannot use ReadFull/ReadAtLeast because it masks Reader errors, such as network failures
|
||||
// and causes panic instead of error.
|
||||
func TryReadFull(r io.Reader, buf []byte) (n int, err error) {
|
||||
for n < len(buf) && err == nil {
|
||||
var nn int
|
||||
nn, err = r.Read(buf[n:])
|
||||
n += nn
|
||||
}
|
||||
if n == len(buf) && err == io.EOF {
|
||||
err = nil // EOF at the end is somewhat expected
|
||||
} else if err == io.EOF {
|
||||
err = errShortRead
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
166
vendor/github.com/DataDog/zstd/zstd_trace.h
generated
vendored
166
vendor/github.com/DataDog/zstd/zstd_trace.h
generated
vendored
@@ -1,166 +0,0 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_TRACE_H
|
||||
#define ZSTD_TRACE_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* weak symbol support
|
||||
* For now, enable conservatively:
|
||||
* - Only GNUC
|
||||
* - Only ELF
|
||||
* - Only x86-64 and i386
|
||||
* Also, explicitly disable on platforms known not to work so they aren't
|
||||
* forgotten in the future.
|
||||
*/
|
||||
#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
|
||||
defined(__GNUC__) && defined(__ELF__) && \
|
||||
(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) && \
|
||||
!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
|
||||
!defined(__CYGWIN__) && !defined(_AIX)
|
||||
# define ZSTD_HAVE_WEAK_SYMBOLS 1
|
||||
#else
|
||||
# define ZSTD_HAVE_WEAK_SYMBOLS 0
|
||||
#endif
|
||||
#if ZSTD_HAVE_WEAK_SYMBOLS
|
||||
# define ZSTD_WEAK_ATTR __attribute__((__weak__))
|
||||
#else
|
||||
# define ZSTD_WEAK_ATTR
|
||||
#endif
|
||||
|
||||
/* Only enable tracing when weak symbols are available. */
|
||||
#ifndef ZSTD_TRACE
|
||||
# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
|
||||
#endif
|
||||
|
||||
#if ZSTD_TRACE
|
||||
|
||||
struct ZSTD_CCtx_s;
|
||||
struct ZSTD_DCtx_s;
|
||||
struct ZSTD_CCtx_params_s;
|
||||
|
||||
typedef struct {
|
||||
/**
|
||||
* ZSTD_VERSION_NUMBER
|
||||
*
|
||||
* This is guaranteed to be the first member of ZSTD_trace.
|
||||
* Otherwise, this struct is not stable between versions. If
|
||||
* the version number does not match your expectation, you
|
||||
* should not interpret the rest of the struct.
|
||||
*/
|
||||
unsigned version;
|
||||
/**
|
||||
* Non-zero if streaming (de)compression is used.
|
||||
*/
|
||||
unsigned streaming;
|
||||
/**
|
||||
* The dictionary ID.
|
||||
*/
|
||||
unsigned dictionaryID;
|
||||
/**
|
||||
* Is the dictionary cold?
|
||||
* Only set on decompression.
|
||||
*/
|
||||
unsigned dictionaryIsCold;
|
||||
/**
|
||||
* The dictionary size or zero if no dictionary.
|
||||
*/
|
||||
size_t dictionarySize;
|
||||
/**
|
||||
* The uncompressed size of the data.
|
||||
*/
|
||||
size_t uncompressedSize;
|
||||
/**
|
||||
* The compressed size of the data.
|
||||
*/
|
||||
size_t compressedSize;
|
||||
/**
|
||||
* The fully resolved CCtx parameters (NULL on decompression).
|
||||
*/
|
||||
struct ZSTD_CCtx_params_s const* params;
|
||||
/**
|
||||
* The ZSTD_CCtx pointer (NULL on decompression).
|
||||
*/
|
||||
struct ZSTD_CCtx_s const* cctx;
|
||||
/**
|
||||
* The ZSTD_DCtx pointer (NULL on compression).
|
||||
*/
|
||||
struct ZSTD_DCtx_s const* dctx;
|
||||
} ZSTD_Trace;
|
||||
|
||||
/**
|
||||
* A tracing context. It must be 0 when tracing is disabled.
|
||||
* Otherwise, any non-zero value returned by a tracing begin()
|
||||
* function is presented to any subsequent calls to end().
|
||||
*
|
||||
* Any non-zero value is treated as tracing is enabled and not
|
||||
* interpreted by the library.
|
||||
*
|
||||
* Two possible uses are:
|
||||
* * A timestamp for when the begin() function was called.
|
||||
* * A unique key identifying the (de)compression, like the
|
||||
* address of the [dc]ctx pointer if you need to track
|
||||
* more information than just a timestamp.
|
||||
*/
|
||||
typedef unsigned long long ZSTD_TraceCtx;
|
||||
|
||||
/**
|
||||
* Trace the beginning of a compression call.
|
||||
* @param cctx The dctx pointer for the compression.
|
||||
* It can be used as a key to map begin() to end().
|
||||
* @returns Non-zero if tracing is enabled. The return value is
|
||||
* passed to ZSTD_trace_compress_end().
|
||||
*/
|
||||
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
|
||||
struct ZSTD_CCtx_s const* cctx);
|
||||
|
||||
/**
|
||||
* Trace the end of a compression call.
|
||||
* @param ctx The return value of ZSTD_trace_compress_begin().
|
||||
* @param trace The zstd tracing info.
|
||||
*/
|
||||
ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
|
||||
ZSTD_TraceCtx ctx,
|
||||
ZSTD_Trace const* trace);
|
||||
|
||||
/**
|
||||
* Trace the beginning of a decompression call.
|
||||
* @param dctx The dctx pointer for the decompression.
|
||||
* It can be used as a key to map begin() to end().
|
||||
* @returns Non-zero if tracing is enabled. The return value is
|
||||
* passed to ZSTD_trace_compress_end().
|
||||
*/
|
||||
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
|
||||
struct ZSTD_DCtx_s const* dctx);
|
||||
|
||||
/**
|
||||
* Trace the end of a decompression call.
|
||||
* @param ctx The return value of ZSTD_trace_decompress_begin().
|
||||
* @param trace The zstd tracing info.
|
||||
*/
|
||||
ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
|
||||
ZSTD_TraceCtx ctx,
|
||||
ZSTD_Trace const* trace);
|
||||
|
||||
#endif /* ZSTD_TRACE */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_TRACE_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
45
vendor/github.com/DataDog/zstd/zstd_v01.c
generated
vendored
45
vendor/github.com/DataDog/zstd/zstd_v01.c
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -205,7 +204,10 @@ typedef signed long long S64;
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define FSE_FORCE_MEMORY_ACCESS 2
|
||||
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
||||
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
||||
# define FSE_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
@@ -255,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
||||
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
||||
#endif // FSE_FORCE_MEMORY_ACCESS
|
||||
|
||||
static U16 FSE_readLE16(const void* memPtr)
|
||||
{
|
||||
@@ -341,7 +343,8 @@ FORCE_INLINE unsigned FSE_highbit32 (U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r;
|
||||
return _BitScanReverse(&r, val) ? (unsigned)r : 0;
|
||||
_BitScanReverse ( &r, val );
|
||||
return (unsigned) r;
|
||||
# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
|
||||
return __builtin_clz (val) ^ 31;
|
||||
# else /* Software version */
|
||||
@@ -1075,7 +1078,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const omax = op + maxDstSize;
|
||||
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
||||
BYTE* const olimit = omax-15;
|
||||
|
||||
const void* ptr = DTable;
|
||||
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
||||
@@ -1089,7 +1092,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
||||
const size_t length1 = FSE_readLE16(jumpTable);
|
||||
const size_t length2 = FSE_readLE16(jumpTable+1);
|
||||
const size_t length3 = FSE_readLE16(jumpTable+2);
|
||||
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
||||
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
|
||||
const char* const start1 = (const char*)(cSrc) + 6;
|
||||
const char* const start2 = start1 + length1;
|
||||
const char* const start3 = start2 + length2;
|
||||
@@ -1147,11 +1150,11 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
||||
|
||||
/* tail */
|
||||
{
|
||||
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
||||
// bitTail = bitD1; // *much* slower : -20% !??!
|
||||
FSE_DStream_t bitTail;
|
||||
bitTail.ptr = bitD1.ptr;
|
||||
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
||||
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
||||
bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
|
||||
bitTail.start = start1;
|
||||
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
||||
{
|
||||
@@ -1277,11 +1280,7 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
|
||||
* Basic Types
|
||||
*********************************************************/
|
||||
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# if defined(_AIX)
|
||||
# include <inttypes.h>
|
||||
# else
|
||||
# include <stdint.h> /* intptr_t */
|
||||
# endif
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef int16_t S16;
|
||||
@@ -1484,9 +1483,7 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
|
||||
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
||||
if (srcSize > 0) {
|
||||
memcpy(dst, src, srcSize);
|
||||
}
|
||||
memcpy(dst, src, srcSize);
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
@@ -1505,7 +1502,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
||||
if (srcSize <= 3) return ERROR(corruption_detected);
|
||||
|
||||
litSize = ip[1] + (ip[0]<<8);
|
||||
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
||||
litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh....
|
||||
op = oend - litSize;
|
||||
|
||||
(void)ctx;
|
||||
@@ -1544,9 +1541,7 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
||||
size_t rleSize = litbp.origSize;
|
||||
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
||||
if (!srcSize) return ERROR(srcSize_wrong);
|
||||
if (rleSize > 0) {
|
||||
memset(oend - rleSize, *ip, rleSize);
|
||||
}
|
||||
memset(oend - rleSize, *ip, rleSize);
|
||||
*litStart = oend - rleSize;
|
||||
*litSize = rleSize;
|
||||
ip++;
|
||||
@@ -1906,10 +1901,8 @@ static size_t ZSTD_decompressSequences(
|
||||
{
|
||||
size_t lastLLSize = litEnd - litPtr;
|
||||
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
||||
if (lastLLSize > 0) {
|
||||
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
}
|
||||
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2157,5 +2150,3 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
5
vendor/github.com/DataDog/zstd/zstd_v01.h
generated
vendored
5
vendor/github.com/DataDog/zstd/zstd_v01.h
generated
vendored
@@ -1,6 +1,5 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -93,5 +92,3 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_V01_H_28739879432 */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user