Compare commits
13 Commits
v1.1
...
79840796f6
| Author | SHA1 | Date | |
|---|---|---|---|
|
79840796f6 | ||
|
|
24507ed6b4 | ||
|
|
484bfa9ba0 | ||
|
|
8f8073ca3e | ||
|
|
7bcc407857 | ||
|
|
8c24a1d8c6 | ||
|
|
4ca2fffb33 | ||
|
|
d198741b37 | ||
|
|
afe6e5384d | ||
|
|
b6f8cf9155 | ||
|
|
9bac4fb257 | ||
|
|
7958db9c0d | ||
|
|
4869c146fe |
83
.golangci.yml
Normal file
83
.golangci.yml
Normal file
@@ -0,0 +1,83 @@
|
||||
version: "2"
|
||||
run:
|
||||
go: "1.21"
|
||||
modules-download-mode: readonly
|
||||
tests: false
|
||||
allow-parallel-runners: true
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- asasalint
|
||||
- asciicheck
|
||||
- bodyclose
|
||||
- containedctx
|
||||
- contextcheck
|
||||
- decorder
|
||||
- durationcheck
|
||||
- errcheck
|
||||
- errchkjson
|
||||
- errorlint
|
||||
- exhaustive
|
||||
- ginkgolinter
|
||||
- gocheckcompilerdirectives
|
||||
- gocritic
|
||||
- godot
|
||||
- gosec
|
||||
- gosmopolitan
|
||||
- govet
|
||||
- grouper
|
||||
- importas
|
||||
- ineffassign
|
||||
- ireturn
|
||||
- makezero
|
||||
- mirror
|
||||
- nakedret
|
||||
- nilerr
|
||||
- nilnil
|
||||
- nlreturn
|
||||
- noctx
|
||||
- nolintlint
|
||||
- nonamedreturns
|
||||
- predeclared
|
||||
- promlinter
|
||||
- reassign
|
||||
- revive
|
||||
- rowserrcheck
|
||||
- staticcheck
|
||||
- unconvert
|
||||
- unused
|
||||
- wastedassign
|
||||
- whitespace
|
||||
- wsl
|
||||
settings:
|
||||
gosec:
|
||||
excludes:
|
||||
# Оно не используется для повышения безопасности или каких-то таких секъюрных вещей, оно используется для выборки
|
||||
# произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью.
|
||||
- G404
|
||||
# Не еби мозги. Сертификаты для локалхоста или для локальной сети могут быть невалидными, если ты сможешь
|
||||
# выписать валидный серт для локалхоста, тебе миллионы разработчиков поставят золотой памятник. Просто потому
|
||||
# что эти пидоры-безпасники всрали всё что только можно всрать своими дебильными политиками.
|
||||
- G402
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- comments
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
- goimports
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
- vendor$
|
||||
12
DESIGN
Normal file
12
DESIGN
Normal file
@@ -0,0 +1,12 @@
|
||||
Для irc admin-модуля:
|
||||
|
||||
1. берём pebbledb как бэкэнд сохранения настроек.
|
||||
|
||||
2. имплементировать trusted-участников чятика для настройки бота под конкретный канальчик.
|
||||
Есть два варианта:
|
||||
- ручной список в конфиге, где прописывается owner бота и доверенные лица (опционально доверенных можно прописывать в pebbledb admin_db)
|
||||
- доверенными являются op-ы
|
||||
|
||||
Первый вариант - довольно гибкий, но у нас всегда bot owner является доверенным. Не то чтобы это неправильно. Но с другой стороны можно сделать owner-а суперадмином, а другим админам дать возможность добавлять user-ов для своего канальчика в список доверенных. И запретить удалять себя из списка доверенных.
|
||||
|
||||
Второй вариант проще. И выглядит более привычным в текущих реалиях. Но есть нюанс - у нас нету простой возможности вынуть список участников вместе с флагами (предположительно). Кроме того, необходимо писать обработчк ивентов для всех изменений состояний юзеров (join/part и op/deop) для поддержания списка участников в актуальном состоянии.
|
||||
5
Makefile
5
Makefile
@@ -6,15 +6,14 @@ BUILDOPTS=-ldflags="-s -w" -a -gcflags=all=-l -trimpath
|
||||
all: clean build
|
||||
|
||||
build:
|
||||
${GOOPTS} go build ${BUILDOPTS} -o aleesa-irc-go types.go globals.go collection.go irc-user-mode.go \
|
||||
settings-db-util.go util.go msg-parsers.go aleesa-irc-go-lib.go main.go
|
||||
${GOOPTS} go build ${BUILDOPTS} -o aleesa-irc-go .
|
||||
|
||||
clean:
|
||||
go clean
|
||||
|
||||
upgrade:
|
||||
rm -rf vendor
|
||||
go get -d -u -t ./...
|
||||
go get -u -t -tool ./...
|
||||
go mod tidy
|
||||
go mod vendor
|
||||
|
||||
|
||||
@@ -1,74 +1,74 @@
|
||||
aleesa-irc-go
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
# aleesa-irc-go
|
||||
|
||||
Сервис для aleesa-bot, работающий с протоколом IRC. С основными сервисами бота работает по протоколу Redis pubsub.
|
||||
|
||||
## Механика работы
|
||||
|
||||
Механика работы
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Сервис подписан на pub-sub redis-канал irc. В этот канал пишут остальные сервисы aleesa-bot-а в ответ на запросы этого
|
||||
сервиса.
|
||||
|
||||
Формат протокола общения по redis pub-sub - см README от aleesa-misc-go.
|
||||
Формат протокола общения по redis pub-sub - см. документацию в репозитории aleesa-doc.
|
||||
|
||||
## Как это собрать?
|
||||
|
||||
Как это собрать?
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Понадобится golang версии 1.19 или более новая.
|
||||
Понадобится компилятор go версии 1.22 или более новый.
|
||||
|
||||
Сам процесс сборки простой, достаточно запустить:
|
||||
|
||||
```bash
|
||||
make
|
||||
```
|
||||
|
||||
и через некоторое время мы получаем бинарник aleesa-irc-go. Это и есть наше приложение.
|
||||
и через некоторое время мы получаем бинарник **aleesa-irc-go**. Это и есть наше приложение.
|
||||
|
||||
|
||||
Как это запустить?
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
## Как это запустить?
|
||||
|
||||
Приложение тестировалось на alpine linux x86_64 3.16, но должно работать на любом (более свежем) linux-е.
|
||||
|
||||
После создания data/config.json из data/config_sample.json, можно запускать приложение.
|
||||
После создания **data/config.json** из **data/config_sample.json**, можно запускать приложение.
|
||||
|
||||
Предполагается, что конфиг написан в формате human json (https://hjson.github.io), который допускает коментарии и
|
||||
Предполагается, что конфиг написан в формате [human json](https://hjson.github.io), который допускает коментарии и
|
||||
некоторые другие вольности в форматировании json-а. Конфиг ищется последовательно по следующим путям по принципу "кто
|
||||
первый того и тапки":
|
||||
|
||||
```
|
||||
~/.aleesa-irc-go.json
|
||||
~/aleesa-irc-go.json
|
||||
/etc/aleesa-irc-go.json
|
||||
data/config.json
|
||||
```
|
||||
|
||||
В каталоге contrib находится скрипт для alpine linux, системы инициализации openrc. Этот скрипт достаточно положить в
|
||||
/etc/init.d/aleesa-irc-go. После чего скопировать бинарник в каталог /var/lib/aleesa-irc-go, положить конфиг в одну
|
||||
из дефолтных локаций (тестировалась /var/lib/aleesa-irc-go/data/config.json) и после этого сервис запускается
|
||||
**/etc/init.d/aleesa-irc-go**. После чего скопировать бинарник в каталог **/var/lib/aleesa-irc-go**, положить конфиг в
|
||||
одну из дефолтных локаций (тестировалась **/var/lib/aleesa-irc-go/data/config.json**) и после этого сервис запускается
|
||||
командой:
|
||||
|
||||
```bash
|
||||
service aleesa-irc-go start
|
||||
```
|
||||
|
||||
В автозагрузку его можно прописать через команду:
|
||||
|
||||
```bash
|
||||
rc-update add aleesa-irc-go default
|
||||
```
|
||||
|
||||
## Nota Bene
|
||||
|
||||
Nota Bene
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
По механике, в процессе работы мы форкаем горутинки на каждое входящее сообщение, особо не контроллируя этот процесс.
|
||||
Предполагается, что на это приложение не будет большой нагрузки - ограничения irc не должны дать возможности развить
|
||||
достаточно большой message rate, чтобы об это споткнуться.
|
||||
|
||||
Формально, более удачным было бы иметь автоматический пул воркеров-горутинок и раскидывать им через channel-ы
|
||||
сообщения. Внутри каждой горутинки можно было бы повиcнуть в бесконечном цикле на канале входящих сообщений и так и
|
||||
работать. Но на практике оно того не стОит.
|
||||
|
||||
|
||||
Nota Bene #2
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
Go не поддерживает системный вызов fork() из-за чего демонизация программ на гошке средствами самой гошки - это в
|
||||
некотором роде проблема. Решается она с помощью утилиты daemonize (https://github.com/bmc/daemonize).
|
||||
некотором роде проблема. Решается она с помощью утилиты [daemonize](https://github.com/bmc/daemonize).
|
||||
|
||||
Соответствено, если для запуска этой программы предполагается использовать возможности демонизации, то можно
|
||||
воспользоваться этой программой.
|
||||
|
||||
В alpine linux есть средства openrc для супервайзинга foreground-сервисов.
|
||||
|
||||
Технически, можно было бы решить эту проблему fork()-а с помощью "github.com/sevlyar/go-daemon", но на текущий момент
|
||||
она не стоит остро, поэтому решения не требуется.
|
||||
|
||||
## Nota Bene 2
|
||||
|
||||
Nota Bene #3
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
К сожалению, в боте не подразумевается функциональность invite-ов. Причина банальна - слишком жёсткие ограничения у
|
||||
многих irc-сетей на количество сообщений в единицу времени, а по механике в большинстве случаев ограничение работает не
|
||||
client-client или client-channel, а глобально, client-server (за исключением случаев, когда на канале у бота есть +v
|
||||
@@ -77,6 +77,6 @@ client-client или client-channel, а глобально, client-server (за
|
||||
Сообщения об invite-ах пишутся в лог с уровнем info.
|
||||
|
||||
|
||||
Special Thanks
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
https://github.com/akyoto/cache за "простой" кэш, из которого я сделал collection.go
|
||||
## Special Thanks
|
||||
|
||||
https://github.com/akyoto/cache за "простой" кэш, из которого я сделал collection.go.
|
||||
14
TODO
Normal file
14
TODO
Normal file
@@ -0,0 +1,14 @@
|
||||
[ ] Имплементировать IRC (например, через https://github.com/thoj/go-ircevent)
|
||||
[*] irc в виде горутинки на вход, как irc-клиент + обработчик входящих ивентов - ctcp-сообщений, dcc и вот это всё
|
||||
[ ] подумать про настройки и нужность админской части (скорее всего нужно, для настройки функций ботов для канала)
|
||||
[ ] подумать, про инвайт бота и про удаление бота средствами !nick get lost, если бот заапрувлен, но уже надоел.
|
||||
[+] irc в виде функи для сообщений на выход, собственно, её будут дёргать как обработчик redis-сообщений, так модуль протокола irc
|
||||
[+] реализовать ограничитель сообщений в единицу времени
|
||||
|
||||
[+] Исправить парсер конфига
|
||||
|
||||
[ ] Мульти-инстанс. В формате нескольких сервисов с отдельными конфигами. (Описать мульти-инстанс в readme)
|
||||
[ ] Вариант запуска как с помощью симлинка (поиск конфига по имени бинаря).
|
||||
[ ] Вариант запуска с указанием положения конфига.
|
||||
|
||||
[ ] Дописать всякие необязательные .md-файлы :)
|
||||
@@ -4,22 +4,22 @@ import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"aleesa-irc-go/internal/boolcollection"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
irc "github.com/thoj/go-ircevent"
|
||||
)
|
||||
|
||||
// ircClientInit горутинка для работы с протоколом irc
|
||||
// ircClientInit горутинка для работы с протоколом irc.
|
||||
func ircClientRun() {
|
||||
for {
|
||||
if shutdown {
|
||||
// Если мы завершаем работу программы, то нам ничего обрабатывать не надо
|
||||
break
|
||||
}
|
||||
|
||||
// Иницализируем irc-клиента
|
||||
for !shutdown {
|
||||
// Иницализируем irc-клиента.
|
||||
// TODO: make use of capabilities: https://defs.ircdocs.horse/defs/clientcaps .
|
||||
// TODO: make use of tags caps https://defs.ircdocs.horse/defs/tags .
|
||||
serverString := fmt.Sprintf("%s:%d", config.Irc.Server, config.Irc.Port)
|
||||
log.Debugf("Preparing to connect to %s", serverString)
|
||||
log.Debugf("Using nick %s and username %s", config.Irc.Nick, config.Irc.User)
|
||||
@@ -29,10 +29,12 @@ func ircClientRun() {
|
||||
|
||||
if config.Irc.Ssl {
|
||||
log.Debug("Force use ssl for connection")
|
||||
|
||||
ircClient.UseTLS = true
|
||||
|
||||
if !config.Irc.SslVerify {
|
||||
log.Debug("Skip server certificate validation")
|
||||
|
||||
ircClient.TLSConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
} else {
|
||||
log.Debug("Force server certificate validation")
|
||||
@@ -47,13 +49,48 @@ func ircClientRun() {
|
||||
ircClient.UseSASL = true
|
||||
}
|
||||
|
||||
// Навесим коллбэков на некоторые ответы сервера на наши запросы
|
||||
// Навесим коллбэков на некоторые ответы сервера на наши запросы.
|
||||
|
||||
// 001 RPL_WELCOME уже есть в github.com/thoj/go-ircevent/irc_callback.go.
|
||||
|
||||
// Сделаем уже что-то полезное! Motd нам уже прислали и теперь можно авторизоваться и джойниться
|
||||
ircClient.AddCallback("004", func(e *irc.Event) {
|
||||
// Из это строки мы можем узнать, какие флаги для user MODE и channem MODE можно навешивать.
|
||||
// Как минимум, эта строка нам нужна, чтобы выяснить, можем ли мы взять себе +B, мы же бот :).
|
||||
e.Connection.Lock()
|
||||
|
||||
// Формат строки с MODE-ами https://datatracker.ietf.org/doc/html/rfc2812#section-5.1 .
|
||||
availableUserModes = boolcollection.NewCollection()
|
||||
availableChanModes = boolcollection.NewCollection()
|
||||
|
||||
log.Debugf("Add %s to list of available user modes", e.Arguments[3])
|
||||
|
||||
for _, mode := range strings.Split(e.Arguments[3], "") {
|
||||
availableUserModes.Set(mode, true)
|
||||
}
|
||||
|
||||
availableUserModes.Set("announced", true)
|
||||
|
||||
log.Debugf("Add %s to list available channel modes", e.Arguments[4])
|
||||
|
||||
for _, mode := range strings.Split(e.Arguments[4], "") {
|
||||
availableChanModes.Set(mode, true)
|
||||
}
|
||||
|
||||
availableChanModes.Set("announced", true)
|
||||
|
||||
e.Connection.Unlock()
|
||||
})
|
||||
|
||||
ircClient.AddCallback("005", func(e *irc.Event) {
|
||||
// TODO: Make use of it. https://defs.ircdocs.horse/defs/isupport
|
||||
})
|
||||
|
||||
ircClient.AddCallback("319", func(e *irc.Event) {
|
||||
/* Это одна из строк с данными, прилетающая в ответ на запрос whois на определённого юзера
|
||||
* Из этой строки нас интересует, на каких каналах пользователь op (то есть с префиксом @) или имеет voice
|
||||
* (то есть с префиксом +) чтобы внести его в свою базу mode-ов.
|
||||
*/
|
||||
log.Debugf("319 RPL_WHOISCHANNELS, %s", e.Raw)
|
||||
channelsWithModes := strings.Split(e.Arguments[2], " ")
|
||||
dstNick := e.Arguments[1]
|
||||
|
||||
@@ -65,27 +102,30 @@ func ircClientRun() {
|
||||
|
||||
if channel == mychannel {
|
||||
// Для каждого канала формат строго 1 из 4-х: #channel | @#channel | +#channel | @+#channel
|
||||
switch {
|
||||
case modes == "@+":
|
||||
switch modes {
|
||||
case "@+":
|
||||
userModeUpdateUser(channel, dstNick, "+o")
|
||||
userModeUpdateUser(channel, dstNick, "+v")
|
||||
case modes == "@":
|
||||
case "@":
|
||||
userModeUpdateUser(channel, dstNick, "+o")
|
||||
case modes == "+":
|
||||
case "+":
|
||||
userModeUpdateUser(channel, dstNick, "+v")
|
||||
default:
|
||||
userModeUpdateUser(channel, dstNick, "-o")
|
||||
userModeUpdateUser(channel, dstNick, "-v")
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 433 ERR_NICKNAMEINUSE уже есть в github.com/thoj/go-ircevent/irc_callback.go.
|
||||
|
||||
ircClient.AddCallback("353", func(e *irc.Event) {
|
||||
// Это одна из строк данных, прилетающая в ответ на запрос names, на канале
|
||||
log.Debugf("353 RPL_NAMREPLY, %s", e.Raw)
|
||||
e.Connection.Lock()
|
||||
namesString := e.Arguments[3]
|
||||
channel := e.Arguments[2]
|
||||
|
||||
@@ -93,6 +133,7 @@ func ircClientRun() {
|
||||
if channel == mychannel {
|
||||
for _, name := range strings.Split(namesString, " ") {
|
||||
mode := name[:1]
|
||||
|
||||
var nick string
|
||||
|
||||
if mode == "@" || mode == "+" {
|
||||
@@ -101,25 +142,58 @@ func ircClientRun() {
|
||||
nick = name
|
||||
}
|
||||
|
||||
switch {
|
||||
case mode == "@":
|
||||
switch mode {
|
||||
case "@":
|
||||
userModeUpdateUser(channel, nick, "+o")
|
||||
case mode == "+":
|
||||
case "+":
|
||||
userModeUpdateUser(channel, nick, "+v")
|
||||
default:
|
||||
userModeUpdateUser(channel, nick, "-o")
|
||||
userModeUpdateUser(channel, nick, "-v")
|
||||
}
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
e.Connection.Unlock()
|
||||
})
|
||||
|
||||
ircClient.AddCallback("352", func(e *irc.Event) {
|
||||
// Относительно бесполезная для нас команда. Не возвращает канала (в случае с ngircd), так что узнать mode
|
||||
// юзера на конкретном канале из этого невозможно. А остальные данные пока непонятно куда применять.
|
||||
log.Warnf("352 RPL_WHOREPLY, %s", e.Raw)
|
||||
// Если сервер не может прочитать MOTD, то он может вернуть 422 ERR_NOMOTD, тоде самое навесим и туда тоже.
|
||||
ircClient.AddCallback("376", func(e *irc.Event) {
|
||||
// TODO: make use of modes https://defs.ircdocs.horse/defs/usermodes
|
||||
// Если у нас есть доступный +B возьмём его себе, мы же бот.
|
||||
announced, _ := availableUserModes.Get("announced")
|
||||
botFlag, _ := availableUserModes.Get("B")
|
||||
|
||||
if announced {
|
||||
if botFlag {
|
||||
log.Info("Grabbing +B flag to mark me as bot")
|
||||
ircClient.Mode(ircClient.GetNick(), "+B")
|
||||
// TODO: Wait for it.
|
||||
} else {
|
||||
log.Info("Skip +B flag, server does not support it, noone will know that I am bot")
|
||||
}
|
||||
} else {
|
||||
// Этого по идее не должно быть.
|
||||
log.Info("Skip +B flag, server did not announce modes (yet?), noone will know that I am bot")
|
||||
}
|
||||
|
||||
if !config.Irc.Sasl && config.Irc.Password != "" && !nickIsUsed {
|
||||
log.Info("Identifying via NickServ")
|
||||
|
||||
message := fmt.Sprintf("identify %s %s", config.Irc.Nick, config.Irc.Password)
|
||||
imChan <- iMsg{ChatID: "NickServ", Text: message}
|
||||
}
|
||||
// TODO: wait until +R flag been set? could be implemented with waiting for channel message.
|
||||
|
||||
log.Debug("Trying to join to preconfigured channels")
|
||||
|
||||
for _, channel := range config.Irc.Channels {
|
||||
log.Infof("Joining to %s channel", channel)
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
// TODO: wait for join or join error.
|
||||
})
|
||||
|
||||
// Навесим коллбэков на все возможные и невозможные error status code, которые мы можем получить и сдампим это
|
||||
@@ -157,6 +231,39 @@ func ircClientRun() {
|
||||
log.Errorf("421 ERR_UNKNOWNCOMMAND, %s", e.Raw)
|
||||
})
|
||||
|
||||
// Аналогичный коллбэк висит на 376 RPL_ENDOFMOTD.
|
||||
ircClient.AddCallback("422", func(e *irc.Event) {
|
||||
// TODO: make use of modes https://defs.ircdocs.horse/defs/usermodes
|
||||
// Если у нас есть доступный +B возьмём его себе, мы же бот.
|
||||
botFlag, ok := availableUserModes.Get("B")
|
||||
|
||||
if ok && botFlag {
|
||||
log.Info("Grabbing +B flag to mark me as bot")
|
||||
ircClient.Mode(ircClient.GetNick(), "+B")
|
||||
} else {
|
||||
if ok {
|
||||
log.Info("Skip +B flag, server does not support it, noone will know that I am bot")
|
||||
} else {
|
||||
log.Info("Skip +B flag, server did not announce modes (yet?), noone will know that I am bot")
|
||||
}
|
||||
}
|
||||
|
||||
if !config.Irc.Sasl && config.Irc.Password != "" && !nickIsUsed {
|
||||
log.Info("Identifying via NickServ")
|
||||
|
||||
message := fmt.Sprintf("identify %s %s", config.Irc.Nick, config.Irc.Password)
|
||||
imChan <- iMsg{ChatID: "NickServ", Text: message}
|
||||
}
|
||||
// TODO: wait until +R flag been set? could be implemented with waiting for channel message.
|
||||
|
||||
log.Debug("Trying to join to preconfigured channels")
|
||||
|
||||
for _, channel := range config.Irc.Channels {
|
||||
log.Infof("Joining to %s channel", channel)
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
ircClient.AddCallback("431", func(e *irc.Event) {
|
||||
log.Errorf("431 ERR_NONICKNAMEGIVEN, %s", e.Raw)
|
||||
})
|
||||
@@ -167,8 +274,11 @@ func ircClientRun() {
|
||||
|
||||
ircClient.AddCallback("433", func(e *irc.Event) {
|
||||
log.Errorf("433 ERR_NICKNAMEINUSE, %s", e.Raw)
|
||||
|
||||
nickIsUsed = true
|
||||
time.Sleep(60 * time.Second)
|
||||
|
||||
<-time.NewTimer(60 * time.Second).C // 1 minute sleep.
|
||||
|
||||
ircClient.Nick(config.Irc.Nick)
|
||||
})
|
||||
|
||||
@@ -179,6 +289,7 @@ func ircClientRun() {
|
||||
log.Errorf("436 ERR_NICKCOLLISION, %s", e.Raw)
|
||||
})
|
||||
|
||||
// 437: ERR_UNAVAILRESOURCE уже есть в github.com/thoj/go-ircevent/irc_callback.go.
|
||||
ircClient.AddCallback("437", func(e *irc.Event) {
|
||||
log.Errorf("437 ERR_UNAVAILRESOURCE, %s", e.Raw)
|
||||
})
|
||||
@@ -191,6 +302,16 @@ func ircClientRun() {
|
||||
log.Warnf("442 ERR_NOTONCHANNEL, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("443", func(e *irc.Event) {
|
||||
// Returned when a client tries to invite a user to a channel they're already on.
|
||||
log.Errorf("443 ERR_USERONCHANNEL, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("446", func(e *irc.Event) {
|
||||
// Returned by USERS when it has been disabled or not implemented.
|
||||
log.Errorf("446 ERR_USERSDISABLED, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("451", func(e *irc.Event) {
|
||||
// Предполагается, что надо авторизоваться, перед тем как что-то делать на сервере
|
||||
log.Errorf("451 ERR_NOTREGISTERED, %s", e.Raw)
|
||||
@@ -213,18 +334,19 @@ func ircClientRun() {
|
||||
log.Errorf("465 ERR_YOUREBANNEDCREEP, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("467", func(e *irc.Event) {
|
||||
log.Errorf("467 ERR_KEYSET, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("471", func(e *irc.Event) {
|
||||
// Это значит, что народу на канале максимальное количество. Будем пробовать присунуться раз в 30 сек.
|
||||
channel := e.Arguments[1]
|
||||
log.Warnf("471 ERR_CHANNELISFULL, %s", e.Raw)
|
||||
time.Sleep(30 * time.Second)
|
||||
<-time.NewTimer(30 * time.Second).C
|
||||
|
||||
// Проверяем, а должны ли мы быть заджоенныеми к указанному, каналу, а то вдруг нет?
|
||||
for _, ircChan := range config.Irc.Channels {
|
||||
if ircChan == channel {
|
||||
ircClient.Join(channel)
|
||||
break
|
||||
}
|
||||
if slices.Contains(config.Irc.Channels, channel) {
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -237,14 +359,11 @@ func ircClientRun() {
|
||||
// По завершении работ +i снимают.
|
||||
channel := e.Arguments[1]
|
||||
log.Errorf("473 ERR_INVITEONLYCHAN, %s", e.Raw)
|
||||
time.Sleep(30 * time.Second)
|
||||
<-time.NewTimer(30 * time.Second).C
|
||||
|
||||
// Проверяем, а должны ли мы быть заджоенныеми к указанному, каналу, а то вдруг нет?
|
||||
for _, ircChan := range config.Irc.Channels {
|
||||
if ircChan == channel {
|
||||
ircClient.Join(channel)
|
||||
break
|
||||
}
|
||||
// Проверяем, а должны ли мы быть заджоенными к указанному, каналу, а то вдруг нет?
|
||||
if slices.Contains(config.Irc.Channels, channel) {
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -254,18 +373,21 @@ func ircClientRun() {
|
||||
|
||||
log.Errorf("474 ERR_BANNEDFROMCHAN, %s", e.Raw)
|
||||
// Если нас забанили, то информацию о MODE-ах пользователей мы теряем
|
||||
time.Sleep(30 * time.Second)
|
||||
<-time.NewTimer(30 * time.Second).C
|
||||
|
||||
// Вдруг, нас забанили, но какбэ не навсегда?
|
||||
// TODO: вынести в настройки?
|
||||
|
||||
// Проверяем, а должны ли мы быть заджоенныеми к указанному, каналу, а то вдруг нет?
|
||||
for _, ircChan := range config.Irc.Channels {
|
||||
if ircChan == channel {
|
||||
ircClient.Join(channel)
|
||||
break
|
||||
}
|
||||
if slices.Contains(config.Irc.Channels, channel) {
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
ircClient.AddCallback("475", func(e *irc.Event) {
|
||||
log.Errorf("475 ERR_BADCHANNELKEY, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("477", func(e *irc.Event) {
|
||||
log.Errorf("477 ERR_NOCHANMODES, %s", e.Raw)
|
||||
})
|
||||
@@ -274,6 +396,10 @@ func ircClientRun() {
|
||||
log.Errorf("478 ERR_BANLISTFULL, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("481", func(e *irc.Event) {
|
||||
log.Errorf("481 ERR_NOPRIVILEGES, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("482", func(e *irc.Event) {
|
||||
log.Errorf("482 ERR_CHANOPRIVSNEEDED, %s", e.Raw)
|
||||
})
|
||||
@@ -286,6 +412,10 @@ func ircClientRun() {
|
||||
log.Errorf("485 ERR_UNIQOPPRIVSNEEDED, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("491", func(e *irc.Event) {
|
||||
log.Errorf("491 ERR_NOOPERHOST, %s", e.Raw)
|
||||
})
|
||||
|
||||
ircClient.AddCallback("501", func(e *irc.Event) {
|
||||
log.Errorf("501 ERR_UMODEUNKNOWNFLAG, %s", e.Raw)
|
||||
})
|
||||
@@ -294,22 +424,6 @@ func ircClientRun() {
|
||||
log.Errorf("502 ERR_USERSDONTMATCH, %s", e.Raw)
|
||||
})
|
||||
|
||||
// Сделаем уже что-то полезное! Motd нам уже прислали и теперь можно авторизоваться и джойниться
|
||||
ircClient.AddCallback("001", func(e *irc.Event) {
|
||||
if !config.Irc.Sasl && config.Irc.Password != "" && !nickIsUsed {
|
||||
log.Info("Identifying via NickServ")
|
||||
message := fmt.Sprintf("identify %s %s", config.Irc.Nick, config.Irc.Password)
|
||||
imChan <- iMsg{ChatId: "NickServ", Text: message}
|
||||
}
|
||||
|
||||
log.Debug("Trying to join to preconfigured channels")
|
||||
|
||||
for _, channel := range config.Irc.Channels {
|
||||
log.Infof("Joining to %s channel", channel)
|
||||
ircClient.Join(channel)
|
||||
}
|
||||
})
|
||||
|
||||
// Навесим коллбэков на другие, интересные нам события
|
||||
ircClient.AddCallback("KICK", func(e *irc.Event) {
|
||||
dstNick := e.Arguments[1]
|
||||
@@ -321,7 +435,8 @@ func ircClientRun() {
|
||||
userMode.Delete(channel)
|
||||
// TODO: reason?
|
||||
log.Warnf("%s kicks us from %s", srcFullNick, channel)
|
||||
time.Sleep(10 * time.Second)
|
||||
<-time.NewTimer(10 * time.Second).C
|
||||
|
||||
ircClient.Join(channel)
|
||||
} else {
|
||||
// Кого-то другого кикнули с канала
|
||||
@@ -341,14 +456,15 @@ func ircClientRun() {
|
||||
log.Warn("I regain my nick")
|
||||
} else {
|
||||
log.Warn("I regain my nick, trying to identify myself via NickServ")
|
||||
|
||||
message := fmt.Sprintf("identify %s %s", config.Irc.Nick, config.Irc.Password)
|
||||
imChan <- iMsg{ChatId: "NickServ", Text: message}
|
||||
imChan <- iMsg{ChatID: "NickServ", Text: message}
|
||||
}
|
||||
} else {
|
||||
log.Infof("%s renames themself to %s", srcNick, dstNick)
|
||||
}
|
||||
|
||||
// Неважно чей ник сменился, надо забыть, что было и снова узнать mode-ы сменишего nick джентельмена
|
||||
// Неважно чей ник сменился, надо забыть, что было и снова узнать mode-ы сменишего nick джентельмена.
|
||||
// TODO: реализовать userModeRenameUser()
|
||||
userModePurgeUser(srcNick)
|
||||
ircClient.Whois(dstNick)
|
||||
@@ -360,12 +476,12 @@ func ircClientRun() {
|
||||
channel := e.Arguments[0]
|
||||
|
||||
if nick == ircClient.GetNick() {
|
||||
// Команда names отправляется автоматом.
|
||||
log.Infof("I joined to %s", channel)
|
||||
// Команда names отправляется автоматом
|
||||
} else {
|
||||
log.Infof("%s joined to %s", fullNick, channel)
|
||||
// Технически, тут не надо спрашивать whois на пользователя, но мы спрашиваем, чтобы уточнить mode,
|
||||
// вдруг сервер проставляет mode заранее (хотя не должен)
|
||||
// вдруг сервер проставляет mode заранее (хотя не должен).
|
||||
ircClient.Whois(nick)
|
||||
}
|
||||
})
|
||||
@@ -403,6 +519,7 @@ func ircClientRun() {
|
||||
channel := e.Arguments[0]
|
||||
fullSrcNick := e.Source
|
||||
srcNick := e.Nick
|
||||
|
||||
var dstNick string
|
||||
|
||||
if len(e.Arguments) >= 3 { // кого-то по-MODE-или на канале
|
||||
@@ -415,6 +532,7 @@ func ircClientRun() {
|
||||
}
|
||||
} else { // Установка mode-а при заходе на сервер
|
||||
dstNick = e.Arguments[0]
|
||||
|
||||
log.Infof("Server set my mode to %s", mode)
|
||||
}
|
||||
|
||||
@@ -454,17 +572,16 @@ func ircClientRun() {
|
||||
ircMsgParser(e.Arguments[0], e.Nick, e.User, e.Source, e.Arguments[1])
|
||||
})
|
||||
|
||||
/*
|
||||
ircClient.AddCallback("*", func(e *irc.Event) {
|
||||
log.Debugf("Incoming EVENT (Raw): %s", e.Raw)
|
||||
})
|
||||
*/
|
||||
ircClient.AddCallback("*", func(e *irc.Event) {
|
||||
log.Debugf("Incoming EVENT (Raw): %s", e.Raw)
|
||||
})
|
||||
|
||||
log.Debugf("Make an actual connection to irc")
|
||||
|
||||
if err := ircClient.Connect(serverString); err != nil {
|
||||
log.Fatalf("Unable to prepare irc connection: %s", err)
|
||||
time.Sleep(3 * time.Second)
|
||||
log.Errorf("Unable to prepare irc connection: %s", err)
|
||||
<-time.NewTimer(3 * time.Second).C // Sleep for 3 seconds.
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -478,21 +595,22 @@ func ircSend() {
|
||||
for {
|
||||
m := <-imChan
|
||||
|
||||
switch {
|
||||
case config.Irc.RateLimit.Type == "simple_delay":
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
switch config.Irc.RateLimit.Type {
|
||||
case "simple_delay":
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text)
|
||||
ircClient.Action(m.ChatID, m.Text)
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
|
||||
sleepDelay := time.Duration(config.Irc.RateLimit.SimpleDelay) * time.Millisecond
|
||||
log.Debugf("Due to delay type simple_delay waiting for %d milliseconds", int(sleepDelay))
|
||||
time.Sleep(sleepDelay)
|
||||
case config.Irc.RateLimit.Type == "token_bucket":
|
||||
<-time.NewTimer(sleepDelay).C
|
||||
case "token_bucket":
|
||||
var currentTimeMs = time.Now().UnixMilli()
|
||||
|
||||
var expirationTimeMs = config.Irc.RateLimit.TokenBucket.ExpirationTime * 1000
|
||||
|
||||
if len(msgBucket.Timestamps) > 0 {
|
||||
@@ -506,6 +624,7 @@ func ircSend() {
|
||||
|
||||
bucketFill := len(newBucket.Timestamps)
|
||||
log.Debugf("Message bucket filled with %d/%d messages", bucketFill, config.Irc.RateLimit.TokenBucket.Size)
|
||||
|
||||
newBucket.Timestamps = append(newBucket.Timestamps, currentTimeMs)
|
||||
|
||||
if bucketFill >= config.Irc.RateLimit.TokenBucket.Size {
|
||||
@@ -516,38 +635,41 @@ func ircSend() {
|
||||
} else {
|
||||
// Проставим время отправки сообщения
|
||||
log.Debug("Message bucket is empty")
|
||||
|
||||
msgBucket.Timestamps = append(msgBucket.Timestamps, currentTimeMs)
|
||||
}
|
||||
|
||||
if msgBucket.IsFull {
|
||||
log.Debug("Message bucket overflowed, hitting ratelimit")
|
||||
|
||||
sleepPeriod := expirationTimeMs - (currentTimeMs - msgBucket.Timestamps[len(msgBucket.Timestamps)-1])
|
||||
|
||||
if config.Irc.RateLimit.TokenBucket.Limit > 0 {
|
||||
sleepPeriod = sleepPeriod / int64(config.Irc.RateLimit.TokenBucket.Limit)
|
||||
sleepPeriod /= int64(config.Irc.RateLimit.TokenBucket.Limit)
|
||||
}
|
||||
|
||||
log.Debugf("Sleeping for %d milliseconds", sleepPeriod)
|
||||
time.Sleep(time.Duration(sleepPeriod) * time.Millisecond)
|
||||
<-time.NewTimer(time.Duration(sleepPeriod) * time.Millisecond).C
|
||||
|
||||
currentTimeMs = time.Now().UnixMilli()
|
||||
// Обновим время отправки сообщения
|
||||
msgBucket.Timestamps[len(msgBucket.Timestamps)-1] = currentTimeMs
|
||||
}
|
||||
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if len(m.Text) > 4 && m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text[4:])
|
||||
ircClient.Action(m.ChatID, m.Text[4:])
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
default:
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
log.Debugf("Sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if len(m.Text) > 4 && m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text[4:])
|
||||
ircClient.Action(m.ChatID, m.Text[4:])
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -556,12 +678,12 @@ func ircSend() {
|
||||
func ircSendUnrestricted() {
|
||||
for {
|
||||
m := <-imChanUnrestricted
|
||||
log.Debugf("Skipping ratelimit and sending to chat %s message: %s", m.ChatId, m.Text)
|
||||
log.Debugf("Skipping ratelimit and sending to chat %s message: %s", m.ChatID, m.Text)
|
||||
|
||||
if len(m.Text) > 4 && m.Text[0:4] == "/me " {
|
||||
ircClient.Action(m.ChatId, m.Text[4:])
|
||||
ircClient.Action(m.ChatID, m.Text[4:])
|
||||
} else {
|
||||
ircClient.Privmsg(m.ChatId, m.Text)
|
||||
ircClient.Privmsg(m.ChatID, m.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
35
globals.go
35
globals.go
@@ -4,47 +4,52 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/cockroachdb/pebble"
|
||||
"aleesa-irc-go/internal/anycollection"
|
||||
"aleesa-irc-go/internal/boolcollection"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
irc "github.com/thoj/go-ircevent"
|
||||
)
|
||||
|
||||
// Config - это у нас глобальная штука :)
|
||||
// Config - это у нас глобальная штука :).
|
||||
var config myConfig
|
||||
|
||||
// To break circular message forwarding we must set some sane default, it can be overridden via config
|
||||
// To break circular message forwarding we must set some sane default, it can be overridden via config.
|
||||
var forwardMax int64 = 5
|
||||
|
||||
// Объектики irc-клиента
|
||||
// Объектики irc-клиента.
|
||||
var ircClient *irc.Connection
|
||||
var nickIsUsed = false
|
||||
|
||||
// Объектики клиента-редиски
|
||||
// Объектики клиента-редиски.
|
||||
var redisClient *redis.Client
|
||||
var subscriber *redis.PubSub
|
||||
|
||||
// Main context
|
||||
// Main context.
|
||||
var ctx = context.Background()
|
||||
|
||||
// Ставится в true, если мы получили сигнал на выключение
|
||||
// Ставится в true, если мы получили сигнал на выключение.
|
||||
var shutdown = false
|
||||
|
||||
// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов
|
||||
// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов.
|
||||
var sigChan = make(chan os.Signal, 1)
|
||||
|
||||
// Канал, в который пишутся сообщения для отправки в IRC в обычном порядке
|
||||
// Канал, в который пишутся сообщения для отправки в IRC в обычном порядке.
|
||||
var imChan = make(chan iMsg, 10000)
|
||||
|
||||
// Канал, в который пишутся сообщения для отправки в IRC, если ограничений нет (у бота +o или +v на канале)
|
||||
// Канал, в который пишутся сообщения для отправки в IRC, если ограничений нет (у бота +o или +v на канале).
|
||||
var imChanUnrestricted = make(chan iMsg, 100)
|
||||
|
||||
// "Ведёрко" с timestamp-ами последних отправленных сообщений
|
||||
// "Ведёрко" с timestamp-ами последних отправленных сообщений.
|
||||
var msgBucket bucket
|
||||
|
||||
// Мапка с открытыми дескрипторами баз с настройками
|
||||
var settingsDB = make(map[string]*pebble.DB)
|
||||
// "Базюлька" с MODE-ами пользователей на каналах.
|
||||
var userMode = anycollection.NewCollection()
|
||||
|
||||
// "Базюлька" с MODE-ами пользователей на каналах
|
||||
var userMode = NewCollection()
|
||||
// "Базюлька" с доступными MODE-ами пользователя.
|
||||
var availableUserModes = boolcollection.NewCollection()
|
||||
|
||||
// "Базюлька" c с доступными MODE-ами каналов.
|
||||
var availableChanModes = boolcollection.NewCollection()
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
|
||||
55
go.mod
55
go.mod
@@ -1,40 +1,49 @@
|
||||
module aleesa-irc-go
|
||||
|
||||
go 1.19
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.2
|
||||
|
||||
require (
|
||||
github.com/cockroachdb/errors v1.9.0
|
||||
github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861
|
||||
github.com/cockroachdb/errors v1.11.3
|
||||
github.com/cockroachdb/pebble v1.1.5
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/hjson/hjson-go v3.3.0+incompatible
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/thoj/go-ircevent v0.0.0-20210723090443-73e444401d64
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/DataDog/zstd v1.5.2 // indirect
|
||||
github.com/DataDog/zstd v1.5.7 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
|
||||
github.com/cockroachdb/redact v1.1.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
github.com/cockroachdb/redact v1.1.6 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/getsentry/sentry-go v0.14.0 // indirect
|
||||
github.com/getsentry/sentry-go v0.31.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/klauspost/compress v1.15.12 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/prometheus/client_golang v1.13.1 // indirect
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.8.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.9.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect
|
||||
golang.org/x/net v0.1.0 // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/text v0.4.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
github.com/prometheus/client_golang v1.21.1 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.63.0 // indirect
|
||||
github.com/prometheus/procfs v0.16.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
|
||||
golang.org/x/net v0.38.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
google.golang.org/protobuf v1.36.6 // indirect
|
||||
)
|
||||
|
||||
tool (
|
||||
aleesa-irc-go
|
||||
aleesa-irc-go/internal/anycollection
|
||||
aleesa-irc-go/internal/boolcollection
|
||||
)
|
||||
|
||||
743
go.sum
743
go.sum
@@ -1,760 +1,139 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
|
||||
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
|
||||
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
|
||||
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
|
||||
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
|
||||
github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
|
||||
github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
|
||||
github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
|
||||
github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
|
||||
github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
|
||||
github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI=
|
||||
github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
|
||||
github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0=
|
||||
github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88=
|
||||
github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ=
|
||||
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74=
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861 h1:YarvrMA6pv032mYkBakTCkzqKpatcAZrdIknHJv+yVg=
|
||||
github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861/go.mod h1:qf9bLis2yy1XyNYD01wvIHPabuC1STzQsvGibYVsom4=
|
||||
github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
|
||||
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
|
||||
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 h1:pU88SPhIFid6/k0egdR5V6eALQYq2qbSmukrkgIh/0A=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILMqgNeV5jiqR4j+sTuvQNHdf2chuKj1M5k=
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
|
||||
github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw=
|
||||
github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo=
|
||||
github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
|
||||
github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
|
||||
github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
|
||||
github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c=
|
||||
github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70=
|
||||
github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I=
|
||||
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
|
||||
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4=
|
||||
github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI=
|
||||
github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
|
||||
github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
|
||||
github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
|
||||
github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
|
||||
github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
|
||||
github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
|
||||
github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
|
||||
github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
|
||||
github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
|
||||
github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk=
|
||||
github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=
|
||||
github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
|
||||
github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=
|
||||
github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
|
||||
github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
|
||||
github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0=
|
||||
github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
|
||||
github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
|
||||
github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
|
||||
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
|
||||
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
|
||||
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
|
||||
github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
|
||||
github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM=
|
||||
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
|
||||
github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
|
||||
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
|
||||
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
|
||||
github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c=
|
||||
github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
|
||||
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
|
||||
github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
|
||||
github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
|
||||
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk=
|
||||
github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k=
|
||||
github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18=
|
||||
github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM=
|
||||
github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
|
||||
github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/thoj/go-ircevent v0.0.0-20210723090443-73e444401d64 h1:l/T7dYuJEQZOwVOpjIXr1180aM9PZL/d1MnMVIxefX4=
|
||||
github.com/thoj/go-ircevent v0.0.0-20210723090443-73e444401d64/go.mod h1:Q1NAJOuRdQCqN/VIWdnaaEhV8LpeO2rtlBP7/iDJNII=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
|
||||
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
|
||||
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
|
||||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
|
||||
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
|
||||
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
|
||||
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
|
||||
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
|
||||
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
|
||||
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
package main
|
||||
package anycollection
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Collection это sync.Map-ка, хранящая в себе произвольные итемы
|
||||
// Collection это sync.Map-ка, хранящая в себе произвольные итемы.
|
||||
type Collection struct {
|
||||
items sync.Map
|
||||
close chan struct{}
|
||||
}
|
||||
|
||||
// Item представляет собой интерфейс для хранения произвольных данных
|
||||
// Item представляет собой интерфейс для хранения произвольных данных.
|
||||
type item struct {
|
||||
data interface{}
|
||||
data any
|
||||
}
|
||||
|
||||
// NewCollection создаёт инстанс коллекции
|
||||
// NewCollection создаёт инстанс коллекции.
|
||||
func NewCollection() *Collection {
|
||||
c := &Collection{
|
||||
close: make(chan struct{}),
|
||||
@@ -24,8 +24,8 @@ func NewCollection() *Collection {
|
||||
return c
|
||||
}
|
||||
|
||||
// Get достаёт данные по заданному ключу из коллекции
|
||||
func (collection *Collection) Get(key interface{}) (interface{}, bool) {
|
||||
// Get достаёт данные по заданному ключу из коллекции.
|
||||
func (collection *Collection) Get(key any) (any, bool) {
|
||||
obj, exists := collection.items.Load(key)
|
||||
|
||||
if !exists {
|
||||
@@ -37,16 +37,16 @@ func (collection *Collection) Get(key interface{}) (interface{}, bool) {
|
||||
return item.data, true
|
||||
}
|
||||
|
||||
// Set сохраняет данные с заданным ключом в коллекцию
|
||||
func (collection *Collection) Set(key interface{}, value interface{}) {
|
||||
// Set сохраняет данные с заданным ключом в коллекцию.
|
||||
func (collection *Collection) Set(key any, value any) {
|
||||
collection.items.Store(key, item{
|
||||
data: value,
|
||||
})
|
||||
}
|
||||
|
||||
// Range применяет функцию f ко всем ключам в коллекции
|
||||
func (collection *Collection) Range(f func(key, value interface{}) bool) {
|
||||
fn := func(key, value interface{}) bool {
|
||||
// Range применяет функцию f ко всем ключам в коллекции.
|
||||
func (collection *Collection) Range(f func(key, value any) bool) {
|
||||
fn := func(key, value any) bool {
|
||||
item := value.(item)
|
||||
|
||||
return f(key, item.data)
|
||||
@@ -55,12 +55,12 @@ func (collection *Collection) Range(f func(key, value interface{}) bool) {
|
||||
collection.items.Range(fn)
|
||||
}
|
||||
|
||||
// Delete удаляет ключ и значение из коллекции данных
|
||||
func (collection *Collection) Delete(key interface{}) {
|
||||
// Delete удаляет ключ и значение из коллекции данных.
|
||||
func (collection *Collection) Delete(key any) {
|
||||
collection.items.Delete(key)
|
||||
}
|
||||
|
||||
// Close очищает и высвобождает ресурсы, занятые коллекцией
|
||||
// Close очищает и высвобождает ресурсы, занятые коллекцией.
|
||||
func (collection *Collection) Close() {
|
||||
collection.close <- struct{}{}
|
||||
collection.items = sync.Map{}
|
||||
58
internal/boolcollection/boolcollection.go
Normal file
58
internal/boolcollection/boolcollection.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package boolcollection
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Collection это sync.Map-ка, хранящая в себе итемы типа bool.
|
||||
type Collection struct {
|
||||
items sync.Map
|
||||
close chan struct{}
|
||||
}
|
||||
|
||||
// Item представляет собой интерфейс для хранения данных типа bool.
|
||||
type item struct {
|
||||
data bool
|
||||
}
|
||||
|
||||
// NewCollection создаёт инстанс коллекции.
|
||||
func NewCollection() *Collection {
|
||||
c := &Collection{
|
||||
close: make(chan struct{}),
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// Get достаёт данные по заданному ключу из коллекции.
|
||||
func (collection *Collection) Get(key string) (bool, bool) {
|
||||
obj, exists := collection.items.Load(key)
|
||||
|
||||
if !exists {
|
||||
return false, false
|
||||
}
|
||||
|
||||
item := obj.(item)
|
||||
|
||||
return item.data, true
|
||||
}
|
||||
|
||||
// Set сохраняет данные с заданным ключом в коллекцию.
|
||||
func (collection *Collection) Set(key string, value bool) {
|
||||
collection.items.Store(key, item{
|
||||
data: value,
|
||||
})
|
||||
}
|
||||
|
||||
// Delete удаляет ключ и значение из коллекции данных.
|
||||
func (collection *Collection) Delete(key string) {
|
||||
collection.items.Delete(key)
|
||||
}
|
||||
|
||||
// Close очищает и высвобождает ресурсы, занятые коллекцией.
|
||||
func (collection *Collection) Close() {
|
||||
collection.close <- struct{}{}
|
||||
collection.items = sync.Map{}
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package setting
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
@@ -11,9 +11,14 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// StoreKV сохраняет в указанной бд ключ и значение
|
||||
// Мапка с открытыми дескрипторами баз с настройками.
|
||||
var settingsDB = make(map[string]*pebble.DB)
|
||||
var DataDir string
|
||||
|
||||
// StoreKV сохраняет в указанной бд ключ и значение.
|
||||
func StoreKV(db *pebble.DB, key string, value string) error {
|
||||
var kArray = []byte(key)
|
||||
|
||||
var vArray = []byte(value)
|
||||
|
||||
err := db.Set(kArray, vArray, pebble.Sync)
|
||||
@@ -21,11 +26,12 @@ func StoreKV(db *pebble.DB, key string, value string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// FetchV достаёт значение по ключу
|
||||
// FetchV достаёт значение по ключу.
|
||||
func FetchV(db *pebble.DB, key string) (string, error) {
|
||||
var kArray = []byte(key)
|
||||
|
||||
var vArray []byte
|
||||
|
||||
var valueString = ""
|
||||
|
||||
vArray, closer, err := db.Get(kArray)
|
||||
@@ -40,8 +46,8 @@ func FetchV(db *pebble.DB, key string) (string, error) {
|
||||
return valueString, err
|
||||
}
|
||||
|
||||
// Достанем настройку из БД с настройками
|
||||
func getSetting(chatID string, setting string) string {
|
||||
// Достанем настройку из БД с настройками.
|
||||
func Get(chatID string, setting string) string {
|
||||
var err error
|
||||
|
||||
chatHash := sha256.Sum256([]byte(chatID))
|
||||
@@ -54,10 +60,11 @@ func getSetting(chatID string, setting string) string {
|
||||
// настройками порождается огромное количество файлов. Умолчальное ограничение на количество файлов - 500 штук,
|
||||
// что нас не устраивает, поэтому немного снизим эту цифру до более приемлемых значений
|
||||
options.L0CompactionFileThreshold = 8
|
||||
settingsDB[database], err = pebble.Open(config.DataDir+"/"+database, &options)
|
||||
settingsDB[database], err = pebble.Open(DataDir+"/"+database, &options)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("Unable to open settings db, %s: %s\n", database, err)
|
||||
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -66,13 +73,14 @@ func getSetting(chatID string, setting string) string {
|
||||
|
||||
// Если из базы ничего не вынулось, по каким-то причинам, то просто вернём пустую строку
|
||||
if err != nil {
|
||||
if errors.Is(err, pebble.ErrNotFound) {
|
||||
switch {
|
||||
case errors.Is(err, pebble.ErrNotFound):
|
||||
log.Debugf("Unable to get value for %s: no record found in db %s", setting, database)
|
||||
} else if errors.Is(err, fs.ErrNotExist) {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
log.Debugf("Unable to get value for %s: db dir %s does not exist", setting, database)
|
||||
} else if errors.Is(err, oserror.ErrNotExist) {
|
||||
case errors.Is(err, oserror.ErrNotExist):
|
||||
log.Debugf("Unable to get value for %s: db dir %s does not exist", setting, database)
|
||||
} else {
|
||||
default:
|
||||
log.Errorf("Unable to get value for %s in db dir %s: %s", setting, database, err)
|
||||
}
|
||||
|
||||
@@ -82,10 +90,12 @@ func getSetting(chatID string, setting string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
// Сохраним настройку в БД с настройками
|
||||
func saveSetting(chatID string, setting string, value string) error {
|
||||
// Сохраним настройку в БД с настройками.
|
||||
func Set(chatID string, setting string, value string) error {
|
||||
var chatHash = sha256.Sum256([]byte(chatID))
|
||||
|
||||
var database = fmt.Sprintf("settings_db/%x", chatHash)
|
||||
|
||||
var err error
|
||||
|
||||
// Если БД не открыта, откроем её
|
||||
@@ -96,10 +106,11 @@ func saveSetting(chatID string, setting string, value string) error {
|
||||
// что нас не устраивает, поэтому немного снизим эту цифру до более приемлемых значений
|
||||
options.L0CompactionFileThreshold = 8
|
||||
|
||||
settingsDB[database], err = pebble.Open(config.DataDir+"/"+database, &options)
|
||||
settingsDB[database], err = pebble.Open(DataDir+"/"+database, &options)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("Unable to open settings db, %s: %s\n", database, err)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -111,4 +122,15 @@ func saveSetting(chatID string, setting string, value string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Закрываем все инстансы баз с настройками.
|
||||
func CloseAll() {
|
||||
if len(settingsDB) > 0 {
|
||||
log.Debug("Closing runtime irc channel settings db")
|
||||
|
||||
for _, db := range settingsDB {
|
||||
_ = db.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
@@ -8,7 +8,7 @@ import "strings"
|
||||
активируют мягкий ratelimit на стороне сервера, без них, сервер просто рвёт соединение по превышению лимитов
|
||||
*/
|
||||
|
||||
// Удаляет сведения о MODE-ах пользователя для заданного канала
|
||||
// Удаляет сведения о MODE-ах пользователя для заданного канала.
|
||||
func userModeDeleteUser(ircChan string, nick string) {
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
@@ -24,16 +24,17 @@ func userModeDeleteUser(ircChan string, nick string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Удаляет сведения о MODE-ах пользователя для всех каналов, на которых есть бот
|
||||
// Удаляет сведения о MODE-ах пользователя для всех каналов, на которых есть бот.
|
||||
func userModePurgeUser(nick string) {
|
||||
for _, channel := range config.Irc.Channels {
|
||||
userModeDeleteUser(channel, nick)
|
||||
}
|
||||
}
|
||||
|
||||
// Обновляет или создаёт запись о mode-ах пользователей
|
||||
// Обновляет или создаёт запись о mode-ах пользователей.
|
||||
func userModeUpdateUser(ircChan string, nick string, modes string) {
|
||||
var channelData map[string]map[string]bool
|
||||
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
@@ -48,12 +49,12 @@ func userModeUpdateUser(ircChan string, nick string, modes string) {
|
||||
channelData[nick] = make(map[string]bool)
|
||||
}
|
||||
|
||||
switch {
|
||||
case modes[:1] == "+":
|
||||
switch modes[:1] {
|
||||
case "+":
|
||||
for _, mode := range strings.Split(modes[1:], "") {
|
||||
channelData[nick][mode] = true
|
||||
}
|
||||
case modes[:1] == "-":
|
||||
case "-":
|
||||
for _, mode := range strings.Split(modes[1:], "") {
|
||||
channelData[nick][mode] = false
|
||||
}
|
||||
@@ -62,9 +63,10 @@ func userModeUpdateUser(ircChan string, nick string, modes string) {
|
||||
userMode.Set(ircChan, channelData)
|
||||
}
|
||||
|
||||
// Возвращает значение MODE-а v для запрошенного ника
|
||||
// Возвращает значение MODE-а v для запрошенного ника.
|
||||
func userModeIsVoiced(ircChan string, nick string) bool {
|
||||
var channelData map[string]map[string]bool
|
||||
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
@@ -88,9 +90,10 @@ func userModeIsVoiced(ircChan string, nick string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Возвращает значение MODE-а o для запрошенного ника
|
||||
// Возвращает значение MODE-а o для запрошенного ника.
|
||||
func userModeIsOped(ircChan string, nick string) bool {
|
||||
var channelData map[string]map[string]bool
|
||||
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
@@ -116,6 +119,7 @@ func userModeIsOped(ircChan string, nick string) bool {
|
||||
|
||||
func userModeIsHere(ircChan string, nick string) bool {
|
||||
var channelData map[string]map[string]bool
|
||||
|
||||
channel, ok := userMode.Get(ircChan)
|
||||
|
||||
if ok {
|
||||
|
||||
8
main.go
8
main.go
@@ -7,11 +7,13 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"aleesa-irc-go/internal/setting"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Производит некоторую инициализацию перед запуском main()
|
||||
// Производит некоторую инициализацию перед запуском main().
|
||||
func init() {
|
||||
log.SetFormatter(&log.TextFormatter{
|
||||
DisableQuote: true,
|
||||
@@ -50,9 +52,11 @@ func init() {
|
||||
|
||||
userMode.Set("channel", usersModes)
|
||||
userMode.Delete("channel")
|
||||
|
||||
setting.DataDir = config.DataDir
|
||||
}
|
||||
|
||||
// Собственно, какбэ "точка входа" - основная процедура в нашем боте
|
||||
// Собственно, какбэ "точка входа" - основная процедура в нашем боте.
|
||||
func main() {
|
||||
// Main context
|
||||
var ctx = context.Background()
|
||||
|
||||
180
msg-parsers.go
180
msg-parsers.go
@@ -6,11 +6,13 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"aleesa-irc-go/internal/setting"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ircMsgParser парсит сообщения, прилетевшие из IRC-ки
|
||||
func ircMsgParser(channel string, nick string, user string, source string, msg string) {
|
||||
// ircMsgParser парсит сообщения, прилетевшие из IRC-ки.
|
||||
func ircMsgParser(channel string, nick string, user string, source string, msg string) { //nolint: revive
|
||||
// nick - это выбранный пользователем nick (если он занят, то его "нарисует" сервер)
|
||||
// user - это короткое имя пользователя, под которым его видит сервер
|
||||
// source - это длинное имя пользователя, оно содержит в себе помимо user, ещё и ip с которого пришёл пользователь
|
||||
@@ -48,68 +50,68 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
switch {
|
||||
case cmd == "help" || msg == "помощь":
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%shelp | %sпомощь - это сообщение", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sanek | %sанек | %sанекдот - рандомный анекдот с anekdot.ru", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sbuni - комикс-стрип hapi buni", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sbunny - кролик", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%srabbit | %sкролик - кролик", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%scat | %sкис - кошечка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sdice | %sroll | %sкости - бросить кости", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sdig | %sкопать - заняться археологией", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sdrink | %sпраздник - какой сегодня праздник?", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfish | %sfisher - порыбачить", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sрыба | %sрыбка | %sрыбалка - порыбачить", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sf | %sф - рандомная фраза из сборника цитат fortune_mod", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfortune | %sфортунка - рандомная фраза из сборника цитат fortune_mod", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfox | %sлис - лисичка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfriday | %sпятница - а не пятница ли сегодня?", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sfrog | %sлягушка - лягушка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%shorse | %sлошадь | %sлошадка - лошадка", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%skarma фраза - посмотреть карму фразы", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sкарма фраза - посмотреть карму фразы", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintln("фраза++ | фраза-- - повысить или понизить карму фразы")}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%slat | %sлат - сгенерировать фразу из крылатого латинского выражения", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%smonkeyuser - комикс-стрип MonkeyUser", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sowl | %sсова - сова", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sping | %sпинг - попинговать бота", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sproverb | %sпословица - рандомная русская пословица", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%ssnail | %sулитка - улитка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%ssome_brew - выдать соответствующий напиток, бармен может налить rum, ром, vodka, водку, tequila, текила, whisky, виски, absinthe, абсент", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sver | %sversion - написать что-то про версию ПО", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sверсия - написать что-то про версию ПО", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sw город | %sп город - погода в городе", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sxkcd - комикс-стрип с xkcb.ru", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%shelp | %sпомощь - это сообщение", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sanek | %sанек | %sанекдот - рандомный анекдот с anekdot.ru", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sbuni - комикс-стрип hapi buni", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sbunny - кролик", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%srabbit | %sкролик - кролик", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%scat | %sкис - кошечка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sdice | %sroll | %sкости - бросить кости", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sdig | %sкопать - заняться археологией", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sdrink | %sпраздник - какой сегодня праздник?", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfish | %sfisher - порыбачить", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sрыба | %sрыбка | %sрыбалка - порыбачить", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sf | %sф - рандомная фраза из сборника цитат fortune_mod", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfortune | %sфортунка - рандомная фраза из сборника цитат fortune_mod", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfox | %sлис - лисичка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfriday | %sпятница - а не пятница ли сегодня?", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sfrog | %sлягушка - лягушка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%shorse | %sлошадь | %sлошадка - лошадка", config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%skarma фраза - посмотреть карму фразы", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sкарма фраза - посмотреть карму фразы", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintln("фраза++ | фраза-- - повысить или понизить карму фразы")}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%slat | %sлат - сгенерировать фразу из крылатого латинского выражения", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%smonkeyuser - комикс-стрип MonkeyUser", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sowl | %sсова - сова", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sping | %sпинг - попинговать бота", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sproverb | %sпословица - рандомная русская пословица", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%ssnail | %sулитка - улитка", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%ssome_brew - выдать соответствующий напиток, бармен может налить rum, ром, vodka, водку, tequila, текила, whisky, виски, absinthe, абсент", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sver | %sversion - написать что-то про версию ПО", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sверсия - написать что-то про версию ПО", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sw <город> | %sп <город> - погода в городе", config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sxkcd - комикс-стрип с xkcb.ru", config.Csign)}
|
||||
|
||||
if userModeIsOped(channel, nick) {
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin - настройки некоторых плагинов бота для канала", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin - настройки некоторых плагинов бота для канала", config.Csign)}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
case cmd == "admin":
|
||||
if userModeIsOped(channel, nick) {
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin oboobs # - где 1 - вкл, 0 - выкл плагина oboobs", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin oboobs показываем ли сисечки по просьбе участников чата (команды %stits, %stities, %sboobs, %sboobies, %sсиси, %sсисечки)", config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin obutts # - где 1 - вкл, 0 - выкл плагина obutts", config.Csign)}
|
||||
imChan <- iMsg{ChatId: nick, Text: fmt.Sprintf("%sadmin obutts показываем ли попки по просьбе участников чата (команды %sass, %sbutt, %sbooty, %sпопа, %sпопка)", config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin oboobs # - где 1 - вкл, 0 - выкл плагина oboobs", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin oboobs показываем ли сисечки по просьбе участников чата (команды %stits, %stities, %sboobs, %sboobies, %sсиси, %sсисечки)", config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin obutts # - где 1 - вкл, 0 - выкл плагина obutts", config.Csign)}
|
||||
imChan <- iMsg{ChatID: nick, Text: fmt.Sprintf("%sadmin obutts показываем ли попки по просьбе участников чата (команды %sass, %sbutt, %sbooty, %sпопа, %sпопка)", config.Csign, config.Csign, config.Csign, config.Csign, config.Csign, config.Csign)}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
case cmd == "admin oboobs":
|
||||
if userModeIsOped(channel, nick) {
|
||||
value := getSetting(channel, "oboobs")
|
||||
value := setting.Get(channel, "oboobs")
|
||||
|
||||
switch {
|
||||
case value == "":
|
||||
_ = saveSetting(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
case value == "0":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
case value == "1":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs включен"}
|
||||
switch value {
|
||||
case "":
|
||||
_ = setting.Set(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
case "0":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
case "1":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs включен"}
|
||||
default:
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,12 +119,12 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin oboobs 1":
|
||||
if userModeIsOped(channel, nick) {
|
||||
err := saveSetting(channel, "oboobs", "1")
|
||||
err := setting.Set(channel, "oboobs", "1")
|
||||
|
||||
if err != nil {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs всё ещё выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs всё ещё выключен"}
|
||||
} else {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs включен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs включен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,26 +132,26 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin oboobs 0":
|
||||
if userModeIsOped(channel, nick) {
|
||||
_ = saveSetting(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин oboobs выключен"}
|
||||
_ = setting.Set(channel, "oboobs", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин oboobs выключен"}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
case cmd == "admin obutts":
|
||||
if userModeIsOped(channel, nick) {
|
||||
value := getSetting(channel, "obutts")
|
||||
value := setting.Get(channel, "obutts")
|
||||
|
||||
switch {
|
||||
case value == "":
|
||||
_ = saveSetting(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
case value == "0":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
case value == "1":
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts включен"}
|
||||
switch value {
|
||||
case "":
|
||||
_ = setting.Set(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
case "0":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
case "1":
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts включен"}
|
||||
default:
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,12 +159,12 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin obutts 1":
|
||||
if userModeIsOped(channel, nick) {
|
||||
err := saveSetting(channel, "obutts", "1")
|
||||
err := setting.Set(channel, "obutts", "1")
|
||||
|
||||
if err != nil {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts всё ещё выключен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts всё ещё выключен"}
|
||||
} else {
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts включен"}
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts включен"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,8 +172,8 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
case cmd == "admin obutts 0":
|
||||
if userModeIsOped(channel, nick) {
|
||||
_ = saveSetting(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatId: nick, Text: "Плагин obutts выключен"}
|
||||
_ = setting.Set(channel, "obutts", "0")
|
||||
imChan <- iMsg{ChatID: nick, Text: "Плагин obutts выключен"}
|
||||
}
|
||||
|
||||
return
|
||||
@@ -182,15 +184,17 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
// Общий список простых команд
|
||||
cmds := []string{"ping", "пинг", "пинх", "pong", "понг", "понх", "coin", "монетка", "roll", "dice", "кости",
|
||||
"ver", "version", "версия", "хэлп", "halp", "kde", "кде", "lat", "лат", "friday", "пятница", "proverb",
|
||||
"пословица", "fortune", "фортунка", "f", "ф", "anek", "анек", "анекдот", "buni", "cat", "кис", "drink",
|
||||
"праздник", "fox", "лис", "frog", "лягушка", "horse", "лошадь", "лошадка", "monkeyuser", "owl", "сова",
|
||||
"сыч", "rabbit", "bunny", "кролик", "snail", "улитка", "xkcd", "dig", "копать", "fish", "fishing", "рыба",
|
||||
"рыбка", "рыбалка", "karma", "карма"}
|
||||
"пословица", "пословиться", "fortune", "фортунка", "f", "ф", "anek", "анек", "анекдот", "buni", "cat",
|
||||
"кис", "drink", "праздник", "fox", "лис", "frog", "лягушка", "horse", "лошадь", "лошадка", "monkeyuser",
|
||||
"owl", "сова", "сыч", "rabbit", "bunny", "кролик", "snail", "улитка", "xkcd", "dig", "копать", "fish",
|
||||
"fishing", "рыба", "рыбка", "рыбалка", "karma", "карма", "w", "п", "weather", "погода", "погодка",
|
||||
"погадка"}
|
||||
|
||||
for _, command := range cmds {
|
||||
if cmd == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -205,6 +209,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmd == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -225,7 +230,8 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
message.Misc.Username = strings.TrimSpace(pile[1])
|
||||
} else {
|
||||
msg = fmt.Sprintf("Я тут не вижу участника с ником %s", userNick)
|
||||
imChan <- iMsg{ChatId: channel, Text: msg}
|
||||
imChan <- iMsg{ChatID: channel, Text: msg}
|
||||
|
||||
return
|
||||
}
|
||||
} // в противном случае юзер просто поставил пробел на конце команды, интерпретируем это как
|
||||
@@ -233,6 +239,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
outgoingMessage = msg
|
||||
done = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -249,6 +256,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmdLen > len(command) && cmd[0:len(command)] == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -256,7 +264,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
// Отключаемые команды
|
||||
if !done {
|
||||
value := getSetting(channel, "obutts")
|
||||
value := setting.Get(channel, "obutts")
|
||||
|
||||
if value == "1" {
|
||||
cmds := []string{"butt", "booty", "ass", "попа", "попка"}
|
||||
@@ -266,13 +274,14 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmd[:cmdLen] == command {
|
||||
done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !done {
|
||||
value = getSetting(channel, "oboobs")
|
||||
value = setting.Get(channel, "oboobs")
|
||||
|
||||
if value == "1" {
|
||||
cmds := []string{"tits", "boobs", "tities", "boobies", "сиси", "сисечки"}
|
||||
@@ -282,6 +291,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
if cmd[:cmdLen] == command {
|
||||
// done = true
|
||||
outgoingMessage = msg
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -296,6 +306,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Unable to to serialize message for redis: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -350,6 +361,7 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("Unable to to serialize message for redis: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -360,11 +372,9 @@ func ircMsgParser(channel string, nick string, user string, source string, msg s
|
||||
log.Debugf("Sent msg to redis channel %s: %s", config.Redis.Channel, string(data))
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// redisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными
|
||||
// redisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными.
|
||||
func redisMsgParser(msg string) {
|
||||
if shutdown {
|
||||
// Если мы завершаем работу программы, то нам ничего обрабатывать не надо
|
||||
@@ -377,43 +387,51 @@ func redisMsgParser(msg string) {
|
||||
|
||||
if err := json.Unmarshal([]byte(msg), &j); err != nil {
|
||||
log.Warnf("Unable to to parse message from redis channel: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Validate our j
|
||||
if exist := j.From; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no from field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Chatid; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no chatid field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Userid; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no userid field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Message; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no message field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Plugin; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no plugin field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if exist := j.Mode; exist == "" {
|
||||
log.Warnf("Incorrect msg from redis, no mode field: %s", msg)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// j.Misc.Answer может и не быть, тогда ответа на такое сообщение не будет
|
||||
if j.Misc.Answer == 0 {
|
||||
log.Debug("Field Misc->Answer = 0, skipping message")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -437,13 +455,11 @@ func redisMsgParser(msg string) {
|
||||
|
||||
for _, message := range lines {
|
||||
if userModeIsOped(j.Chatid, ircClient.GetNick()) || userModeIsVoiced(j.Chatid, ircClient.GetNick()) {
|
||||
imChanUnrestricted <- iMsg{ChatId: j.Chatid, Text: message}
|
||||
imChanUnrestricted <- iMsg{ChatID: j.Chatid, Text: message}
|
||||
} else {
|
||||
imChan <- iMsg{ChatId: j.Chatid, Text: message}
|
||||
imChan <- iMsg{ChatID: j.Chatid, Text: message}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */
|
||||
|
||||
13
types.go
13
types.go
@@ -1,6 +1,6 @@
|
||||
package main
|
||||
|
||||
// Конфиг
|
||||
// Конфиг.
|
||||
type myConfig struct {
|
||||
Redis struct {
|
||||
Server string `json:"server,omitempty"`
|
||||
@@ -35,7 +35,7 @@ type myConfig struct {
|
||||
DataDir string `json:"data_dir,omitempty"`
|
||||
}
|
||||
|
||||
// Входящее сообщение из pubsub-канала redis-ки
|
||||
// Входящее сообщение из pubsub-канала redis-ки.
|
||||
type rMsg struct {
|
||||
From string `json:"from,omitempty"`
|
||||
Chatid string `json:"chatid,omitempty"`
|
||||
@@ -55,7 +55,7 @@ type rMsg struct {
|
||||
} `json:"Misc"`
|
||||
}
|
||||
|
||||
// Исходящее сообщение в pubsub-канал redis-ки
|
||||
// Исходящее сообщение в pubsub-канал redis-ки.
|
||||
type sMsg struct {
|
||||
From string `json:"from"`
|
||||
Chatid string `json:"chatid"`
|
||||
@@ -75,14 +75,13 @@ type sMsg struct {
|
||||
} `json:"misc"`
|
||||
}
|
||||
|
||||
// Сообщение в irc
|
||||
// TODO: Имплементировать тип сообщения - privmsg или action
|
||||
// TODO: Имплементировать тип сообщения - privmsg или action.
|
||||
type iMsg struct {
|
||||
ChatId string
|
||||
ChatID string
|
||||
Text string
|
||||
}
|
||||
|
||||
// "Ведёрко" для реализации ограничителя скорости отправки исходящих сообщений - Token Bucket
|
||||
// "Ведёрко" для реализации ограничителя скорости отправки исходящих сообщений - Token Bucket.
|
||||
type bucket struct {
|
||||
Timestamps []int64
|
||||
IsFull bool
|
||||
|
||||
32
util.go
32
util.go
@@ -3,16 +3,17 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"aleesa-irc-go/internal/setting"
|
||||
|
||||
"github.com/hjson/hjson-go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Читает и валидирует конфиг, а также выставляет некоторые default-ы, если значений для параметров в конфиге нет
|
||||
// Читает и валидирует конфиг, а также выставляет некоторые default-ы, если значений для параметров в конфиге нет.
|
||||
func readConfig() {
|
||||
configLoaded := false
|
||||
executablePath, err := os.Executable()
|
||||
@@ -41,14 +42,16 @@ func readConfig() {
|
||||
// Конфиг-файл длинноват для конфига, попробуем следующего кандидата
|
||||
if fileInfo.Size() > 65535 {
|
||||
log.Warnf("Config file %s is too long for config, skipping", location)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadFile(location)
|
||||
buf, err := os.ReadFile(location)
|
||||
|
||||
// Не удалось прочитать, попробуем следующего кандидата
|
||||
if err != nil {
|
||||
log.Warnf("Skip reading config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -56,12 +59,14 @@ func readConfig() {
|
||||
// Интереснее на выходе получить структурку: то есть мы вначале конфиг преобразуем в map-ку, затем эту map-ку
|
||||
// сериализуем в json, а потом json преврщааем в стркутурку. Не очень эффективно, но он и не часто требуется.
|
||||
var sampleConfig myConfig
|
||||
|
||||
var tmp map[string]interface{}
|
||||
err = hjson.Unmarshal(buf, &tmp)
|
||||
|
||||
// Не удалось распарсить - попробуем следующего кандидата
|
||||
if err != nil {
|
||||
log.Warnf("Skip parsing config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -70,17 +75,20 @@ func readConfig() {
|
||||
// Не удалось преобразовать map-ку в json
|
||||
if err != nil {
|
||||
log.Warnf("Skip parsing config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(tmpjson, &sampleConfig); err != nil {
|
||||
log.Warnf("Skip parsing config file %s: %s", location, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Валидируем значения из конфига
|
||||
if sampleConfig.Redis.Server == "" {
|
||||
sampleConfig.Redis.Server = "localhost"
|
||||
|
||||
log.Infof("Redis server is not defined in config, using localhost")
|
||||
}
|
||||
|
||||
@@ -101,15 +109,17 @@ func readConfig() {
|
||||
// Значения для IRC-клиента
|
||||
if sampleConfig.Irc.Server == "" {
|
||||
sampleConfig.Irc.Server = "localhost"
|
||||
|
||||
log.Errorf("Irc server is not defined in config, using localhost")
|
||||
}
|
||||
|
||||
if sampleConfig.Irc.Port == 0 {
|
||||
sampleConfig.Irc.Port = 6667
|
||||
|
||||
log.Infof("Irc port is not defined in config, using 6667")
|
||||
}
|
||||
|
||||
if sampleConfig.Irc.Ssl != true {
|
||||
if !sampleConfig.Irc.Ssl {
|
||||
sampleConfig.Irc.SslVerify = false
|
||||
}
|
||||
|
||||
@@ -182,7 +192,9 @@ func readConfig() {
|
||||
|
||||
config = sampleConfig
|
||||
configLoaded = true
|
||||
|
||||
log.Infof("Using %s as config file", location)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
@@ -192,7 +204,7 @@ func readConfig() {
|
||||
}
|
||||
}
|
||||
|
||||
// Хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения
|
||||
// Хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения.
|
||||
func sigHandler() {
|
||||
var err error
|
||||
|
||||
@@ -229,16 +241,12 @@ func sigHandler() {
|
||||
|
||||
log.Debug("Close irc connection")
|
||||
ircClient.Quit()
|
||||
|
||||
log.Debug("Close userModeDB")
|
||||
userMode.Close()
|
||||
|
||||
if len(settingsDB) > 0 {
|
||||
log.Debug("Closing runtime irc channel settings db")
|
||||
|
||||
for _, db := range settingsDB {
|
||||
_ = db.Close()
|
||||
}
|
||||
}
|
||||
log.Debug("Close settingsDB")
|
||||
setting.CloseAll()
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
6
vendor/github.com/DataDog/zstd/README.md
generated
vendored
6
vendor/github.com/DataDog/zstd/README.md
generated
vendored
@@ -6,8 +6,8 @@
|
||||
|
||||
[C Zstd Homepage](https://github.com/facebook/zstd)
|
||||
|
||||
The current headers and C files are from *v1.5.0* (Commit
|
||||
[10f0e699](https://github.com/facebook/zstd/releases/tag/v1.5.0)).
|
||||
The current headers and C files are from *v1.5.7* (Commit
|
||||
[f8745da](https://github.com/facebook/zstd/releases/tag/v1.5.7)).
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -41,7 +41,7 @@ go build -tags external_libzstd
|
||||
// Compress compresses the byte array given in src and writes it to dst.
|
||||
// If you already have a buffer allocated, you can pass it to prevent allocation
|
||||
// If not, you can pass nil as dst.
|
||||
// If the buffer is too small, it will be reallocated, resized, and returned bu the function
|
||||
// If the buffer is too small, it will be reallocated, resized, and returned by the function
|
||||
// If dst is nil, this will allocate the worst case size (CompressBound(src))
|
||||
Compress(dst, src []byte) ([]byte, error)
|
||||
```
|
||||
|
||||
8
vendor/github.com/DataDog/zstd/ZSTD_LICENSE
generated
vendored
8
vendor/github.com/DataDog/zstd/ZSTD_LICENSE
generated
vendored
@@ -2,7 +2,7 @@ BSD License
|
||||
|
||||
For Zstandard software
|
||||
|
||||
Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
|
||||
Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
@@ -14,9 +14,9 @@ are permitted provided that the following conditions are met:
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name Facebook nor the names of its contributors may be used to
|
||||
endorse or promote products derived from this software without specific
|
||||
prior written permission.
|
||||
* Neither the name Facebook, nor Meta, nor the names of its contributors may
|
||||
be used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
|
||||
58
vendor/github.com/DataDog/zstd/allocations.h
generated
vendored
Normal file
58
vendor/github.com/DataDog/zstd/allocations.h
generated
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* This file provides custom allocation primitives
|
||||
*/
|
||||
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
|
||||
|
||||
#include "compiler.h" /* MEM_STATIC */
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h" /* ZSTD_customMem */
|
||||
|
||||
#ifndef ZSTD_ALLOCATIONS_H
|
||||
#define ZSTD_ALLOCATIONS_H
|
||||
|
||||
/* custom memory allocation functions */
|
||||
|
||||
MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc)
|
||||
return customMem.customAlloc(customMem.opaque, size);
|
||||
return ZSTD_malloc(size);
|
||||
}
|
||||
|
||||
MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc) {
|
||||
/* calloc implemented as malloc+memset;
|
||||
* not as efficient as calloc, but next best guess for custom malloc */
|
||||
void* const ptr = customMem.customAlloc(customMem.opaque, size);
|
||||
ZSTD_memset(ptr, 0, size);
|
||||
return ptr;
|
||||
}
|
||||
return ZSTD_calloc(1, size);
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
|
||||
{
|
||||
if (ptr!=NULL) {
|
||||
if (customMem.customFree)
|
||||
customMem.customFree(customMem.opaque, ptr);
|
||||
else
|
||||
ZSTD_free(ptr);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ZSTD_ALLOCATIONS_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
208
vendor/github.com/DataDog/zstd/bits.h
generated
vendored
Normal file
208
vendor/github.com/DataDog/zstd/bits.h
generated
vendored
Normal file
@@ -0,0 +1,208 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_BITS_H
|
||||
#define ZSTD_BITS_H
|
||||
|
||||
#include "mem.h"
|
||||
|
||||
MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
|
||||
30, 22, 20, 15, 25, 17, 4, 8,
|
||||
31, 27, 13, 23, 21, 19, 16, 7,
|
||||
26, 12, 18, 6, 11, 5, 10, 9};
|
||||
return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
#if defined(_MSC_VER)
|
||||
# if STATIC_BMI2
|
||||
return (unsigned)_tzcnt_u32(val);
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
__assume(0); /* Should not reach this code path */
|
||||
}
|
||||
# endif
|
||||
#elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return (unsigned)__builtin_ctz(val);
|
||||
#elif defined(__ICCARM__)
|
||||
return (unsigned)__builtin_ctz(val);
|
||||
#else
|
||||
return ZSTD_countTrailingZeros32_fallback(val);
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
|
||||
11, 14, 16, 18, 22, 25, 3, 30,
|
||||
8, 12, 20, 28, 15, 17, 24, 7,
|
||||
19, 27, 23, 6, 26, 5, 4, 31};
|
||||
val |= val >> 1;
|
||||
val |= val >> 2;
|
||||
val |= val >> 4;
|
||||
val |= val >> 8;
|
||||
val |= val >> 16;
|
||||
return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27];
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
#if defined(_MSC_VER)
|
||||
# if STATIC_BMI2
|
||||
return (unsigned)_lzcnt_u32(val);
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)(31 - r);
|
||||
} else {
|
||||
__assume(0); /* Should not reach this code path */
|
||||
}
|
||||
# endif
|
||||
#elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return (unsigned)__builtin_clz(val);
|
||||
#elif defined(__ICCARM__)
|
||||
return (unsigned)__builtin_clz(val);
|
||||
#else
|
||||
return ZSTD_countLeadingZeros32_fallback(val);
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
#if defined(_MSC_VER) && defined(_WIN64)
|
||||
# if STATIC_BMI2
|
||||
return (unsigned)_tzcnt_u64(val);
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward64(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
__assume(0); /* Should not reach this code path */
|
||||
}
|
||||
# endif
|
||||
#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
|
||||
return (unsigned)__builtin_ctzll(val);
|
||||
#elif defined(__ICCARM__)
|
||||
return (unsigned)__builtin_ctzll(val);
|
||||
#else
|
||||
{
|
||||
U32 mostSignificantWord = (U32)(val >> 32);
|
||||
U32 leastSignificantWord = (U32)val;
|
||||
if (leastSignificantWord == 0) {
|
||||
return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
|
||||
} else {
|
||||
return ZSTD_countTrailingZeros32(leastSignificantWord);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
#if defined(_MSC_VER) && defined(_WIN64)
|
||||
# if STATIC_BMI2
|
||||
return (unsigned)_lzcnt_u64(val);
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse64(&r, val);
|
||||
return (unsigned)(63 - r);
|
||||
} else {
|
||||
__assume(0); /* Should not reach this code path */
|
||||
}
|
||||
# endif
|
||||
#elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return (unsigned)(__builtin_clzll(val));
|
||||
#elif defined(__ICCARM__)
|
||||
return (unsigned)(__builtin_clzll(val));
|
||||
#else
|
||||
{
|
||||
U32 mostSignificantWord = (U32)(val >> 32);
|
||||
U32 leastSignificantWord = (U32)val;
|
||||
if (mostSignificantWord == 0) {
|
||||
return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
|
||||
} else {
|
||||
return ZSTD_countLeadingZeros32(mostSignificantWord);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
return ZSTD_countTrailingZeros64((U64)val) >> 3;
|
||||
} else {
|
||||
return ZSTD_countTrailingZeros32((U32)val) >> 3;
|
||||
}
|
||||
} else { /* Big Endian CPU */
|
||||
if (MEM_64bits()) {
|
||||
return ZSTD_countLeadingZeros64((U64)val) >> 3;
|
||||
} else {
|
||||
return ZSTD_countLeadingZeros32((U32)val) >> 3;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
|
||||
{
|
||||
assert(val != 0);
|
||||
return 31 - ZSTD_countLeadingZeros32(val);
|
||||
}
|
||||
|
||||
/* ZSTD_rotateRight_*():
|
||||
* Rotates a bitfield to the right by "count" bits.
|
||||
* https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
|
||||
*/
|
||||
MEM_STATIC
|
||||
U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
|
||||
assert(count < 64);
|
||||
count &= 0x3F; /* for fickle pattern recognition */
|
||||
return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
|
||||
}
|
||||
|
||||
MEM_STATIC
|
||||
U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
|
||||
assert(count < 32);
|
||||
count &= 0x1F; /* for fickle pattern recognition */
|
||||
return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
|
||||
}
|
||||
|
||||
MEM_STATIC
|
||||
U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
|
||||
assert(count < 16);
|
||||
count &= 0x0F; /* for fickle pattern recognition */
|
||||
return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
|
||||
}
|
||||
|
||||
#endif /* ZSTD_BITS_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
192
vendor/github.com/DataDog/zstd/bitstream.h
generated
vendored
192
vendor/github.com/DataDog/zstd/bitstream.h
generated
vendored
@@ -2,7 +2,7 @@
|
||||
/* ******************************************************************
|
||||
* bitstream
|
||||
* Part of FSE library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -15,9 +15,6 @@
|
||||
#ifndef BITSTREAM_H_MODULE
|
||||
#define BITSTREAM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
/*
|
||||
* This API consists of small unitary functions, which must be inlined for best performance.
|
||||
* Since link-time-optimization is not available for all compilers,
|
||||
@@ -31,14 +28,14 @@ extern "C" {
|
||||
#include "compiler.h" /* UNLIKELY() */
|
||||
#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
|
||||
#include "error_private.h" /* error codes and messages */
|
||||
|
||||
#include "bits.h" /* ZSTD_highbit32 */
|
||||
|
||||
/*=========================================
|
||||
* Target specific
|
||||
=========================================*/
|
||||
#ifndef ZSTD_NO_INTRINSICS
|
||||
# if defined(__BMI__) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental)/bzhi */
|
||||
# elif defined(__ICCARM__)
|
||||
# include <intrinsics.h>
|
||||
# endif
|
||||
@@ -52,12 +49,13 @@ extern "C" {
|
||||
/*-******************************************
|
||||
* bitStream encoding API (write forward)
|
||||
********************************************/
|
||||
typedef size_t BitContainerType;
|
||||
/* bitStream can mix input from multiple sources.
|
||||
* A critical property of these streams is that they encode and decode in **reverse** direction.
|
||||
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
|
||||
*/
|
||||
typedef struct {
|
||||
size_t bitContainer;
|
||||
BitContainerType bitContainer;
|
||||
unsigned bitPos;
|
||||
char* startPtr;
|
||||
char* ptr;
|
||||
@@ -65,7 +63,7 @@ typedef struct {
|
||||
} BIT_CStream_t;
|
||||
|
||||
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
|
||||
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
|
||||
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
||||
|
||||
@@ -74,7 +72,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
||||
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
|
||||
*
|
||||
* bits are first added to a local register.
|
||||
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
|
||||
* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
|
||||
* Writing data into memory is an explicit operation, performed by the flushBits function.
|
||||
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
|
||||
* After a flushBits, a maximum of 7 bits might still be stored into local register.
|
||||
@@ -91,28 +89,28 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
||||
* bitStream decoding API (read backward)
|
||||
**********************************************/
|
||||
typedef struct {
|
||||
size_t bitContainer;
|
||||
BitContainerType bitContainer;
|
||||
unsigned bitsConsumed;
|
||||
const char* ptr;
|
||||
const char* start;
|
||||
const char* limitPtr;
|
||||
} BIT_DStream_t;
|
||||
|
||||
typedef enum { BIT_DStream_unfinished = 0,
|
||||
BIT_DStream_endOfBuffer = 1,
|
||||
BIT_DStream_completed = 2,
|
||||
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
|
||||
typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */
|
||||
BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */
|
||||
BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */
|
||||
BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */
|
||||
} BIT_DStream_status; /* result of BIT_reloadDStream() */
|
||||
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
|
||||
|
||||
/* Start by invoking BIT_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
|
||||
@@ -124,7 +122,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
/*-****************************************
|
||||
* unsafe API
|
||||
******************************************/
|
||||
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
|
||||
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
|
||||
|
||||
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
|
||||
@@ -133,48 +131,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/* faster, but works only if nbBits >= 1 */
|
||||
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
# if STATIC_BMI2 == 1
|
||||
return _lzcnt_u32(val) ^ 31;
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
||||
return __builtin_clz (val) ^ 31;
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
return 31 - __CLZ(val);
|
||||
# else /* Software version */
|
||||
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,
|
||||
11, 14, 16, 18, 22, 25, 3, 30,
|
||||
8, 12, 20, 28, 15, 17, 24, 7,
|
||||
19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
/*===== Local Constants =====*/
|
||||
static const unsigned BIT_mask[] = {
|
||||
0, 1, 3, 7, 0xF, 0x1F,
|
||||
@@ -204,16 +160,31 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
|
||||
return 0;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits)
|
||||
{
|
||||
#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS)
|
||||
# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__)
|
||||
return _bzhi_u64(bitContainer, nbBits);
|
||||
# else
|
||||
DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32));
|
||||
return _bzhi_u32(bitContainer, nbBits);
|
||||
# endif
|
||||
#else
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
return bitContainer & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
/*! BIT_addBits() :
|
||||
* can add up to 31 bits into `bitC`.
|
||||
* Note : does not check for register overflow ! */
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
|
||||
size_t value, unsigned nbBits)
|
||||
BitContainerType value, unsigned nbBits)
|
||||
{
|
||||
DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
||||
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
|
||||
bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos;
|
||||
bitC->bitPos += nbBits;
|
||||
}
|
||||
|
||||
@@ -221,7 +192,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
|
||||
* works only if `value` is _clean_,
|
||||
* meaning all high bits above nbBits are 0 */
|
||||
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
|
||||
size_t value, unsigned nbBits)
|
||||
BitContainerType value, unsigned nbBits)
|
||||
{
|
||||
assert((value>>nbBits) == 0);
|
||||
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
||||
@@ -268,7 +239,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
|
||||
BIT_addBitsFast(bitC, 1, 1); /* endMark */
|
||||
BIT_flushBits(bitC);
|
||||
if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
|
||||
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
|
||||
return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -292,35 +263,35 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
|
||||
bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
} else {
|
||||
bitD->ptr = bitD->start;
|
||||
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
||||
switch(srcSize)
|
||||
{
|
||||
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
|
||||
case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
|
||||
ZSTD_FALLTHROUGH;
|
||||
|
||||
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
|
||||
case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
|
||||
ZSTD_FALLTHROUGH;
|
||||
|
||||
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
|
||||
case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
|
||||
ZSTD_FALLTHROUGH;
|
||||
|
||||
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
|
||||
case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24;
|
||||
ZSTD_FALLTHROUGH;
|
||||
|
||||
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
|
||||
case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16;
|
||||
ZSTD_FALLTHROUGH;
|
||||
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
|
||||
case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8;
|
||||
ZSTD_FALLTHROUGH;
|
||||
|
||||
default: break;
|
||||
}
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
|
||||
bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */
|
||||
}
|
||||
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
|
||||
@@ -329,12 +300,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
|
||||
FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
|
||||
{
|
||||
return bitContainer >> start;
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
|
||||
{
|
||||
U32 const regMask = sizeof(bitContainer)*8 - 1;
|
||||
/* if start > regMask, bitstream is corrupted, and result is undefined */
|
||||
@@ -344,30 +315,20 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
|
||||
* such cpus old (pre-Haswell, 2013) and their performance is not of that
|
||||
* importance.
|
||||
*/
|
||||
#if defined(__x86_64__) || defined(_M_X86)
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
|
||||
#else
|
||||
return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
{
|
||||
#if defined(STATIC_BMI2) && STATIC_BMI2 == 1
|
||||
return _bzhi_u64(bitContainer, nbBits);
|
||||
#else
|
||||
assert(nbBits < BIT_MASK_SIZE);
|
||||
return bitContainer & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
/*! BIT_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified.
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted */
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
/* arbitrate between double-shift and shift+mask */
|
||||
#if 1
|
||||
@@ -383,14 +344,14 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U3
|
||||
|
||||
/*! BIT_lookBitsFast() :
|
||||
* unsafe version; only works if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
assert(nbBits >= 1);
|
||||
return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
|
||||
}
|
||||
|
||||
MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
@@ -399,23 +360,38 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value. */
|
||||
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBits(bitD, nbBits);
|
||||
BitContainerType const value = BIT_lookBits(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BIT_readBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
* unsafe version; only works if nbBits >= 1 */
|
||||
MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBitsFast(bitD, nbBits);
|
||||
BitContainerType const value = BIT_lookBitsFast(bitD, nbBits);
|
||||
assert(nbBits >= 1);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStream_internal() :
|
||||
* Simple variant of BIT_reloadDStream(), with two conditions:
|
||||
* 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
|
||||
* 2. look window is valid after shifted down : bitD->ptr >= bitD->start
|
||||
*/
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD)
|
||||
{
|
||||
assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
assert(bitD->ptr >= bitD->start);
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BIT_DStream_unfinished;
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStreamFast() :
|
||||
* Similar to BIT_reloadDStream(), but with two differences:
|
||||
* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
|
||||
@@ -426,31 +402,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (UNLIKELY(bitD->ptr < bitD->limitPtr))
|
||||
return BIT_DStream_overflow;
|
||||
assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BIT_DStream_unfinished;
|
||||
return BIT_reloadDStream_internal(bitD);
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStream() :
|
||||
* Refill `bitD` from buffer previously set in BIT_initDStream() .
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* This function is safe, it guarantees it will not never beyond src buffer.
|
||||
* @return : status of `BIT_DStream_t` internal register.
|
||||
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */
|
||||
/* note : once in overflow mode, a bitstream remains in this mode until it's reset */
|
||||
if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) {
|
||||
static const BitContainerType zeroFilled = 0;
|
||||
bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */
|
||||
/* overflow detected, erroneous scenario or end of stream: no update */
|
||||
return BIT_DStream_overflow;
|
||||
}
|
||||
|
||||
assert(bitD->ptr >= bitD->start);
|
||||
|
||||
if (bitD->ptr >= bitD->limitPtr) {
|
||||
return BIT_reloadDStreamFast(bitD);
|
||||
return BIT_reloadDStream_internal(bitD);
|
||||
}
|
||||
if (bitD->ptr == bitD->start) {
|
||||
/* reached end of bitStream => no update */
|
||||
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
|
||||
return BIT_DStream_completed;
|
||||
}
|
||||
/* start < ptr < limitPtr */
|
||||
/* start < ptr < limitPtr => cautious update */
|
||||
{ U32 nbBytes = bitD->bitsConsumed >> 3;
|
||||
BIT_DStream_status result = BIT_DStream_unfinished;
|
||||
if (bitD->ptr - nbBytes < bitD->start) {
|
||||
@@ -472,10 +452,6 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
||||
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* BITSTREAM_H_MODULE */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/clevels.h
generated
vendored
2
vendor/github.com/DataDog/zstd/clevels.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
211
vendor/github.com/DataDog/zstd/compiler.h
generated
vendored
211
vendor/github.com/DataDog/zstd/compiler.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,6 +12,8 @@
|
||||
#ifndef ZSTD_COMPILER_H
|
||||
#define ZSTD_COMPILER_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "portability_macros.h"
|
||||
|
||||
/*-*******************************************************
|
||||
@@ -26,7 +28,7 @@
|
||||
# define INLINE_KEYWORD
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
|
||||
# define FORCE_INLINE_ATTR __attribute__((always_inline))
|
||||
#elif defined(_MSC_VER)
|
||||
# define FORCE_INLINE_ATTR __forceinline
|
||||
@@ -52,12 +54,19 @@
|
||||
# define WIN_CDECL
|
||||
#endif
|
||||
|
||||
/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
|
||||
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
|
||||
# define UNUSED_ATTR __attribute__((unused))
|
||||
#else
|
||||
# define UNUSED_ATTR
|
||||
#endif
|
||||
|
||||
/**
|
||||
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
|
||||
* parameters. They must be inlined for the compiler to eliminate the constant
|
||||
* branches.
|
||||
*/
|
||||
#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
|
||||
#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR
|
||||
/**
|
||||
* HINT_INLINE is used to help the compiler generate better code. It is *not*
|
||||
* used for "templates", so it can be tweaked based on the compilers
|
||||
@@ -72,21 +81,37 @@
|
||||
#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
|
||||
# define HINT_INLINE static INLINE_KEYWORD
|
||||
#else
|
||||
# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
|
||||
# define HINT_INLINE FORCE_INLINE_TEMPLATE
|
||||
#endif
|
||||
|
||||
/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
|
||||
/* "soft" inline :
|
||||
* The compiler is free to select if it's a good idea to inline or not.
|
||||
* The main objective is to silence compiler warnings
|
||||
* when a defined function in included but not used.
|
||||
*
|
||||
* Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit.
|
||||
* Updating the prefix is probably preferable, but requires a fairly large codemod,
|
||||
* since this name is used everywhere.
|
||||
*/
|
||||
#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */
|
||||
#if defined(__GNUC__)
|
||||
# define UNUSED_ATTR __attribute__((unused))
|
||||
# define MEM_STATIC static __inline UNUSED_ATTR
|
||||
#elif defined(__IAR_SYSTEMS_ICC__)
|
||||
# define MEM_STATIC static inline UNUSED_ATTR
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define MEM_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define MEM_STATIC static __inline
|
||||
#else
|
||||
# define UNUSED_ATTR
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* force no inlining */
|
||||
#ifdef _MSC_VER
|
||||
# define FORCE_NOINLINE static __declspec(noinline)
|
||||
#else
|
||||
# if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
|
||||
# define FORCE_NOINLINE static __attribute__((__noinline__))
|
||||
# else
|
||||
# define FORCE_NOINLINE static
|
||||
@@ -95,7 +120,7 @@
|
||||
|
||||
|
||||
/* target attribute */
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
|
||||
# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
|
||||
#else
|
||||
# define TARGET_ATTRIBUTE(target)
|
||||
@@ -110,10 +135,10 @@
|
||||
/* prefetch
|
||||
* can be disabled, by declaring NO_PREFETCH build macro */
|
||||
#if defined(NO_PREFETCH)
|
||||
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */
|
||||
# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */
|
||||
#else
|
||||
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
|
||||
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */
|
||||
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
||||
# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
|
||||
# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
|
||||
@@ -121,24 +146,25 @@
|
||||
# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
|
||||
# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
|
||||
# elif defined(__aarch64__)
|
||||
# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr)))
|
||||
# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr)))
|
||||
# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0)
|
||||
# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0)
|
||||
# else
|
||||
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */
|
||||
# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */
|
||||
# endif
|
||||
#endif /* NO_PREFETCH */
|
||||
|
||||
#define CACHELINE_SIZE 64
|
||||
|
||||
#define PREFETCH_AREA(p, s) { \
|
||||
const char* const _ptr = (const char*)(p); \
|
||||
size_t const _size = (size_t)(s); \
|
||||
size_t _pos; \
|
||||
for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
|
||||
PREFETCH_L2(_ptr + _pos); \
|
||||
} \
|
||||
}
|
||||
#define PREFETCH_AREA(p, s) \
|
||||
do { \
|
||||
const char* const _ptr = (const char*)(p); \
|
||||
size_t const _size = (size_t)(s); \
|
||||
size_t _pos; \
|
||||
for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
|
||||
PREFETCH_L2(_ptr + _pos); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* vectorization
|
||||
* older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
|
||||
@@ -166,6 +192,12 @@
|
||||
#define UNLIKELY(x) (x)
|
||||
#endif
|
||||
|
||||
#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)))
|
||||
# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0)
|
||||
#else
|
||||
# define ZSTD_UNREACHABLE do { assert(0); } while (0)
|
||||
#endif
|
||||
|
||||
/* disable warnings */
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
@@ -176,28 +208,21 @@
|
||||
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
||||
#endif
|
||||
|
||||
/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
|
||||
#ifndef STATIC_BMI2
|
||||
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
|
||||
# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
|
||||
# define STATIC_BMI2 1
|
||||
# endif
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifndef STATIC_BMI2
|
||||
#define STATIC_BMI2 0
|
||||
#endif
|
||||
|
||||
/* compile time determination of SIMD support */
|
||||
#if !defined(ZSTD_NO_INTRINSICS)
|
||||
# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
|
||||
# if defined(__AVX2__)
|
||||
# define ZSTD_ARCH_X86_AVX2
|
||||
# endif
|
||||
# if defined(__SSE2__) || defined(_M_X64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
|
||||
# define ZSTD_ARCH_X86_SSE2
|
||||
# endif
|
||||
# if defined(__ARM_NEON) || defined(_M_ARM64)
|
||||
# define ZSTD_ARCH_ARM_NEON
|
||||
# endif
|
||||
#
|
||||
# if defined(ZSTD_ARCH_X86_AVX2)
|
||||
# include <immintrin.h>
|
||||
# endif
|
||||
# if defined(ZSTD_ARCH_X86_SSE2)
|
||||
# include <emmintrin.h>
|
||||
# elif defined(ZSTD_ARCH_ARM_NEON)
|
||||
@@ -242,9 +267,15 @@
|
||||
#endif
|
||||
|
||||
/*-**************************************************************
|
||||
* Alignment check
|
||||
* Alignment
|
||||
*****************************************************************/
|
||||
|
||||
/* @return 1 if @u is a 2^n value, 0 otherwise
|
||||
* useful to check a value is valid for alignment restrictions */
|
||||
MEM_STATIC int ZSTD_isPower2(size_t u) {
|
||||
return (u & (u-1)) == 0;
|
||||
}
|
||||
|
||||
/* this test was initially positioned in mem.h,
|
||||
* but this file is removed (or replaced) for linux kernel
|
||||
* so it's now hosted in compiler.h,
|
||||
@@ -270,11 +301,105 @@
|
||||
# endif
|
||||
#endif /* ZSTD_ALIGNOF */
|
||||
|
||||
#ifndef ZSTD_ALIGNED
|
||||
/* C90-compatible alignment macro (GCC/Clang). Adjust for other compilers if needed. */
|
||||
# if defined(__GNUC__) || defined(__clang__)
|
||||
# define ZSTD_ALIGNED(a) __attribute__((aligned(a)))
|
||||
# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
|
||||
# define ZSTD_ALIGNED(a) _Alignas(a)
|
||||
#elif defined(_MSC_VER)
|
||||
# define ZSTD_ALIGNED(n) __declspec(align(n))
|
||||
# else
|
||||
/* this compiler will require its own alignment instruction */
|
||||
# define ZSTD_ALIGNED(...)
|
||||
# endif
|
||||
#endif /* ZSTD_ALIGNED */
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Sanitizer
|
||||
*****************************************************************/
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER
|
||||
/**
|
||||
* Zstd relies on pointer overflow in its decompressor.
|
||||
* We add this attribute to functions that rely on pointer overflow.
|
||||
*/
|
||||
#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
# if __has_attribute(no_sanitize)
|
||||
# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8
|
||||
/* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */
|
||||
# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow")))
|
||||
# else
|
||||
/* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */
|
||||
# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow")))
|
||||
# endif
|
||||
# else
|
||||
# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Helper function to perform a wrapped pointer difference without triggering
|
||||
* UBSAN.
|
||||
*
|
||||
* @returns lhs - rhs with wrapping
|
||||
*/
|
||||
MEM_STATIC
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs)
|
||||
{
|
||||
return lhs - rhs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to perform a wrapped pointer add without triggering UBSAN.
|
||||
*
|
||||
* @return ptr + add with wrapping
|
||||
*/
|
||||
MEM_STATIC
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add)
|
||||
{
|
||||
return ptr + add;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to perform a wrapped pointer subtraction without triggering
|
||||
* UBSAN.
|
||||
*
|
||||
* @return ptr - sub with wrapping
|
||||
*/
|
||||
MEM_STATIC
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub)
|
||||
{
|
||||
return ptr - sub;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to add to a pointer that works around C's undefined behavior
|
||||
* of adding 0 to NULL.
|
||||
*
|
||||
* @returns `ptr + add` except it defines `NULL + 0 == NULL`.
|
||||
*/
|
||||
MEM_STATIC
|
||||
unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add)
|
||||
{
|
||||
return add > 0 ? ptr + add : ptr;
|
||||
}
|
||||
|
||||
/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
|
||||
* abundance of caution, disable our custom poisoning on mingw. */
|
||||
#ifdef __MINGW32__
|
||||
#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE
|
||||
#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1
|
||||
#endif
|
||||
#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE
|
||||
#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
/* Not all platforms that support msan provide sanitizers/msan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
* include the header file... */
|
||||
@@ -293,9 +418,13 @@ void __msan_poison(const volatile void *a, size_t size);
|
||||
/* Returns the offset of the first (at least partially) poisoned byte in the
|
||||
memory range, or -1 if the whole range is good. */
|
||||
intptr_t __msan_test_shadow(const volatile void *x, size_t size);
|
||||
|
||||
/* Print shadow and origin for the memory range to stderr in a human-readable
|
||||
format. */
|
||||
void __msan_print_shadow(const volatile void *x, size_t size);
|
||||
#endif
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* Not all platforms that support asan provide sanitizers/asan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
* include the header file... */
|
||||
|
||||
141
vendor/github.com/DataDog/zstd/cover.c
generated
vendored
141
vendor/github.com/DataDog/zstd/cover.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -22,8 +22,17 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
/* qsort_r is an extension. */
|
||||
#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
|
||||
defined(__CYGWIN__) || defined(__MSYS__)
|
||||
#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
|
||||
#define _GNU_SOURCE
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <stdlib.h> /* malloc, free, qsort_r */
|
||||
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
|
||||
@@ -32,9 +41,10 @@
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* read */
|
||||
#include "pool.h"
|
||||
#include "threading.h"
|
||||
#include "pool.h" /* POOL_ctx */
|
||||
#include "threading.h" /* ZSTD_pthread_mutex_t */
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "bits.h" /* ZSTD_highbit32 */
|
||||
#include "zdict.h"
|
||||
#include "cover.h"
|
||||
|
||||
@@ -78,7 +88,7 @@ static clock_t g_time = 0;
|
||||
#undef LOCALDISPLAYUPDATE
|
||||
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
||||
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
||||
g_time = clock(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} \
|
||||
@@ -232,8 +242,10 @@ typedef struct {
|
||||
unsigned d;
|
||||
} COVER_ctx_t;
|
||||
|
||||
/* We need a global context for qsort... */
|
||||
#if !defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER)
|
||||
/* C90 only offers qsort() that needs a global context. */
|
||||
static COVER_ctx_t *g_coverCtx = NULL;
|
||||
#endif
|
||||
|
||||
/*-*************************************
|
||||
* Helper functions
|
||||
@@ -276,11 +288,15 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
|
||||
|
||||
/**
|
||||
* Same as COVER_cmp() except ties are broken by pointer value
|
||||
* NOTE: g_coverCtx must be set to call this function. A global is required because
|
||||
* qsort doesn't take an opaque pointer.
|
||||
*/
|
||||
static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
int result = COVER_cmp(g_coverCtx, lp, rp);
|
||||
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
|
||||
static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) {
|
||||
#elif defined(_GNU_SOURCE)
|
||||
static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) {
|
||||
#else /* C90 fallback.*/
|
||||
static int COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
#endif
|
||||
int result = COVER_cmp((COVER_ctx_t*)g_coverCtx, lp, rp);
|
||||
if (result == 0) {
|
||||
result = lp < rp ? -1 : 1;
|
||||
}
|
||||
@@ -289,21 +305,58 @@ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
/**
|
||||
* Faster version for d <= 8.
|
||||
*/
|
||||
static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
|
||||
int result = COVER_cmp8(g_coverCtx, lp, rp);
|
||||
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
|
||||
static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) {
|
||||
#elif defined(_GNU_SOURCE)
|
||||
static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) {
|
||||
#else /* C90 fallback.*/
|
||||
static int COVER_strict_cmp8(const void *lp, const void *rp) {
|
||||
#endif
|
||||
int result = COVER_cmp8((COVER_ctx_t*)g_coverCtx, lp, rp);
|
||||
if (result == 0) {
|
||||
result = lp < rp ? -1 : 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract away divergence of qsort_r() parameters.
|
||||
* Hopefully when C11 become the norm, we will be able
|
||||
* to clean it up.
|
||||
*/
|
||||
static void stableSort(COVER_ctx_t *ctx) {
|
||||
#if defined(__APPLE__)
|
||||
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
ctx,
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
#elif defined(_GNU_SOURCE)
|
||||
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
|
||||
ctx);
|
||||
#elif defined(_WIN32) && defined(_MSC_VER)
|
||||
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
|
||||
ctx);
|
||||
#elif defined(__OpenBSD__)
|
||||
g_coverCtx = ctx;
|
||||
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
#else /* C90 fallback.*/
|
||||
g_coverCtx = ctx;
|
||||
/* TODO(cavalcanti): implement a reentrant qsort() when is not available. */
|
||||
qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first pointer in [first, last) whose element does not compare
|
||||
* less than value. If no such element exists it returns last.
|
||||
*/
|
||||
static const size_t *COVER_lower_bound(const size_t *first, const size_t *last,
|
||||
static const size_t *COVER_lower_bound(const size_t* first, const size_t* last,
|
||||
size_t value) {
|
||||
size_t count = last - first;
|
||||
size_t count = (size_t)(last - first);
|
||||
assert(last >= first);
|
||||
while (count != 0) {
|
||||
size_t step = count / 2;
|
||||
const size_t *ptr = first;
|
||||
@@ -542,14 +595,15 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
|
||||
|
||||
/**
|
||||
* Prepare a context for dictionary building.
|
||||
* The context is only dependent on the parameter `d` and can used multiple
|
||||
* The context is only dependent on the parameter `d` and can be used multiple
|
||||
* times.
|
||||
* Returns 0 on success or error code on error.
|
||||
* The context must be destroyed with `COVER_ctx_destroy()`.
|
||||
*/
|
||||
static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
unsigned d, double splitPoint) {
|
||||
unsigned d, double splitPoint)
|
||||
{
|
||||
const BYTE *const samples = (const BYTE *)samplesBuffer;
|
||||
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
|
||||
/* Split samples into testing and training sets */
|
||||
@@ -618,17 +672,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
for (i = 0; i < ctx->suffixSize; ++i) {
|
||||
ctx->suffix[i] = i;
|
||||
}
|
||||
/* qsort doesn't take an opaque pointer, so pass as a global.
|
||||
* On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
|
||||
*/
|
||||
g_coverCtx = ctx;
|
||||
#if defined(__OpenBSD__)
|
||||
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
#else
|
||||
qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
||||
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
||||
#endif
|
||||
stableSort(ctx);
|
||||
}
|
||||
DISPLAYLEVEL(2, "Computing frequencies\n");
|
||||
/* For each dmer group (group of positions with the same first d bytes):
|
||||
@@ -647,7 +691,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
|
||||
void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
|
||||
{
|
||||
const double ratio = (double)nbDmers / maxDictSize;
|
||||
const double ratio = (double)nbDmers / (double)maxDictSize;
|
||||
if (ratio >= 10) {
|
||||
return;
|
||||
}
|
||||
@@ -733,7 +777,7 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
|
||||
return tail;
|
||||
}
|
||||
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
||||
ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
|
||||
void *dictBuffer, size_t dictBufferCapacity,
|
||||
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_cover_params_t parameters)
|
||||
@@ -907,8 +951,10 @@ void COVER_best_start(COVER_best_t *best) {
|
||||
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
|
||||
* If this dictionary is the best so far save it and its parameters.
|
||||
*/
|
||||
void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
|
||||
COVER_dictSelection_t selection) {
|
||||
void COVER_best_finish(COVER_best_t* best,
|
||||
ZDICT_cover_params_t parameters,
|
||||
COVER_dictSelection_t selection)
|
||||
{
|
||||
void* dict = selection.dictContent;
|
||||
size_t compressedSize = selection.totalCompressedSize;
|
||||
size_t dictSize = selection.dictSize;
|
||||
@@ -951,9 +997,17 @@ void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
|
||||
}
|
||||
}
|
||||
|
||||
static COVER_dictSelection_t setDictSelection(BYTE* buf, size_t s, size_t csz)
|
||||
{
|
||||
COVER_dictSelection_t ds;
|
||||
ds.dictContent = buf;
|
||||
ds.dictSize = s;
|
||||
ds.totalCompressedSize = csz;
|
||||
return ds;
|
||||
}
|
||||
|
||||
COVER_dictSelection_t COVER_dictSelectionError(size_t error) {
|
||||
COVER_dictSelection_t selection = { NULL, 0, error };
|
||||
return selection;
|
||||
return setDictSelection(NULL, 0, error);
|
||||
}
|
||||
|
||||
unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) {
|
||||
@@ -972,8 +1026,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
|
||||
size_t largestCompressed = 0;
|
||||
BYTE* customDictContentEnd = customDictContent + dictContentSize;
|
||||
|
||||
BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
|
||||
BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
|
||||
BYTE* largestDictbuffer = (BYTE*)malloc(dictBufferCapacity);
|
||||
BYTE* candidateDictBuffer = (BYTE*)malloc(dictBufferCapacity);
|
||||
double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
|
||||
|
||||
if (!largestDictbuffer || !candidateDictBuffer) {
|
||||
@@ -1006,9 +1060,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
|
||||
}
|
||||
|
||||
if (params.shrinkDict == 0) {
|
||||
COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
|
||||
free(candidateDictBuffer);
|
||||
return selection;
|
||||
return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize);
|
||||
}
|
||||
|
||||
largestDict = dictContentSize;
|
||||
@@ -1040,20 +1093,16 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe
|
||||
return COVER_dictSelectionError(totalCompressedSize);
|
||||
}
|
||||
|
||||
if (totalCompressedSize <= largestCompressed * regressionTolerance) {
|
||||
COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize };
|
||||
if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) {
|
||||
free(largestDictbuffer);
|
||||
return selection;
|
||||
return setDictSelection( candidateDictBuffer, dictContentSize, totalCompressedSize );
|
||||
}
|
||||
dictContentSize *= 2;
|
||||
}
|
||||
dictContentSize = largestDict;
|
||||
totalCompressedSize = largestCompressed;
|
||||
{
|
||||
COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
|
||||
free(candidateDictBuffer);
|
||||
return selection;
|
||||
}
|
||||
free(candidateDictBuffer);
|
||||
return setDictSelection( largestDictbuffer, dictContentSize, totalCompressedSize );
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1116,7 +1165,7 @@ _cleanup:
|
||||
free(freqs);
|
||||
}
|
||||
|
||||
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer,
|
||||
const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_cover_params_t* parameters)
|
||||
|
||||
12
vendor/github.com/DataDog/zstd/cover.h
generated
vendored
12
vendor/github.com/DataDog/zstd/cover.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -13,14 +13,8 @@
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
#include "mem.h" /* read */
|
||||
#include "pool.h"
|
||||
#include "threading.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "threading.h" /* ZSTD_pthread_mutex_t */
|
||||
#include "mem.h" /* U32, BYTE */
|
||||
#include "zdict.h"
|
||||
|
||||
/**
|
||||
|
||||
38
vendor/github.com/DataDog/zstd/cpu.h
generated
vendored
38
vendor/github.com/DataDog/zstd/cpu.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -36,6 +36,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
|
||||
U32 f7b = 0;
|
||||
U32 f7c = 0;
|
||||
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
|
||||
#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16
|
||||
int reg[4];
|
||||
__cpuid((int*)reg, 0);
|
||||
{
|
||||
@@ -51,6 +52,41 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
|
||||
f7c = (U32)reg[2];
|
||||
}
|
||||
}
|
||||
#else
|
||||
/* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in
|
||||
* which the `__cpuid` intrinsic does not save and restore `rbx` as it needs
|
||||
* to due to being a reserved register. So in that case, do the `cpuid`
|
||||
* ourselves. Clang supports inline assembly anyway.
|
||||
*/
|
||||
U32 n;
|
||||
__asm__(
|
||||
"pushq %%rbx\n\t"
|
||||
"cpuid\n\t"
|
||||
"popq %%rbx\n\t"
|
||||
: "=a"(n)
|
||||
: "a"(0)
|
||||
: "rcx", "rdx");
|
||||
if (n >= 1) {
|
||||
U32 f1a;
|
||||
__asm__(
|
||||
"pushq %%rbx\n\t"
|
||||
"cpuid\n\t"
|
||||
"popq %%rbx\n\t"
|
||||
: "=a"(f1a), "=c"(f1c), "=d"(f1d)
|
||||
: "a"(1)
|
||||
:);
|
||||
}
|
||||
if (n >= 7) {
|
||||
__asm__(
|
||||
"pushq %%rbx\n\t"
|
||||
"cpuid\n\t"
|
||||
"movq %%rbx, %%rax\n\t"
|
||||
"popq %%rbx"
|
||||
: "=a"(f7b), "=c"(f7c)
|
||||
: "a"(7), "c"(0)
|
||||
: "rdx");
|
||||
}
|
||||
#endif
|
||||
#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
|
||||
/* The following block like the normal cpuid branch below, but gcc
|
||||
* reserves ebx for use of its pic register so we must specially
|
||||
|
||||
8
vendor/github.com/DataDog/zstd/debug.c
generated
vendored
8
vendor/github.com/DataDog/zstd/debug.c
generated
vendored
@@ -2,7 +2,7 @@
|
||||
/* ******************************************************************
|
||||
* debug
|
||||
* Part of FSE library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -22,6 +22,12 @@
|
||||
|
||||
#include "debug.h"
|
||||
|
||||
#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2)
|
||||
/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a
|
||||
* translation unit is empty. So remove this from Linux kernel builds, but
|
||||
* otherwise just leave it in.
|
||||
*/
|
||||
int g_debuglevel = DEBUGLEVEL;
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
42
vendor/github.com/DataDog/zstd/debug.h
generated
vendored
42
vendor/github.com/DataDog/zstd/debug.h
generated
vendored
@@ -2,7 +2,7 @@
|
||||
/* ******************************************************************
|
||||
* debug
|
||||
* Part of FSE library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -33,10 +33,6 @@
|
||||
#ifndef DEBUG_H_12987983217
|
||||
#define DEBUG_H_12987983217
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* static assert is triggered at compile time, leaving no runtime artefact.
|
||||
* static assert only works with compile-time constants.
|
||||
@@ -86,23 +82,27 @@ extern int g_debuglevel; /* the variable is only declared,
|
||||
It's useful when enabling very verbose levels
|
||||
on selective conditions (such as position in src) */
|
||||
|
||||
# define RAWLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
ZSTD_DEBUG_PRINT(__VA_ARGS__); \
|
||||
} }
|
||||
# define DEBUGLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \
|
||||
ZSTD_DEBUG_PRINT(" \n"); \
|
||||
} }
|
||||
# define RAWLOG(l, ...) \
|
||||
do { \
|
||||
if (l<=g_debuglevel) { \
|
||||
ZSTD_DEBUG_PRINT(__VA_ARGS__); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define STRINGIFY(x) #x
|
||||
#define TOSTRING(x) STRINGIFY(x)
|
||||
#define LINE_AS_STRING TOSTRING(__LINE__)
|
||||
|
||||
# define DEBUGLOG(l, ...) \
|
||||
do { \
|
||||
if (l<=g_debuglevel) { \
|
||||
ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \
|
||||
ZSTD_DEBUG_PRINT(" \n"); \
|
||||
} \
|
||||
} while (0)
|
||||
#else
|
||||
# define RAWLOG(l, ...) {} /* disabled */
|
||||
# define DEBUGLOG(l, ...) {} /* disabled */
|
||||
#endif
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
# define RAWLOG(l, ...) do { } while (0) /* disabled */
|
||||
# define DEBUGLOG(l, ...) do { } while (0) /* disabled */
|
||||
#endif
|
||||
|
||||
#endif /* DEBUG_H_12987983217 */
|
||||
|
||||
10
vendor/github.com/DataDog/zstd/divsufsort.h
generated
vendored
10
vendor/github.com/DataDog/zstd/divsufsort.h
generated
vendored
@@ -28,11 +28,6 @@
|
||||
#ifndef _DIVSUFSORT_H
|
||||
#define _DIVSUFSORT_H 1
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif /* __cplusplus */
|
||||
|
||||
|
||||
/*- Prototypes -*/
|
||||
|
||||
/**
|
||||
@@ -60,11 +55,6 @@ divsufsort(const unsigned char *T, int *SA, int n, int openMP);
|
||||
int
|
||||
divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP);
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* extern "C" */
|
||||
#endif /* __cplusplus */
|
||||
|
||||
#endif /* _DIVSUFSORT_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
52
vendor/github.com/DataDog/zstd/entropy_common.c
generated
vendored
52
vendor/github.com/DataDog/zstd/entropy_common.c
generated
vendored
@@ -1,7 +1,7 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* Common functions of New Generation Entropy library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -20,8 +20,8 @@
|
||||
#include "error_private.h" /* ERR_*, ERROR */
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
|
||||
#include "huf.h"
|
||||
#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
|
||||
|
||||
|
||||
/*=== Version ===*/
|
||||
@@ -39,34 +39,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
****************************************************************/
|
||||
static U32 FSE_ctz(U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return __builtin_ctz(val);
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
return __CTZ(val);
|
||||
# else /* Software version */
|
||||
U32 count = 0;
|
||||
while ((val & 1) == 0) {
|
||||
val >>= 1;
|
||||
++count;
|
||||
}
|
||||
return count;
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
@@ -114,7 +86,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
||||
* repeat.
|
||||
* Avoid UB by setting the high bit to 1.
|
||||
*/
|
||||
int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
|
||||
int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
|
||||
while (repeats >= 12) {
|
||||
charnum += 3 * 12;
|
||||
if (LIKELY(ip <= iend-7)) {
|
||||
@@ -125,7 +97,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
|
||||
repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
|
||||
}
|
||||
charnum += 3 * repeats;
|
||||
bitStream >>= 2 * repeats;
|
||||
@@ -190,7 +162,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
||||
* know that threshold > 1.
|
||||
*/
|
||||
if (remaining <= 1) break;
|
||||
nbBits = BIT_highbit32(remaining) + 1;
|
||||
nbBits = ZSTD_highbit32(remaining) + 1;
|
||||
threshold = 1 << (nbBits - 1);
|
||||
}
|
||||
if (charnum >= maxSV1) break;
|
||||
@@ -265,7 +237,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
|
||||
return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
|
||||
return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t
|
||||
@@ -313,14 +285,14 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
if (weightTotal == 0) return ERROR(corruption_detected);
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
||||
{ U32 const tableLog = ZSTD_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BIT_highbit32(rest);
|
||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||
U32 const verif = 1 << ZSTD_highbit32(rest);
|
||||
U32 const lastWeight = ZSTD_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
@@ -357,14 +329,14 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize,
|
||||
void* workSpace, size_t wkspSize,
|
||||
int bmi2)
|
||||
int flags)
|
||||
{
|
||||
#if DYNAMIC_BMI2
|
||||
if (bmi2) {
|
||||
if (flags & HUF_flags_bmi2) {
|
||||
return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
|
||||
}
|
||||
#endif
|
||||
(void)bmi2;
|
||||
(void)flags;
|
||||
return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
|
||||
}
|
||||
|
||||
|
||||
12
vendor/github.com/DataDog/zstd/error_private.c
generated
vendored
12
vendor/github.com/DataDog/zstd/error_private.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -28,9 +28,11 @@ const char* ERR_getErrorString(ERR_enum code)
|
||||
case PREFIX(version_unsupported): return "Version not supported";
|
||||
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
|
||||
case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
|
||||
case PREFIX(corruption_detected): return "Corrupted block detected";
|
||||
case PREFIX(corruption_detected): return "Data corruption detected";
|
||||
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
||||
case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
|
||||
case PREFIX(parameter_unsupported): return "Unsupported parameter";
|
||||
case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
|
||||
case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
|
||||
case PREFIX(init_missing): return "Context should be init first";
|
||||
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
|
||||
@@ -39,17 +41,23 @@ const char* ERR_getErrorString(ERR_enum code)
|
||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||
case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block";
|
||||
case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
|
||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
||||
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||
case PREFIX(srcSize_wrong): return "Src size is incorrect";
|
||||
case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
|
||||
case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
|
||||
case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
|
||||
/* following error codes are not stable and may be removed or changed in a future version */
|
||||
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
|
||||
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
||||
case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
|
||||
case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
|
||||
case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
|
||||
case PREFIX(externalSequences_invalid): return "External sequences are not valid";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
}
|
||||
|
||||
93
vendor/github.com/DataDog/zstd/error_private.h
generated
vendored
93
vendor/github.com/DataDog/zstd/error_private.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -14,11 +14,6 @@
|
||||
#ifndef ERROR_H_MODULE
|
||||
#define ERROR_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
@@ -27,7 +22,6 @@ extern "C" {
|
||||
#include "debug.h"
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
@@ -61,8 +55,13 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
|
||||
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
|
||||
|
||||
/* check and forward error code */
|
||||
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
|
||||
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
||||
#define CHECK_V_F(e, f) \
|
||||
size_t const e = f; \
|
||||
do { \
|
||||
if (ERR_isError(e)) \
|
||||
return e; \
|
||||
} while (0)
|
||||
#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0)
|
||||
|
||||
|
||||
/*-****************************************
|
||||
@@ -96,10 +95,12 @@ void _force_has_format_string(const char *format, ...) {
|
||||
* We want to force this function invocation to be syntactically correct, but
|
||||
* we don't want to force runtime evaluation of its arguments.
|
||||
*/
|
||||
#define _FORCE_HAS_FORMAT_STRING(...) \
|
||||
if (0) { \
|
||||
_force_has_format_string(__VA_ARGS__); \
|
||||
}
|
||||
#define _FORCE_HAS_FORMAT_STRING(...) \
|
||||
do { \
|
||||
if (0) { \
|
||||
_force_has_format_string(__VA_ARGS__); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define ERR_QUOTE(str) #str
|
||||
|
||||
@@ -110,52 +111,50 @@ void _force_has_format_string(const char *format, ...) {
|
||||
* In order to do that (particularly, printing the conditional that failed),
|
||||
* this can't just wrap RETURN_ERROR().
|
||||
*/
|
||||
#define RETURN_ERROR_IF(cond, err, ...) \
|
||||
if (cond) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
}
|
||||
#define RETURN_ERROR_IF(cond, err, ...) \
|
||||
do { \
|
||||
if (cond) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/**
|
||||
* Unconditionally return the specified error.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
*/
|
||||
#define RETURN_ERROR(err, ...) \
|
||||
do { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
} while(0);
|
||||
#define RETURN_ERROR(err, ...) \
|
||||
do { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return ERROR(err); \
|
||||
} while(0)
|
||||
|
||||
/**
|
||||
* If the provided expression evaluates to an error code, returns that error code.
|
||||
*
|
||||
* In debug modes, prints additional information.
|
||||
*/
|
||||
#define FORWARD_IF_ERROR(err, ...) \
|
||||
do { \
|
||||
size_t const err_code = (err); \
|
||||
if (ERR_isError(err_code)) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return err_code; \
|
||||
} \
|
||||
} while(0);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#define FORWARD_IF_ERROR(err, ...) \
|
||||
do { \
|
||||
size_t const err_code = (err); \
|
||||
if (ERR_isError(err_code)) { \
|
||||
RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
|
||||
__FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
|
||||
_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
|
||||
RAWLOG(3, ": " __VA_ARGS__); \
|
||||
RAWLOG(3, "\n"); \
|
||||
return err_code; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#endif /* ERROR_H_MODULE */
|
||||
|
||||
|
||||
8
vendor/github.com/DataDog/zstd/fastcover.c
generated
vendored
8
vendor/github.com/DataDog/zstd/fastcover.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -305,7 +305,7 @@ FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx)
|
||||
|
||||
/**
|
||||
* Prepare a context for dictionary building.
|
||||
* The context is only dependent on the parameter `d` and can used multiple
|
||||
* The context is only dependent on the parameter `d` and can be used multiple
|
||||
* times.
|
||||
* Returns 0 on success or error code on error.
|
||||
* The context must be destroyed with `FASTCOVER_ctx_destroy()`.
|
||||
@@ -546,7 +546,7 @@ FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams,
|
||||
}
|
||||
|
||||
|
||||
ZDICTLIB_API size_t
|
||||
ZDICTLIB_STATIC_API size_t
|
||||
ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer,
|
||||
const size_t* samplesSizes, unsigned nbSamples,
|
||||
@@ -615,7 +615,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
|
||||
}
|
||||
|
||||
|
||||
ZDICTLIB_API size_t
|
||||
ZDICTLIB_STATIC_API size_t
|
||||
ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer,
|
||||
|
||||
108
vendor/github.com/DataDog/zstd/fse.h
generated
vendored
108
vendor/github.com/DataDog/zstd/fse.h
generated
vendored
@@ -2,7 +2,7 @@
|
||||
/* ******************************************************************
|
||||
* FSE : Finite State Entropy codec
|
||||
* Public Prototypes declaration
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -12,11 +12,6 @@
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
****************************************************************** */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef FSE_H
|
||||
#define FSE_H
|
||||
|
||||
@@ -26,7 +21,6 @@ extern "C" {
|
||||
******************************************/
|
||||
#include "zstd_deps.h" /* size_t, ptrdiff_t */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE_PUBLIC_API : control library symbols visibility
|
||||
******************************************/
|
||||
@@ -54,34 +48,6 @@ extern "C" {
|
||||
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE simple functions
|
||||
******************************************/
|
||||
/*! FSE_compress() :
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
|
||||
@return : size of compressed data (<= dstCapacity).
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
|
||||
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
||||
*/
|
||||
FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/*! FSE_decompress():
|
||||
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstCapacity'.
|
||||
@return : size of regenerated data (<= maxDstSize),
|
||||
or an error code, which can be tested using FSE_isError() .
|
||||
|
||||
** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
|
||||
Why ? : making this distinction requires a header.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
@@ -92,20 +58,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return
|
||||
FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE advanced functions
|
||||
******************************************/
|
||||
/*! FSE_compress2() :
|
||||
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
|
||||
Both parameters can be defined as '0' to mean : use default value
|
||||
@return : size of compressed data
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
||||
if FSE_isError(return), it's an error code.
|
||||
*/
|
||||
FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE detailed API
|
||||
******************************************/
|
||||
@@ -165,8 +117,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
||||
/*! Constructor and Destructor of FSE_CTable.
|
||||
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
|
||||
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
|
||||
FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
|
||||
FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
|
||||
|
||||
/*! FSE_buildCTable():
|
||||
Builds `ct`, which must be already allocated, using FSE_createCTable().
|
||||
@@ -242,23 +192,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
|
||||
unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
|
||||
const void* rBuffer, size_t rBuffSize, int bmi2);
|
||||
|
||||
/*! Constructor and Destructor of FSE_DTable.
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
|
||||
FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
|
||||
|
||||
/*! FSE_buildDTable():
|
||||
Builds 'dt', which must be already allocated, using FSE_createDTable().
|
||||
return : 0, or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSE_decompress_usingDTable():
|
||||
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
|
||||
into `dst` which must be already allocated.
|
||||
@return : size of regenerated data (necessarily <= `dstCapacity`),
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
@@ -290,13 +224,11 @@ If there is an error, the function will return an error code, which can be teste
|
||||
|
||||
#endif /* FSE_H */
|
||||
|
||||
|
||||
#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
|
||||
#define FSE_H_FSE_STATIC_LINKING_ONLY
|
||||
|
||||
/* *** Dependency *** */
|
||||
#include "bitstream.h"
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Static allocation
|
||||
*******************************************/
|
||||
@@ -321,16 +253,6 @@ If there is an error, the function will return an error code, which can be teste
|
||||
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
||||
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
||||
|
||||
/* FSE_compress_wksp() :
|
||||
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
|
||||
*/
|
||||
#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
|
||||
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||
|
||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
||||
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
||||
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
|
||||
|
||||
@@ -348,19 +270,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
|
||||
FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||
/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
||||
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
|
||||
#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
|
||||
#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
|
||||
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
|
||||
|
||||
size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
|
||||
/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
|
||||
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
|
||||
* Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
|
||||
|
||||
typedef enum {
|
||||
FSE_repeat_none, /**< Cannot use the previous table */
|
||||
@@ -543,20 +457,20 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un
|
||||
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
|
||||
BIT_addBits(bitC, statePtr->value, nbBitsOut);
|
||||
BIT_addBits(bitC, (BitContainerType)statePtr->value, nbBitsOut);
|
||||
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
|
||||
{
|
||||
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
|
||||
BIT_addBits(bitC, (BitContainerType)statePtr->value, statePtr->stateLog);
|
||||
BIT_flushBits(bitC);
|
||||
}
|
||||
|
||||
|
||||
/* FSE_getMaxNbBits() :
|
||||
* Approximate maximum cost of a symbol, in bits.
|
||||
* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
|
||||
* Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
|
||||
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
||||
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
||||
MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
|
||||
@@ -709,12 +623,6 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
|
||||
#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
|
||||
|
||||
|
||||
#endif /* FSE_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
146
vendor/github.com/DataDog/zstd/fse_compress.c
generated
vendored
146
vendor/github.com/DataDog/zstd/fse_compress.c
generated
vendored
@@ -1,7 +1,7 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* FSE : Finite State Entropy encoder
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -26,7 +26,8 @@
|
||||
#include "error_private.h"
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#define ZSTD_DEPS_NEED_MATH64
|
||||
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
|
||||
#include "zstd_deps.h" /* ZSTD_memset */
|
||||
#include "bits.h" /* ZSTD_highbit32 */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -91,7 +92,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
assert(tableLog < 16); /* required for threshold strategy to work */
|
||||
|
||||
/* For explanations on how to distribute symbol values over the table :
|
||||
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
||||
* https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
||||
|
||||
#ifdef __clang_analyzer__
|
||||
ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
|
||||
@@ -192,7 +193,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
break;
|
||||
default :
|
||||
assert(normalizedCounter[s] > 1);
|
||||
{ U32 const maxBitsOut = tableLog - BIT_highbit32 ((U32)normalizedCounter[s]-1);
|
||||
{ U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1);
|
||||
U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut;
|
||||
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
|
||||
symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]);
|
||||
@@ -225,8 +226,8 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
|
||||
size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog
|
||||
+ 4 /* bitCount initialized at 4 */
|
||||
+ 2 /* first two symbols may use one additional bit each */) / 8)
|
||||
+ 1 /* round up to whole nb bytes */
|
||||
+ 2 /* additional two bytes for bitstream flush */;
|
||||
+ 1 /* round up to whole nb bytes */
|
||||
+ 2 /* additional two bytes for bitstream flush */;
|
||||
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
||||
}
|
||||
|
||||
@@ -255,7 +256,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
/* Init */
|
||||
remaining = tableSize+1; /* +1 for extra accuracy */
|
||||
threshold = tableSize;
|
||||
nbBits = tableLog+1;
|
||||
nbBits = (int)tableLog+1;
|
||||
|
||||
while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */
|
||||
if (previousIs0) {
|
||||
@@ -274,7 +275,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
}
|
||||
while (symbol >= start+3) {
|
||||
start+=3;
|
||||
bitStream += 3 << bitCount;
|
||||
bitStream += 3U << bitCount;
|
||||
bitCount += 2;
|
||||
}
|
||||
bitStream += (symbol-start) << bitCount;
|
||||
@@ -294,7 +295,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
count++; /* +1 for extra accuracy */
|
||||
if (count>=threshold)
|
||||
count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
|
||||
bitStream += count << bitCount;
|
||||
bitStream += (U32)count << bitCount;
|
||||
bitCount += nbBits;
|
||||
bitCount -= (count<max);
|
||||
previousIs0 = (count==1);
|
||||
@@ -322,7 +323,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
out[1] = (BYTE)(bitStream>>8);
|
||||
out+= (bitCount+7) /8;
|
||||
|
||||
return (out-ostart);
|
||||
assert(out >= ostart);
|
||||
return (size_t)(out-ostart);
|
||||
}
|
||||
|
||||
|
||||
@@ -343,21 +345,11 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
||||
* FSE Compression Code
|
||||
****************************************************************/
|
||||
|
||||
FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
size_t size;
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
|
||||
return (FSE_CTable*)ZSTD_malloc(size);
|
||||
}
|
||||
|
||||
void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
|
||||
|
||||
/* provides the minimum logSize to safely represent a distribution */
|
||||
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1;
|
||||
U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
|
||||
U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1;
|
||||
U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2;
|
||||
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
|
||||
assert(srcSize > 1); /* Not supported, RLE should be used instead */
|
||||
return minBits;
|
||||
@@ -365,7 +357,7 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||
|
||||
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
|
||||
{
|
||||
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
|
||||
U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus;
|
||||
U32 tableLog = maxTableLog;
|
||||
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
|
||||
assert(srcSize > 1); /* Not supported, RLE should be used instead */
|
||||
@@ -533,40 +525,6 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
return tableLog;
|
||||
}
|
||||
|
||||
|
||||
/* fake FSE_CTable, for raw (uncompressed) input */
|
||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
|
||||
{
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSymbolValue = tableMask;
|
||||
void* const ptr = ct;
|
||||
U16* const tableU16 = ( (U16*) ptr) + 2;
|
||||
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
|
||||
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
||||
unsigned s;
|
||||
|
||||
/* Sanity checks */
|
||||
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
||||
|
||||
/* header */
|
||||
tableU16[-2] = (U16) nbBits;
|
||||
tableU16[-1] = (U16) maxSymbolValue;
|
||||
|
||||
/* Build table */
|
||||
for (s=0; s<tableSize; s++)
|
||||
tableU16[s] = (U16)(tableSize + s);
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
symbolTT[s].deltaNbBits = deltaNbBits;
|
||||
symbolTT[s].deltaFindState = s-1;
|
||||
} }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* fake FSE_CTable, for rle input (always same symbol) */
|
||||
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
|
||||
{
|
||||
@@ -665,80 +623,6 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
|
||||
|
||||
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
|
||||
|
||||
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
||||
/* FSE_compress_wksp() :
|
||||
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
||||
* `wkspSize` size must be `(1<<tableLog)`.
|
||||
*/
|
||||
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
unsigned count[FSE_MAX_SYMBOL_VALUE+1];
|
||||
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
|
||||
FSE_CTable* CTable = (FSE_CTable*)workSpace;
|
||||
size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
|
||||
void* scratchBuffer = (void*)(CTable + CTableSize);
|
||||
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
|
||||
|
||||
/* init conditions */
|
||||
if (wkspSize < FSE_COMPRESS_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
|
||||
if (srcSize <= 1) return 0; /* Not compressible */
|
||||
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) );
|
||||
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */
|
||||
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
|
||||
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
|
||||
}
|
||||
|
||||
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
|
||||
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) );
|
||||
|
||||
/* Write table description header */
|
||||
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
||||
op += nc_err;
|
||||
}
|
||||
|
||||
/* Compress */
|
||||
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) );
|
||||
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) );
|
||||
if (cSize == 0) return 0; /* not enough space for compressed data */
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
/* check compressibility */
|
||||
if ( (size_t)(op-ostart) >= srcSize-1 ) return 0;
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
||||
union {
|
||||
U32 hist_wksp[HIST_WKSP_SIZE_U32];
|
||||
BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
|
||||
} workspace;
|
||||
} fseWkspMax_t;
|
||||
|
||||
size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
fseWkspMax_t scratchBuffer;
|
||||
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
|
||||
}
|
||||
|
||||
size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
144
vendor/github.com/DataDog/zstd/fse_decompress.c
generated
vendored
144
vendor/github.com/DataDog/zstd/fse_decompress.c
generated
vendored
@@ -1,7 +1,7 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* FSE : Finite State Entropy decoder
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -23,8 +23,8 @@
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#include "error_private.h"
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#include "zstd_deps.h"
|
||||
#include "zstd_deps.h" /* ZSTD_memcpy */
|
||||
#include "bits.h" /* ZSTD_highbit32 */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -56,19 +56,6 @@
|
||||
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
|
||||
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
|
||||
|
||||
|
||||
/* Function templates */
|
||||
FSE_DTable* FSE_createDTable (unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
}
|
||||
|
||||
void FSE_freeDTable (FSE_DTable* dt)
|
||||
{
|
||||
ZSTD_free(dt);
|
||||
}
|
||||
|
||||
static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
||||
@@ -97,7 +84,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
||||
symbolNext[s] = 1;
|
||||
} else {
|
||||
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
||||
symbolNext[s] = normalizedCounter[s];
|
||||
symbolNext[s] = (U16)normalizedCounter[s];
|
||||
} } }
|
||||
ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
}
|
||||
@@ -112,8 +99,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
||||
* all symbols have counts <= 8. We ensure we have 8 bytes at the end of
|
||||
* our buffer to handle the over-write.
|
||||
*/
|
||||
{
|
||||
U64 const add = 0x0101010101010101ull;
|
||||
{ U64 const add = 0x0101010101010101ull;
|
||||
size_t pos = 0;
|
||||
U64 sv = 0;
|
||||
U32 s;
|
||||
@@ -124,14 +110,13 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
||||
for (i = 8; i < n; i += 8) {
|
||||
MEM_write64(spread + pos + i, sv);
|
||||
}
|
||||
pos += n;
|
||||
}
|
||||
}
|
||||
pos += (size_t)n;
|
||||
} }
|
||||
/* Now we spread those positions across the table.
|
||||
* The benefit of doing it in two stages is that we avoid the the
|
||||
* The benefit of doing it in two stages is that we avoid the
|
||||
* variable size inner loop, which caused lots of branch misses.
|
||||
* Now we can run through all the positions without any branch misses.
|
||||
* We unroll the loop twice, since that is what emperically worked best.
|
||||
* We unroll the loop twice, since that is what empirically worked best.
|
||||
*/
|
||||
{
|
||||
size_t position = 0;
|
||||
@@ -167,7 +152,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
||||
for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
U32 const nextState = symbolNext[symbol]++;
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
|
||||
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
||||
} }
|
||||
|
||||
@@ -185,49 +170,6 @@ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsi
|
||||
/*-*******************************************************
|
||||
* Decompression (Byte symbols)
|
||||
*********************************************************/
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
|
||||
|
||||
DTableH->tableLog = 0;
|
||||
DTableH->fastMode = 0;
|
||||
|
||||
cell->newState = 0;
|
||||
cell->symbol = symbolValue;
|
||||
cell->nbBits = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSV1 = tableMask+1;
|
||||
unsigned s;
|
||||
|
||||
/* Sanity checks */
|
||||
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
||||
|
||||
/* Build Decoding Table */
|
||||
DTableH->tableLog = (U16)nbBits;
|
||||
DTableH->fastMode = 1;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
dinfo[s].newState = 0;
|
||||
dinfo[s].symbol = (BYTE)s;
|
||||
dinfo[s].nbBits = (BYTE)nbBits;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
|
||||
void* dst, size_t maxDstSize,
|
||||
@@ -249,6 +191,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
|
||||
FSE_initDState(&state1, &bitD, dt);
|
||||
FSE_initDState(&state2, &bitD, dt);
|
||||
|
||||
RETURN_ERROR_IF(BIT_reloadDStream(&bitD)==BIT_DStream_overflow, corruption_detected, "");
|
||||
|
||||
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
|
||||
|
||||
/* 4 symbols per loop */
|
||||
@@ -288,32 +232,12 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
|
||||
break;
|
||||
} }
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSE_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
||||
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
|
||||
assert(op >= ostart);
|
||||
return (size_t)(op-ostart);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
short ncount[FSE_MAX_SYMBOL_VALUE + 1];
|
||||
FSE_DTable dtable[1]; /* Dynamically sized */
|
||||
} FSE_DecompressWksp;
|
||||
|
||||
|
||||
@@ -328,13 +252,18 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
|
||||
unsigned tableLog;
|
||||
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
|
||||
size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable);
|
||||
FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos;
|
||||
|
||||
DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
|
||||
FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
|
||||
if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
|
||||
|
||||
/* correct offset to dtable depends on this property */
|
||||
FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0);
|
||||
|
||||
/* normal FSE decoding mode */
|
||||
{
|
||||
size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
|
||||
{ size_t const NCountLength =
|
||||
FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
|
||||
if (FSE_isError(NCountLength)) return NCountLength;
|
||||
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
|
||||
assert(NCountLength <= cSrcSize);
|
||||
@@ -343,19 +272,20 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
|
||||
}
|
||||
|
||||
if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
|
||||
workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
|
||||
assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize);
|
||||
workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
|
||||
wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
|
||||
|
||||
CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
|
||||
CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
|
||||
|
||||
{
|
||||
const void* ptr = wksp->dtable;
|
||||
const void* ptr = dtable;
|
||||
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
|
||||
return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
|
||||
if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 1);
|
||||
return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -383,24 +313,6 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc,
|
||||
return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
|
||||
}
|
||||
|
||||
|
||||
typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
||||
|
||||
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
||||
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
|
||||
U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
|
||||
return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
|
||||
}
|
||||
|
||||
size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
||||
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
12
vendor/github.com/DataDog/zstd/hist.c
generated
vendored
12
vendor/github.com/DataDog/zstd/hist.c
generated
vendored
@@ -2,7 +2,7 @@
|
||||
/* ******************************************************************
|
||||
* hist : Histogram functions
|
||||
* part of Finite State Entropy project
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -27,6 +27,16 @@ unsigned HIST_isError(size_t code) { return ERR_isError(code); }
|
||||
/*-**************************************************************
|
||||
* Histogram functions
|
||||
****************************************************************/
|
||||
void HIST_add(unsigned* count, const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const end = ip + srcSize;
|
||||
|
||||
while (ip<end) {
|
||||
count[*ip++]++;
|
||||
}
|
||||
}
|
||||
|
||||
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
|
||||
9
vendor/github.com/DataDog/zstd/hist.h
generated
vendored
9
vendor/github.com/DataDog/zstd/hist.h
generated
vendored
@@ -2,7 +2,7 @@
|
||||
/* ******************************************************************
|
||||
* hist : Histogram functions
|
||||
* part of Finite State Entropy project
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -75,4 +75,11 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/*! HIST_add() :
|
||||
* Lowest level: just add nb of occurrences of characters from @src into @count.
|
||||
* @count is not reset. @count array is presumed large enough (i.e. 1 KB).
|
||||
@ This function does not need any additional stack memory.
|
||||
*/
|
||||
void HIST_add(unsigned* count, const void* src, size_t srcSize);
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
255
vendor/github.com/DataDog/zstd/huf.h
generated
vendored
255
vendor/github.com/DataDog/zstd/huf.h
generated
vendored
@@ -2,7 +2,7 @@
|
||||
/* ******************************************************************
|
||||
* huff0 huffman codec,
|
||||
* part of Finite State Entropy library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -13,108 +13,26 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
****************************************************************** */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef HUF_H_298734234
|
||||
#define HUF_H_298734234
|
||||
|
||||
/* *** Dependencies *** */
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
|
||||
|
||||
/* *** library symbols visibility *** */
|
||||
/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
|
||||
* HUF symbols remain "private" (internal symbols for library only).
|
||||
* Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
|
||||
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
|
||||
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
|
||||
# define HUF_PUBLIC_API __declspec(dllexport)
|
||||
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
|
||||
# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
|
||||
#else
|
||||
# define HUF_PUBLIC_API
|
||||
#endif
|
||||
|
||||
|
||||
/* ========================== */
|
||||
/* *** simple functions *** */
|
||||
/* ========================== */
|
||||
|
||||
/** HUF_compress() :
|
||||
* Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
|
||||
* 'dst' buffer must be already allocated.
|
||||
* Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
|
||||
* `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
|
||||
* @return : size of compressed data (<= `dstCapacity`).
|
||||
* Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
* if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
||||
*/
|
||||
HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/** HUF_decompress() :
|
||||
* Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||
* into already allocated buffer 'dst', of minimum size 'dstSize'.
|
||||
* `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
||||
* Note : in contrast with FSE, HUF_decompress can regenerate
|
||||
* RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
||||
* because it knows size to regenerate (originalSize).
|
||||
* @return : size of regenerated data (== originalSize),
|
||||
* or an error code, which can be tested using HUF_isError()
|
||||
*/
|
||||
HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/* *** Tool functions *** */
|
||||
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
|
||||
HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
|
||||
|
||||
/* Error Management */
|
||||
HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
|
||||
HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/* *** Advanced function *** */
|
||||
|
||||
/** HUF_compress2() :
|
||||
* Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
|
||||
* `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
|
||||
* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
|
||||
HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/** HUF_compress4X_wksp() :
|
||||
* Same as HUF_compress2(), but uses externally allocated `workSpace`.
|
||||
* `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
|
||||
#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
|
||||
#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
|
||||
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize);
|
||||
|
||||
#endif /* HUF_H_298734234 */
|
||||
|
||||
/* ******************************************************************
|
||||
* WARNING !!
|
||||
* The following section contains advanced and experimental definitions
|
||||
* which shall never be used in the context of a dynamic library,
|
||||
* because they are not guaranteed to remain stable in the future.
|
||||
* Only consider them in association with static linking.
|
||||
* *****************************************************************/
|
||||
#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
|
||||
#define HUF_H_HUF_STATIC_LINKING_ONLY
|
||||
|
||||
/* *** Dependencies *** */
|
||||
#include "mem.h" /* U32 */
|
||||
#include "mem.h" /* U32 */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
|
||||
/* *** Tool functions *** */
|
||||
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
|
||||
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
|
||||
|
||||
/* Error Management */
|
||||
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
|
||||
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
|
||||
#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
|
||||
|
||||
/* *** Constants *** */
|
||||
#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
|
||||
@@ -155,25 +73,49 @@ typedef U32 HUF_DTable;
|
||||
/* ****************************************
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
||||
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
||||
#endif
|
||||
/**
|
||||
* Huffman flags bitset.
|
||||
* For all flags, 0 is the default value.
|
||||
*/
|
||||
typedef enum {
|
||||
/**
|
||||
* If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
|
||||
* Otherwise: Ignored.
|
||||
*/
|
||||
HUF_flags_bmi2 = (1 << 0),
|
||||
/**
|
||||
* If set: Test possible table depths to find the one that produces the smallest header + encoded size.
|
||||
* If unset: Use heuristic to find the table depth.
|
||||
*/
|
||||
HUF_flags_optimalDepth = (1 << 1),
|
||||
/**
|
||||
* If set: If the previous table can encode the input, always reuse the previous table.
|
||||
* If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
|
||||
*/
|
||||
HUF_flags_preferRepeat = (1 << 2),
|
||||
/**
|
||||
* If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
|
||||
* If unset: Always histogram the entire input.
|
||||
*/
|
||||
HUF_flags_suspectUncompressible = (1 << 3),
|
||||
/**
|
||||
* If set: Don't use assembly implementations
|
||||
* If unset: Allow using assembly implementations
|
||||
*/
|
||||
HUF_flags_disableAsm = (1 << 4),
|
||||
/**
|
||||
* If set: Don't use the fast decoding loop, always use the fallback decoding loop.
|
||||
* If unset: Use the fast decoding loop when possible.
|
||||
*/
|
||||
HUF_flags_disableFast = (1 << 5)
|
||||
} HUF_flags_e;
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* HUF detailed API
|
||||
* ****************************************/
|
||||
#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
|
||||
|
||||
/*! HUF_compress() does the following:
|
||||
* 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
|
||||
@@ -186,12 +128,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
|
||||
* For example, it's possible to compress several blocks using the same 'CTable',
|
||||
* or to save and regenerate 'CTable' using external methods.
|
||||
*/
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
||||
unsigned HUF_minTableLog(unsigned symbolCardinality);
|
||||
unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
|
||||
size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
|
||||
size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
|
||||
size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
|
||||
int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
|
||||
|
||||
@@ -200,6 +142,7 @@ typedef enum {
|
||||
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
|
||||
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
||||
} HUF_repeat;
|
||||
|
||||
/** HUF_compress4X_repeat() :
|
||||
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
* If it uses hufTable it does not modify hufTable or repeat.
|
||||
@@ -210,13 +153,13 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
|
||||
|
||||
/** HUF_buildCTable_wksp() :
|
||||
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
||||
* `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
|
||||
*/
|
||||
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
|
||||
#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
|
||||
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
|
||||
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
|
||||
const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
|
||||
@@ -242,7 +185,7 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
|
||||
U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize,
|
||||
void* workspace, size_t wkspSize,
|
||||
int bmi2);
|
||||
int flags);
|
||||
|
||||
/** HUF_readCTable() :
|
||||
* Loading a CTable saved with HUF_writeCTable() */
|
||||
@@ -250,9 +193,22 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
|
||||
|
||||
/** HUF_getNbBitsFromCTable() :
|
||||
* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
|
||||
* Note 1 : is not inlined, as HUF_CElt definition is private */
|
||||
* Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0
|
||||
* Note 2 : is not inlined, as HUF_CElt definition is private
|
||||
*/
|
||||
U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
|
||||
|
||||
typedef struct {
|
||||
BYTE tableLog;
|
||||
BYTE maxSymbolValue;
|
||||
BYTE unused[sizeof(size_t) - 2];
|
||||
} HUF_CTableHeader;
|
||||
|
||||
/** HUF_readCTableHeader() :
|
||||
* @returns The header from the CTable specifying the tableLog and the maxSymbolValue.
|
||||
*/
|
||||
HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable);
|
||||
|
||||
/*
|
||||
* HUF_decompress() does the following:
|
||||
* 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
|
||||
@@ -280,32 +236,12 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
||||
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#endif
|
||||
|
||||
|
||||
/* ====================== */
|
||||
/* single stream variants */
|
||||
/* ====================== */
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
|
||||
/** HUF_compress1X_repeat() :
|
||||
* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
* If it uses hufTable it does not modify hufTable or repeat.
|
||||
@@ -316,52 +252,29 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned tableLog,
|
||||
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
|
||||
|
||||
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */
|
||||
#endif
|
||||
|
||||
/* BMI2 variants.
|
||||
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
|
||||
*/
|
||||
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
||||
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
|
||||
#endif
|
||||
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
||||
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
|
||||
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
|
||||
#endif
|
||||
|
||||
#endif /* HUF_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#endif /* HUF_H_298734234 */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
472
vendor/github.com/DataDog/zstd/huf_compress.c
generated
vendored
472
vendor/github.com/DataDog/zstd/huf_compress.c
generated
vendored
@@ -1,7 +1,7 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/* ******************************************************************
|
||||
* Huffman encoder, part of New Generation Entropy library
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
@@ -30,9 +30,9 @@
|
||||
#include "hist.h"
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
|
||||
#include "fse.h" /* header compression */
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "error_private.h"
|
||||
#include "bits.h" /* ZSTD_highbit32 */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -43,13 +43,67 @@
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Utils
|
||||
* Required declarations
|
||||
****************************************************************/
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
||||
typedef struct nodeElt_s {
|
||||
U32 count;
|
||||
U16 parent;
|
||||
BYTE byte;
|
||||
BYTE nbBits;
|
||||
} nodeElt;
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Debug Traces
|
||||
****************************************************************/
|
||||
|
||||
#if DEBUGLEVEL >= 2
|
||||
|
||||
static size_t showU32(const U32* arr, size_t size)
|
||||
{
|
||||
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
|
||||
size_t u;
|
||||
for (u=0; u<size; u++) {
|
||||
RAWLOG(6, " %u", arr[u]); (void)arr;
|
||||
}
|
||||
RAWLOG(6, " \n");
|
||||
return size;
|
||||
}
|
||||
|
||||
static size_t HUF_getNbBits(HUF_CElt elt);
|
||||
|
||||
static size_t showCTableBits(const HUF_CElt* ctable, size_t size)
|
||||
{
|
||||
size_t u;
|
||||
for (u=0; u<size; u++) {
|
||||
RAWLOG(6, " %zu", HUF_getNbBits(ctable[u])); (void)ctable;
|
||||
}
|
||||
RAWLOG(6, " \n");
|
||||
return size;
|
||||
|
||||
}
|
||||
|
||||
static size_t showHNodeSymbols(const nodeElt* hnode, size_t size)
|
||||
{
|
||||
size_t u;
|
||||
for (u=0; u<size; u++) {
|
||||
RAWLOG(6, " %u", hnode[u].byte); (void)hnode;
|
||||
}
|
||||
RAWLOG(6, " \n");
|
||||
return size;
|
||||
}
|
||||
|
||||
static size_t showHNodeBits(const nodeElt* hnode, size_t size)
|
||||
{
|
||||
size_t u;
|
||||
for (u=0; u<size; u++) {
|
||||
RAWLOG(6, " %u", hnode[u].nbBits); (void)hnode;
|
||||
}
|
||||
RAWLOG(6, " \n");
|
||||
return size;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* HUF : Huffman block compression
|
||||
@@ -90,7 +144,10 @@ typedef struct {
|
||||
S16 norm[HUF_TABLELOG_MAX+1];
|
||||
} HUF_CompressWeightsWksp;
|
||||
|
||||
static size_t HUF_compressWeights(void* dst, size_t dstSize, const void* weightTable, size_t wtSize, void* workspace, size_t workspaceSize)
|
||||
static size_t
|
||||
HUF_compressWeights(void* dst, size_t dstSize,
|
||||
const void* weightTable, size_t wtSize,
|
||||
void* workspace, size_t workspaceSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
@@ -141,7 +198,7 @@ static size_t HUF_getNbBitsFast(HUF_CElt elt)
|
||||
|
||||
static size_t HUF_getValue(HUF_CElt elt)
|
||||
{
|
||||
return elt & ~0xFF;
|
||||
return elt & ~(size_t)0xFF;
|
||||
}
|
||||
|
||||
static size_t HUF_getValueFast(HUF_CElt elt)
|
||||
@@ -164,6 +221,25 @@ static void HUF_setValue(HUF_CElt* elt, size_t value)
|
||||
}
|
||||
}
|
||||
|
||||
HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable)
|
||||
{
|
||||
HUF_CTableHeader header;
|
||||
ZSTD_memcpy(&header, ctable, sizeof(header));
|
||||
return header;
|
||||
}
|
||||
|
||||
static void HUF_writeCTableHeader(HUF_CElt* ctable, U32 tableLog, U32 maxSymbolValue)
|
||||
{
|
||||
HUF_CTableHeader header;
|
||||
HUF_STATIC_ASSERT(sizeof(ctable[0]) == sizeof(header));
|
||||
ZSTD_memset(&header, 0, sizeof(header));
|
||||
assert(tableLog < 256);
|
||||
header.tableLog = (BYTE)tableLog;
|
||||
assert(maxSymbolValue < 256);
|
||||
header.maxSymbolValue = (BYTE)maxSymbolValue;
|
||||
ZSTD_memcpy(ctable, &header, sizeof(header));
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
HUF_CompressWeightsWksp wksp;
|
||||
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
|
||||
@@ -179,6 +255,11 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
|
||||
U32 n;
|
||||
HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
|
||||
|
||||
HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp));
|
||||
|
||||
assert(HUF_readCTableHeader(CTable).maxSymbolValue == maxSymbolValue);
|
||||
assert(HUF_readCTableHeader(CTable).tableLog == huffLog);
|
||||
|
||||
/* check conditions */
|
||||
if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
|
||||
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
||||
@@ -208,16 +289,6 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
|
||||
return ((maxSymbolValue+1)/2) + 1;
|
||||
}
|
||||
|
||||
/*! HUF_writeCTable() :
|
||||
`CTable` : Huffman tree to save, using huf representation.
|
||||
@return : size of saved CTable */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
HUF_WriteCTableWksp wksp;
|
||||
return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp));
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights)
|
||||
{
|
||||
@@ -235,7 +306,9 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
|
||||
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
||||
if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall);
|
||||
|
||||
CTable[0] = tableLog;
|
||||
*maxSymbolValuePtr = nbSymbols - 1;
|
||||
|
||||
HUF_writeCTableHeader(CTable, tableLog, *maxSymbolValuePtr);
|
||||
|
||||
/* Prepare base value per rank */
|
||||
{ U32 n, nextRankStart = 0;
|
||||
@@ -267,74 +340,71 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); }
|
||||
}
|
||||
|
||||
*maxSymbolValuePtr = nbSymbols - 1;
|
||||
return readSize;
|
||||
}
|
||||
|
||||
U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue)
|
||||
{
|
||||
const HUF_CElt* ct = CTable + 1;
|
||||
const HUF_CElt* const ct = CTable + 1;
|
||||
assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
|
||||
if (symbolValue > HUF_readCTableHeader(CTable).maxSymbolValue)
|
||||
return 0;
|
||||
return (U32)HUF_getNbBits(ct[symbolValue]);
|
||||
}
|
||||
|
||||
|
||||
typedef struct nodeElt_s {
|
||||
U32 count;
|
||||
U16 parent;
|
||||
BYTE byte;
|
||||
BYTE nbBits;
|
||||
} nodeElt;
|
||||
|
||||
/**
|
||||
* HUF_setMaxHeight():
|
||||
* Enforces maxNbBits on the Huffman tree described in huffNode.
|
||||
* Try to enforce @targetNbBits on the Huffman tree described in @huffNode.
|
||||
*
|
||||
* It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts
|
||||
* the tree to so that it is a valid canonical Huffman tree.
|
||||
* It attempts to convert all nodes with nbBits > @targetNbBits
|
||||
* to employ @targetNbBits instead. Then it adjusts the tree
|
||||
* so that it remains a valid canonical Huffman tree.
|
||||
*
|
||||
* @pre The sum of the ranks of each symbol == 2^largestBits,
|
||||
* where largestBits == huffNode[lastNonNull].nbBits.
|
||||
* @post The sum of the ranks of each symbol == 2^largestBits,
|
||||
* where largestBits is the return value <= maxNbBits.
|
||||
* where largestBits is the return value (expected <= targetNbBits).
|
||||
*
|
||||
* @param huffNode The Huffman tree modified in place to enforce maxNbBits.
|
||||
* @param huffNode The Huffman tree modified in place to enforce targetNbBits.
|
||||
* It's presumed sorted, from most frequent to rarest symbol.
|
||||
* @param lastNonNull The symbol with the lowest count in the Huffman tree.
|
||||
* @param maxNbBits The maximum allowed number of bits, which the Huffman tree
|
||||
* @param targetNbBits The allowed number of bits, which the Huffman tree
|
||||
* may not respect. After this function the Huffman tree will
|
||||
* respect maxNbBits.
|
||||
* @return The maximum number of bits of the Huffman tree after adjustment,
|
||||
* necessarily no more than maxNbBits.
|
||||
* respect targetNbBits.
|
||||
* @return The maximum number of bits of the Huffman tree after adjustment.
|
||||
*/
|
||||
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits)
|
||||
{
|
||||
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
||||
/* early exit : no elt > maxNbBits, so the tree is already valid. */
|
||||
if (largestBits <= maxNbBits) return largestBits;
|
||||
/* early exit : no elt > targetNbBits, so the tree is already valid. */
|
||||
if (largestBits <= targetNbBits) return largestBits;
|
||||
|
||||
DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits);
|
||||
|
||||
/* there are several too large elements (at least >= 2) */
|
||||
{ int totalCost = 0;
|
||||
const U32 baseCost = 1 << (largestBits - maxNbBits);
|
||||
const U32 baseCost = 1 << (largestBits - targetNbBits);
|
||||
int n = (int)lastNonNull;
|
||||
|
||||
/* Adjust any ranks > maxNbBits to maxNbBits.
|
||||
/* Adjust any ranks > targetNbBits to targetNbBits.
|
||||
* Compute totalCost, which is how far the sum of the ranks is
|
||||
* we are over 2^largestBits after adjust the offending ranks.
|
||||
*/
|
||||
while (huffNode[n].nbBits > maxNbBits) {
|
||||
while (huffNode[n].nbBits > targetNbBits) {
|
||||
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
|
||||
huffNode[n].nbBits = (BYTE)maxNbBits;
|
||||
huffNode[n].nbBits = (BYTE)targetNbBits;
|
||||
n--;
|
||||
}
|
||||
/* n stops at huffNode[n].nbBits <= maxNbBits */
|
||||
assert(huffNode[n].nbBits <= maxNbBits);
|
||||
/* n end at index of smallest symbol using < maxNbBits */
|
||||
while (huffNode[n].nbBits == maxNbBits) --n;
|
||||
/* n stops at huffNode[n].nbBits <= targetNbBits */
|
||||
assert(huffNode[n].nbBits <= targetNbBits);
|
||||
/* n end at index of smallest symbol using < targetNbBits */
|
||||
while (huffNode[n].nbBits == targetNbBits) --n;
|
||||
|
||||
/* renorm totalCost from 2^largestBits to 2^maxNbBits
|
||||
/* renorm totalCost from 2^largestBits to 2^targetNbBits
|
||||
* note : totalCost is necessarily a multiple of baseCost */
|
||||
assert((totalCost & (baseCost - 1)) == 0);
|
||||
totalCost >>= (largestBits - maxNbBits);
|
||||
assert(((U32)totalCost & (baseCost - 1)) == 0);
|
||||
totalCost >>= (largestBits - targetNbBits);
|
||||
assert(totalCost > 0);
|
||||
|
||||
/* repay normalized cost */
|
||||
@@ -343,19 +413,19 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
|
||||
/* Get pos of last (smallest = lowest cum. count) symbol per rank */
|
||||
ZSTD_memset(rankLast, 0xF0, sizeof(rankLast));
|
||||
{ U32 currentNbBits = maxNbBits;
|
||||
{ U32 currentNbBits = targetNbBits;
|
||||
int pos;
|
||||
for (pos=n ; pos >= 0; pos--) {
|
||||
if (huffNode[pos].nbBits >= currentNbBits) continue;
|
||||
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
|
||||
rankLast[maxNbBits-currentNbBits] = (U32)pos;
|
||||
currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */
|
||||
rankLast[targetNbBits-currentNbBits] = (U32)pos;
|
||||
} }
|
||||
|
||||
while (totalCost > 0) {
|
||||
/* Try to reduce the next power of 2 above totalCost because we
|
||||
* gain back half the rank.
|
||||
*/
|
||||
U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1;
|
||||
U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1;
|
||||
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
|
||||
U32 const highPos = rankLast[nBitsToDecrease];
|
||||
U32 const lowPos = rankLast[nBitsToDecrease-1];
|
||||
@@ -395,7 +465,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
rankLast[nBitsToDecrease] = noSymbol;
|
||||
else {
|
||||
rankLast[nBitsToDecrease]--;
|
||||
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
|
||||
if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease)
|
||||
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
|
||||
}
|
||||
} /* while (totalCost > 0) */
|
||||
@@ -407,11 +477,11 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
* TODO.
|
||||
*/
|
||||
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
|
||||
/* special case : no rank 1 symbol (using maxNbBits-1);
|
||||
* let's create one from largest rank 0 (using maxNbBits).
|
||||
/* special case : no rank 1 symbol (using targetNbBits-1);
|
||||
* let's create one from largest rank 0 (using targetNbBits).
|
||||
*/
|
||||
if (rankLast[1] == noSymbol) {
|
||||
while (huffNode[n].nbBits == maxNbBits) n--;
|
||||
while (huffNode[n].nbBits == targetNbBits) n--;
|
||||
huffNode[n+1].nbBits--;
|
||||
assert(n >= 0);
|
||||
rankLast[1] = (U32)(n+1);
|
||||
@@ -425,7 +495,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
} /* repay normalized cost */
|
||||
} /* there are several too large elements (at least >= 2) */
|
||||
|
||||
return maxNbBits;
|
||||
return targetNbBits;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
@@ -433,7 +503,7 @@ typedef struct {
|
||||
U16 curr;
|
||||
} rankPos;
|
||||
|
||||
typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
|
||||
typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)];
|
||||
|
||||
/* Number of buckets available for HUF_sort() */
|
||||
#define RANK_POSITION_TABLE_SIZE 192
|
||||
@@ -452,8 +522,8 @@ typedef struct {
|
||||
* Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing.
|
||||
*/
|
||||
#define RANK_POSITION_MAX_COUNT_LOG 32
|
||||
#define RANK_POSITION_LOG_BUCKETS_BEGIN (RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */
|
||||
#define RANK_POSITION_DISTINCT_COUNT_CUTOFF RANK_POSITION_LOG_BUCKETS_BEGIN + BIT_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */
|
||||
#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */)
|
||||
#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */)
|
||||
|
||||
/* Return the appropriate bucket index for a given count. See definition of
|
||||
* RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy.
|
||||
@@ -461,7 +531,7 @@ typedef struct {
|
||||
static U32 HUF_getIndex(U32 const count) {
|
||||
return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF)
|
||||
? count
|
||||
: BIT_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN;
|
||||
: ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN;
|
||||
}
|
||||
|
||||
/* Helper swap function for HUF_quickSortPartition() */
|
||||
@@ -584,7 +654,7 @@ static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSy
|
||||
|
||||
/* Sort each bucket. */
|
||||
for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) {
|
||||
U32 const bucketSize = rankPosition[n].curr-rankPosition[n].base;
|
||||
int const bucketSize = rankPosition[n].curr - rankPosition[n].base;
|
||||
U32 const bucketStartIdx = rankPosition[n].base;
|
||||
if (bucketSize > 1) {
|
||||
assert(bucketStartIdx < maxSymbolValue1);
|
||||
@@ -595,6 +665,7 @@ static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSy
|
||||
assert(HUF_isSorted(huffNode, maxSymbolValue1));
|
||||
}
|
||||
|
||||
|
||||
/** HUF_buildCTable_wksp() :
|
||||
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
||||
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables).
|
||||
@@ -615,6 +686,7 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
|
||||
int lowS, lowN;
|
||||
int nodeNb = STARTNODE;
|
||||
int n, nodeRoot;
|
||||
DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1);
|
||||
/* init for parents */
|
||||
nonNullRank = (int)maxSymbolValue;
|
||||
while(huffNode[nonNullRank].count == 0) nonNullRank--;
|
||||
@@ -641,6 +713,8 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
||||
|
||||
DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1));
|
||||
|
||||
return nonNullRank;
|
||||
}
|
||||
|
||||
@@ -675,31 +749,40 @@ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, i
|
||||
HUF_setNbBits(ct + huffNode[n].byte, huffNode[n].nbBits); /* push nbBits per symbol, symbol order */
|
||||
for (n=0; n<alphabetSize; n++)
|
||||
HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); /* assign value within rank, symbol order */
|
||||
CTable[0] = maxNbBits;
|
||||
|
||||
HUF_writeCTableHeader(CTable, maxNbBits, maxSymbolValue);
|
||||
}
|
||||
|
||||
size_t HUF_buildCTable_wksp (HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
|
||||
size_t
|
||||
HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32));
|
||||
HUF_buildCTable_wksp_tables* const wksp_tables =
|
||||
(HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32));
|
||||
nodeElt* const huffNode0 = wksp_tables->huffNodeTbl;
|
||||
nodeElt* const huffNode = huffNode0+1;
|
||||
int nonNullRank;
|
||||
|
||||
HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables));
|
||||
|
||||
DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1);
|
||||
|
||||
/* safety checks */
|
||||
if (wkspSize < sizeof(HUF_buildCTable_wksp_tables))
|
||||
return ERROR(workSpace_tooSmall);
|
||||
return ERROR(workSpace_tooSmall);
|
||||
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
||||
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
|
||||
return ERROR(maxSymbolValue_tooLarge);
|
||||
return ERROR(maxSymbolValue_tooLarge);
|
||||
ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable));
|
||||
|
||||
/* sort, decreasing order */
|
||||
HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition);
|
||||
DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1));
|
||||
|
||||
/* build tree */
|
||||
nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
|
||||
|
||||
/* enforce maxTableLog */
|
||||
/* determine and enforce maxTableLog */
|
||||
maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits);
|
||||
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
|
||||
|
||||
@@ -720,13 +803,20 @@ size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count,
|
||||
}
|
||||
|
||||
int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
|
||||
HUF_CElt const* ct = CTable + 1;
|
||||
int bad = 0;
|
||||
int s;
|
||||
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
||||
bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0);
|
||||
}
|
||||
return !bad;
|
||||
HUF_CTableHeader header = HUF_readCTableHeader(CTable);
|
||||
HUF_CElt const* ct = CTable + 1;
|
||||
int bad = 0;
|
||||
int s;
|
||||
|
||||
assert(header.tableLog <= HUF_TABLELOG_ABSOLUTEMAX);
|
||||
|
||||
if (header.maxSymbolValue < maxSymbolValue)
|
||||
return 0;
|
||||
|
||||
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
||||
bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0);
|
||||
}
|
||||
return !bad;
|
||||
}
|
||||
|
||||
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
|
||||
@@ -808,7 +898,7 @@ FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int id
|
||||
#if DEBUGLEVEL >= 1
|
||||
{
|
||||
size_t const nbBits = HUF_getNbBits(elt);
|
||||
size_t const dirtyBits = nbBits == 0 ? 0 : BIT_highbit32((U32)nbBits) + 1;
|
||||
size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1;
|
||||
(void)dirtyBits;
|
||||
/* Middle bits are 0. */
|
||||
assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0);
|
||||
@@ -888,7 +978,7 @@ static size_t HUF_closeCStream(HUF_CStream_t* bitC)
|
||||
{
|
||||
size_t const nbBits = bitC->bitPos[0] & 0xFF;
|
||||
if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
|
||||
return (bitC->ptr - bitC->startPtr) + (nbBits > 0);
|
||||
return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -968,17 +1058,17 @@ HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable)
|
||||
{
|
||||
U32 const tableLog = (U32)CTable[0];
|
||||
U32 const tableLog = HUF_readCTableHeader(CTable).tableLog;
|
||||
HUF_CElt const* ct = CTable + 1;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
HUF_CStream_t bitC;
|
||||
|
||||
/* init */
|
||||
if (dstSize < 8) return 0; /* not enough space to compress */
|
||||
{ size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op));
|
||||
{ BYTE* op = ostart;
|
||||
size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op));
|
||||
if (HUF_isError(initErr)) return 0; }
|
||||
|
||||
if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11)
|
||||
@@ -1049,9 +1139,9 @@ HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize,
|
||||
static size_t
|
||||
HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable, const int bmi2)
|
||||
const HUF_CElt* CTable, const int flags)
|
||||
{
|
||||
if (bmi2) {
|
||||
if (flags & HUF_flags_bmi2) {
|
||||
return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
|
||||
}
|
||||
return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
|
||||
@@ -1062,28 +1152,23 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
static size_t
|
||||
HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable, const int bmi2)
|
||||
const HUF_CElt* CTable, const int flags)
|
||||
{
|
||||
(void)bmi2;
|
||||
(void)flags;
|
||||
return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
|
||||
{
|
||||
return HUF_compress1X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
|
||||
{
|
||||
return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
|
||||
return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
|
||||
}
|
||||
|
||||
static size_t
|
||||
HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const HUF_CElt* CTable, int bmi2)
|
||||
const HUF_CElt* CTable, int flags)
|
||||
{
|
||||
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
@@ -1097,7 +1182,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
op += 6; /* jumpTable */
|
||||
|
||||
assert(op <= oend);
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
|
||||
if (cSize == 0 || cSize > 65535) return 0;
|
||||
MEM_writeLE16(ostart, (U16)cSize);
|
||||
op += cSize;
|
||||
@@ -1105,7 +1190,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
|
||||
ip += segmentSize;
|
||||
assert(op <= oend);
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
|
||||
if (cSize == 0 || cSize > 65535) return 0;
|
||||
MEM_writeLE16(ostart+2, (U16)cSize);
|
||||
op += cSize;
|
||||
@@ -1113,7 +1198,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
|
||||
ip += segmentSize;
|
||||
assert(op <= oend);
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
|
||||
if (cSize == 0 || cSize > 65535) return 0;
|
||||
MEM_writeLE16(ostart+4, (U16)cSize);
|
||||
op += cSize;
|
||||
@@ -1122,7 +1207,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
ip += segmentSize;
|
||||
assert(op <= oend);
|
||||
assert(ip <= iend);
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) );
|
||||
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) );
|
||||
if (cSize == 0 || cSize > 65535) return 0;
|
||||
op += cSize;
|
||||
}
|
||||
@@ -1130,14 +1215,9 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
||||
return (size_t)(op-ostart);
|
||||
}
|
||||
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
|
||||
{
|
||||
return HUF_compress4X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
|
||||
{
|
||||
return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
|
||||
return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
|
||||
}
|
||||
|
||||
typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
|
||||
@@ -1145,11 +1225,11 @@ typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
|
||||
static size_t HUF_compressCTable_internal(
|
||||
BYTE* const ostart, BYTE* op, BYTE* const oend,
|
||||
const void* src, size_t srcSize,
|
||||
HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
|
||||
HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags)
|
||||
{
|
||||
size_t const cSize = (nbStreams==HUF_singleStream) ?
|
||||
HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) :
|
||||
HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2);
|
||||
HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) :
|
||||
HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags);
|
||||
if (HUF_isError(cSize)) { return cSize; }
|
||||
if (cSize==0) { return 0; } /* uncompressible */
|
||||
op += cSize;
|
||||
@@ -1172,6 +1252,81 @@ typedef struct {
|
||||
#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096
|
||||
#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */
|
||||
|
||||
unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue)
|
||||
{
|
||||
unsigned cardinality = 0;
|
||||
unsigned i;
|
||||
|
||||
for (i = 0; i < maxSymbolValue + 1; i++) {
|
||||
if (count[i] != 0) cardinality += 1;
|
||||
}
|
||||
|
||||
return cardinality;
|
||||
}
|
||||
|
||||
unsigned HUF_minTableLog(unsigned symbolCardinality)
|
||||
{
|
||||
U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1;
|
||||
return minBitsSymbols;
|
||||
}
|
||||
|
||||
unsigned HUF_optimalTableLog(
|
||||
unsigned maxTableLog,
|
||||
size_t srcSize,
|
||||
unsigned maxSymbolValue,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* table,
|
||||
const unsigned* count,
|
||||
int flags)
|
||||
{
|
||||
assert(srcSize > 1); /* Not supported, RLE should be used instead */
|
||||
assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
|
||||
|
||||
if (!(flags & HUF_flags_optimalDepth)) {
|
||||
/* cheap evaluation, based on FSE */
|
||||
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
|
||||
}
|
||||
|
||||
{ BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
|
||||
size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
|
||||
size_t hSize, newSize;
|
||||
const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
|
||||
const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
|
||||
size_t optSize = ((size_t) ~0) - 1;
|
||||
unsigned optLog = maxTableLog, optLogGuess;
|
||||
|
||||
DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
|
||||
|
||||
/* Search until size increases */
|
||||
for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
|
||||
DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
|
||||
|
||||
{ size_t maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
|
||||
if (ERR_isError(maxBits)) continue;
|
||||
|
||||
if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
|
||||
|
||||
hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
|
||||
}
|
||||
|
||||
if (ERR_isError(hSize)) continue;
|
||||
|
||||
newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
|
||||
|
||||
if (newSize > optSize + 1) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (newSize < optSize) {
|
||||
optSize = newSize;
|
||||
optLog = optLogGuess;
|
||||
}
|
||||
}
|
||||
assert(optLog <= HUF_TABLELOG_MAX);
|
||||
return optLog;
|
||||
}
|
||||
}
|
||||
|
||||
/* HUF_compress_internal() :
|
||||
* `workSpace_align4` must be aligned on 4-bytes boundaries,
|
||||
* and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */
|
||||
@@ -1181,14 +1336,14 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
HUF_nbStreams_e nbStreams,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
|
||||
const int bmi2, unsigned suspectUncompressible)
|
||||
HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags)
|
||||
{
|
||||
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
|
||||
DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize);
|
||||
HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE);
|
||||
|
||||
/* checks & inits */
|
||||
@@ -1202,16 +1357,17 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
||||
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
|
||||
|
||||
/* Heuristic : If old table is valid, use it for small inputs */
|
||||
if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
|
||||
if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
nbStreams, oldHufTable, bmi2);
|
||||
nbStreams, oldHufTable, flags);
|
||||
}
|
||||
|
||||
/* If uncompressible data is suspected, do a smaller sampling first */
|
||||
DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2);
|
||||
if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
|
||||
if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
|
||||
size_t largestTotal = 0;
|
||||
DEBUGLOG(5, "input suspected incompressible : sampling to check");
|
||||
{ unsigned maxSymbolValueBegin = maxSymbolValue;
|
||||
CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) );
|
||||
largestTotal += largestBegin;
|
||||
@@ -1228,6 +1384,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
||||
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
||||
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
|
||||
}
|
||||
DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1));
|
||||
|
||||
/* Check validity of previous table */
|
||||
if ( repeat
|
||||
@@ -1236,25 +1393,20 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
||||
*repeat = HUF_repeat_none;
|
||||
}
|
||||
/* Heuristic : use existing table for small inputs */
|
||||
if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
|
||||
if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
nbStreams, oldHufTable, bmi2);
|
||||
nbStreams, oldHufTable, flags);
|
||||
}
|
||||
|
||||
/* Build Huffman Tree */
|
||||
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
||||
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags);
|
||||
{ size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
|
||||
maxSymbolValue, huffLog,
|
||||
&table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
|
||||
CHECK_F(maxBits);
|
||||
huffLog = (U32)maxBits;
|
||||
}
|
||||
/* Zero unused symbols in CTable, so we can check it for validity */
|
||||
{
|
||||
size_t const ctableSize = HUF_CTABLE_SIZE_ST(maxSymbolValue);
|
||||
size_t const unusedSize = sizeof(table->CTable) - ctableSize * sizeof(HUF_CElt);
|
||||
ZSTD_memset(table->CTable + ctableSize, 0, unusedSize);
|
||||
DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1));
|
||||
}
|
||||
|
||||
/* Write table description header */
|
||||
@@ -1267,7 +1419,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
||||
if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
nbStreams, oldHufTable, bmi2);
|
||||
nbStreams, oldHufTable, flags);
|
||||
} }
|
||||
|
||||
/* Use the new huffman table */
|
||||
@@ -1279,95 +1431,37 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
||||
}
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
nbStreams, table->CTable, bmi2);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, HUF_singleStream,
|
||||
workSpace, wkspSize,
|
||||
NULL, NULL, 0, 0 /*bmi2*/, 0);
|
||||
nbStreams, table->CTable, flags);
|
||||
}
|
||||
|
||||
size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
|
||||
int bmi2, unsigned suspectUncompressible)
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
|
||||
{
|
||||
DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, HUF_singleStream,
|
||||
workSpace, wkspSize, hufTable,
|
||||
repeat, preferRepeat, bmi2, suspectUncompressible);
|
||||
}
|
||||
|
||||
/* HUF_compress4X_repeat():
|
||||
* compress input using 4 streams.
|
||||
* provide workspace to generate compression tables */
|
||||
size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, HUF_fourStreams,
|
||||
workSpace, wkspSize,
|
||||
NULL, NULL, 0, 0 /*bmi2*/, 0);
|
||||
repeat, flags);
|
||||
}
|
||||
|
||||
/* HUF_compress4X_repeat():
|
||||
* compress input using 4 streams.
|
||||
* consider skipping quickly
|
||||
* re-use an existing huffman compression table */
|
||||
* reuse an existing huffman compression table */
|
||||
size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible)
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
|
||||
{
|
||||
DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, HUF_fourStreams,
|
||||
workSpace, wkspSize,
|
||||
hufTable, repeat, preferRepeat, bmi2, suspectUncompressible);
|
||||
hufTable, repeat, flags);
|
||||
}
|
||||
|
||||
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
||||
/** HUF_buildCTable() :
|
||||
* @return : maxNbBits
|
||||
* Note : count is used before tree is written, so they can safely overlap
|
||||
*/
|
||||
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits)
|
||||
{
|
||||
HUF_buildCTable_wksp_tables workspace;
|
||||
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace));
|
||||
}
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
U64 workSpace[HUF_WORKSPACE_SIZE_U64];
|
||||
return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
||||
}
|
||||
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
U64 workSpace[HUF_WORKSPACE_SIZE_U64];
|
||||
return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
||||
}
|
||||
|
||||
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
1049
vendor/github.com/DataDog/zstd/huf_decompress.c
generated
vendored
1049
vendor/github.com/DataDog/zstd/huf_decompress.c
generated
vendored
File diff suppressed because it is too large
Load Diff
139
vendor/github.com/DataDog/zstd/huf_decompress_amd64.S
generated
vendored
139
vendor/github.com/DataDog/zstd/huf_decompress_amd64.S
generated
vendored
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -10,34 +10,53 @@
|
||||
|
||||
#include "portability_macros.h"
|
||||
|
||||
#if defined(__ELF__) && defined(__GNUC__)
|
||||
/* Stack marking
|
||||
* ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart
|
||||
*/
|
||||
#if defined(__ELF__) && defined(__GNUC__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
|
||||
#if defined(__aarch64__)
|
||||
/* Mark that this assembly supports BTI & PAC, because it is empty for aarch64.
|
||||
* See: https://github.com/facebook/zstd/issues/3841
|
||||
* See: https://gcc.godbolt.org/z/sqr5T4ffK
|
||||
* See: https://lore.kernel.org/linux-arm-kernel/20200429211641.9279-8-broonie@kernel.org/
|
||||
* See: https://reviews.llvm.org/D62609
|
||||
*/
|
||||
.pushsection .note.gnu.property, "a"
|
||||
.p2align 3
|
||||
.long 4 /* size of the name - "GNU\0" */
|
||||
.long 0x10 /* size of descriptor */
|
||||
.long 0x5 /* NT_GNU_PROPERTY_TYPE_0 */
|
||||
.asciz "GNU"
|
||||
.long 0xc0000000 /* pr_type - GNU_PROPERTY_AARCH64_FEATURE_1_AND */
|
||||
.long 4 /* pr_datasz - 4 bytes */
|
||||
.long 3 /* pr_data - GNU_PROPERTY_AARCH64_FEATURE_1_BTI | GNU_PROPERTY_AARCH64_FEATURE_1_PAC */
|
||||
.p2align 3 /* pr_padding - bring everything to 8 byte alignment */
|
||||
.popsection
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
|
||||
/* Calling convention:
|
||||
*
|
||||
* %rdi contains the first argument: HUF_DecompressAsmArgs*.
|
||||
* %rdi (or %rcx on Windows) contains the first argument: HUF_DecompressAsmArgs*.
|
||||
* %rbp isn't maintained (no frame pointer).
|
||||
* %rsp contains the stack pointer that grows down.
|
||||
* No red-zone is assumed, only addresses >= %rsp are used.
|
||||
* All register contents are preserved.
|
||||
*
|
||||
* TODO: Support Windows calling convention.
|
||||
*/
|
||||
|
||||
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
|
||||
.global HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
|
||||
.global HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
|
||||
.global _HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
|
||||
.global _HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
|
||||
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop)
|
||||
ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
|
||||
.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop
|
||||
.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop
|
||||
.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop
|
||||
.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop
|
||||
.text
|
||||
|
||||
/* Sets up register mappings for clarity.
|
||||
@@ -95,8 +114,9 @@ ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
|
||||
/* Define both _HUF_* & HUF_* symbols because MacOS
|
||||
* C symbols are prefixed with '_' & Linux symbols aren't.
|
||||
*/
|
||||
_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
_HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
|
||||
HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
|
||||
ZSTD_CET_ENDBRANCH
|
||||
/* Save all registers - even if they are callee saved for simplicity. */
|
||||
push %rax
|
||||
push %rbx
|
||||
@@ -115,7 +135,11 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
push %r15
|
||||
|
||||
/* Read HUF_DecompressAsmArgs* args from %rax */
|
||||
#if defined(_WIN32)
|
||||
movq %rcx, %rax
|
||||
#else
|
||||
movq %rdi, %rax
|
||||
#endif
|
||||
movq 0(%rax), %ip0
|
||||
movq 8(%rax), %ip1
|
||||
movq 16(%rax), %ip2
|
||||
@@ -130,7 +154,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
movq 88(%rax), %bits3
|
||||
movq 96(%rax), %dtable
|
||||
push %rax /* argument */
|
||||
push 104(%rax) /* ilimit */
|
||||
push 104(%rax) /* ilowest */
|
||||
push 112(%rax) /* oend */
|
||||
push %olimit /* olimit space */
|
||||
|
||||
@@ -155,11 +179,11 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
shrq $2, %r15
|
||||
|
||||
movq %ip0, %rax /* rax = ip0 */
|
||||
movq 40(%rsp), %rdx /* rdx = ilimit */
|
||||
subq %rdx, %rax /* rax = ip0 - ilimit */
|
||||
movq %rax, %rbx /* rbx = ip0 - ilimit */
|
||||
movq 40(%rsp), %rdx /* rdx = ilowest */
|
||||
subq %rdx, %rax /* rax = ip0 - ilowest */
|
||||
movq %rax, %rbx /* rbx = ip0 - ilowest */
|
||||
|
||||
/* rdx = (ip0 - ilimit) / 7 */
|
||||
/* rdx = (ip0 - ilowest) / 7 */
|
||||
movabsq $2635249153387078803, %rdx
|
||||
mulq %rdx
|
||||
subq %rdx, %rbx
|
||||
@@ -182,9 +206,8 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
|
||||
/* If (op3 + 20 > olimit) */
|
||||
movq %op3, %rax /* rax = op3 */
|
||||
addq $20, %rax /* rax = op3 + 20 */
|
||||
cmpq %rax, %olimit /* op3 + 20 > olimit */
|
||||
jb .L_4X1_exit
|
||||
cmpq %rax, %olimit /* op3 == olimit */
|
||||
je .L_4X1_exit
|
||||
|
||||
/* If (ip1 < ip0) go to exit */
|
||||
cmpq %ip0, %ip1
|
||||
@@ -315,7 +338,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
/* Restore stack (oend & olimit) */
|
||||
pop %rax /* olimit */
|
||||
pop %rax /* oend */
|
||||
pop %rax /* ilimit */
|
||||
pop %rax /* ilowest */
|
||||
pop %rax /* arg */
|
||||
|
||||
/* Save ip / op / bits */
|
||||
@@ -350,8 +373,9 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
|
||||
pop %rax
|
||||
ret
|
||||
|
||||
_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
_HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
|
||||
HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
|
||||
ZSTD_CET_ENDBRANCH
|
||||
/* Save all registers - even if they are callee saved for simplicity. */
|
||||
push %rax
|
||||
push %rbx
|
||||
@@ -369,7 +393,12 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
push %r14
|
||||
push %r15
|
||||
|
||||
/* Read HUF_DecompressAsmArgs* args from %rax */
|
||||
#if defined(_WIN32)
|
||||
movq %rcx, %rax
|
||||
#else
|
||||
movq %rdi, %rax
|
||||
#endif
|
||||
movq 0(%rax), %ip0
|
||||
movq 8(%rax), %ip1
|
||||
movq 16(%rax), %ip2
|
||||
@@ -385,7 +414,7 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
movq 96(%rax), %dtable
|
||||
push %rax /* argument */
|
||||
push %rax /* olimit */
|
||||
push 104(%rax) /* ilimit */
|
||||
push 104(%rax) /* ilowest */
|
||||
|
||||
movq 112(%rax), %rax
|
||||
push %rax /* oend3 */
|
||||
@@ -412,9 +441,9 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
|
||||
/* We can consume up to 7 input bytes each iteration. */
|
||||
movq %ip0, %rax /* rax = ip0 */
|
||||
movq 40(%rsp), %rdx /* rdx = ilimit */
|
||||
subq %rdx, %rax /* rax = ip0 - ilimit */
|
||||
movq %rax, %r15 /* r15 = ip0 - ilimit */
|
||||
movq 40(%rsp), %rdx /* rdx = ilowest */
|
||||
subq %rdx, %rax /* rax = ip0 - ilowest */
|
||||
movq %rax, %r15 /* r15 = ip0 - ilowest */
|
||||
|
||||
/* rdx = rax / 7 */
|
||||
movabsq $2635249153387078803, %rdx
|
||||
@@ -424,44 +453,33 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
addq %r15, %rdx
|
||||
shrq $2, %rdx
|
||||
|
||||
/* r15 = (ip0 - ilimit) / 7 */
|
||||
/* r15 = (ip0 - ilowest) / 7 */
|
||||
movq %rdx, %r15
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 8(%rsp), %rax /* rax = oend0 */
|
||||
subq %op0, %rax /* rax = oend0 - op0 */
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
/* r15 = min(r15, min(oend0 - op0, oend1 - op1, oend2 - op2, oend3 - op3) / 10) */
|
||||
movq 8(%rsp), %rax /* rax = oend0 */
|
||||
subq %op0, %rax /* rax = oend0 - op0 */
|
||||
movq 16(%rsp), %rdx /* rdx = oend1 */
|
||||
subq %op1, %rdx /* rdx = oend1 - op1 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
cmpq %rax, %rdx
|
||||
cmova %rax, %rdx /* rdx = min(%rdx, %rax) */
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 16(%rsp), %rax /* rax = oend1 */
|
||||
subq %op1, %rax /* rax = oend1 - op1 */
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 24(%rsp), %rax /* rax = oend2 */
|
||||
subq %op2, %rax /* rax = oend2 - op2 */
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
cmova %rdx, %r15
|
||||
cmpq %rax, %rdx
|
||||
cmova %rax, %rdx /* rdx = min(%rdx, %rax) */
|
||||
|
||||
movabsq $-3689348814741910323, %rdx
|
||||
movq 32(%rsp), %rax /* rax = oend3 */
|
||||
subq %op3, %rax /* rax = oend3 - op3 */
|
||||
|
||||
cmpq %rax, %rdx
|
||||
cmova %rax, %rdx /* rdx = min(%rdx, %rax) */
|
||||
|
||||
movabsq $-3689348814741910323, %rax
|
||||
mulq %rdx
|
||||
shrq $3, %rdx /* rdx = rax / 10 */
|
||||
shrq $3, %rdx /* rdx = rdx / 10 */
|
||||
|
||||
/* r15 = min(%rdx, %r15) */
|
||||
cmpq %rdx, %r15
|
||||
@@ -476,9 +494,8 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
|
||||
/* If (op3 + 10 > olimit) */
|
||||
movq %op3, %rax /* rax = op3 */
|
||||
addq $10, %rax /* rax = op3 + 10 */
|
||||
cmpq %rax, %olimit /* op3 + 10 > olimit */
|
||||
jb .L_4X2_exit
|
||||
cmpq %rax, %olimit /* op3 == olimit */
|
||||
je .L_4X2_exit
|
||||
|
||||
/* If (ip1 < ip0) go to exit */
|
||||
cmpq %ip0, %ip1
|
||||
@@ -546,7 +563,7 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
|
||||
pop %rax /* oend1 */
|
||||
pop %rax /* oend2 */
|
||||
pop %rax /* oend3 */
|
||||
pop %rax /* ilimit */
|
||||
pop %rax /* ilowest */
|
||||
pop %rax /* olimit */
|
||||
pop %rax /* arg */
|
||||
|
||||
|
||||
112
vendor/github.com/DataDog/zstd/mem.h
generated
vendored
112
vendor/github.com/DataDog/zstd/mem.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,10 +12,6 @@
|
||||
#ifndef MEM_H_MODULE
|
||||
#define MEM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
@@ -31,15 +27,8 @@ extern "C" {
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# include <stdlib.h> /* _byteswap_ulong */
|
||||
# include <intrin.h> /* _byteswap_* */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
# define MEM_STATIC static __inline __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define MEM_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define MEM_STATIC static __inline
|
||||
#else
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#elif defined(__ICCARM__)
|
||||
# include <intrinsics.h>
|
||||
#endif
|
||||
|
||||
/*-**************************************************************
|
||||
@@ -84,7 +73,6 @@ extern "C" {
|
||||
typedef signed long long S64;
|
||||
#endif
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Memory I/O API
|
||||
*****************************************************************/
|
||||
@@ -134,21 +122,15 @@ MEM_STATIC size_t MEM_swapST(size_t in);
|
||||
/*-**************************************************************
|
||||
* Memory I/O Implementation
|
||||
*****************************************************************/
|
||||
/* MEM_FORCE_MEMORY_ACCESS :
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory:
|
||||
* Method 0 : always use `memcpy()`. Safe and portable.
|
||||
* Method 1 : Use compiler extension to set unaligned access.
|
||||
* Method 2 : direct access. This method is portable but violate C standard.
|
||||
* It can generate buggy code on targets depending on alignment.
|
||||
* In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
|
||||
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
* Default : method 1 if supported, else method 0
|
||||
*/
|
||||
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
|
||||
# ifdef __GNUC__
|
||||
# define MEM_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
@@ -166,10 +148,12 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
||||
return 1;
|
||||
#elif defined(__clang__) && __BIG_ENDIAN__
|
||||
return 0;
|
||||
#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
|
||||
#elif defined(_MSC_VER) && (_M_X64 || _M_IX86)
|
||||
return 1;
|
||||
#elif defined(__DMC__) && defined(_M_IX86)
|
||||
return 1;
|
||||
#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__
|
||||
return 1;
|
||||
#else
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
|
||||
return one.c[0];
|
||||
@@ -191,30 +175,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
||||
|
||||
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
|
||||
__pragma( pack(push, 1) )
|
||||
typedef struct { U16 v; } unalign16;
|
||||
typedef struct { U32 v; } unalign32;
|
||||
typedef struct { U64 v; } unalign64;
|
||||
typedef struct { size_t v; } unalignArch;
|
||||
__pragma( pack(pop) )
|
||||
#else
|
||||
typedef struct { U16 v; } __attribute__((packed)) unalign16;
|
||||
typedef struct { U32 v; } __attribute__((packed)) unalign32;
|
||||
typedef struct { U64 v; } __attribute__((packed)) unalign64;
|
||||
typedef struct { size_t v; } __attribute__((packed)) unalignArch;
|
||||
#endif
|
||||
typedef __attribute__((aligned(1))) U16 unalign16;
|
||||
typedef __attribute__((aligned(1))) U32 unalign32;
|
||||
typedef __attribute__((aligned(1))) U64 unalign64;
|
||||
typedef __attribute__((aligned(1))) size_t unalignArch;
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
|
||||
MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; }
|
||||
MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; }
|
||||
|
||||
#else
|
||||
|
||||
@@ -258,6 +231,14 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
||||
|
||||
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
||||
|
||||
MEM_STATIC U32 MEM_swap32_fallback(U32 in)
|
||||
{
|
||||
return ((in << 24) & 0xff000000 ) |
|
||||
((in << 8) & 0x00ff0000 ) |
|
||||
((in >> 8) & 0x0000ff00 ) |
|
||||
((in >> 24) & 0x000000ff );
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_swap32(U32 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
@@ -265,14 +246,25 @@ MEM_STATIC U32 MEM_swap32(U32 in)
|
||||
#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
|
||||
|| (defined(__clang__) && __has_builtin(__builtin_bswap32))
|
||||
return __builtin_bswap32(in);
|
||||
#elif defined(__ICCARM__)
|
||||
return __REV(in);
|
||||
#else
|
||||
return ((in << 24) & 0xff000000 ) |
|
||||
((in << 8) & 0x00ff0000 ) |
|
||||
((in >> 8) & 0x0000ff00 ) |
|
||||
((in >> 24) & 0x000000ff );
|
||||
return MEM_swap32_fallback(in);
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_swap64_fallback(U64 in)
|
||||
{
|
||||
return ((in << 56) & 0xff00000000000000ULL) |
|
||||
((in << 40) & 0x00ff000000000000ULL) |
|
||||
((in << 24) & 0x0000ff0000000000ULL) |
|
||||
((in << 8) & 0x000000ff00000000ULL) |
|
||||
((in >> 8) & 0x00000000ff000000ULL) |
|
||||
((in >> 24) & 0x0000000000ff0000ULL) |
|
||||
((in >> 40) & 0x000000000000ff00ULL) |
|
||||
((in >> 56) & 0x00000000000000ffULL);
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_swap64(U64 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
@@ -281,14 +273,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
|
||||
|| (defined(__clang__) && __has_builtin(__builtin_bswap64))
|
||||
return __builtin_bswap64(in);
|
||||
#else
|
||||
return ((in << 56) & 0xff00000000000000ULL) |
|
||||
((in << 40) & 0x00ff000000000000ULL) |
|
||||
((in << 24) & 0x0000ff0000000000ULL) |
|
||||
((in << 8) & 0x000000ff00000000ULL) |
|
||||
((in >> 8) & 0x00000000ff000000ULL) |
|
||||
((in >> 24) & 0x0000000000ff0000ULL) |
|
||||
((in >> 40) & 0x000000000000ff00ULL) |
|
||||
((in >> 56) & 0x00000000000000ffULL);
|
||||
return MEM_swap64_fallback(in);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -435,11 +420,6 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
|
||||
/* code only tested on 32 and 64 bits systems */
|
||||
MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MEM_H_MODULE */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
38
vendor/github.com/DataDog/zstd/pool.c
generated
vendored
38
vendor/github.com/DataDog/zstd/pool.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -11,9 +11,9 @@
|
||||
|
||||
|
||||
/* ====== Dependencies ======= */
|
||||
#include "allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */
|
||||
#include "zstd_deps.h" /* size_t */
|
||||
#include "debug.h" /* assert */
|
||||
#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */
|
||||
#include "pool.h"
|
||||
|
||||
/* ====== Compiler specifics ====== */
|
||||
@@ -97,9 +97,7 @@ static void* POOL_thread(void* opaque) {
|
||||
/* If the intended queue size was 0, signal after finishing job */
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
ctx->numThreadsBusy--;
|
||||
if (ctx->queueSize == 1) {
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
}
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
}
|
||||
} /* for (;;) */
|
||||
@@ -129,7 +127,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
* empty and full queues.
|
||||
*/
|
||||
ctx->queueSize = queueSize + 1;
|
||||
ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
||||
ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
||||
ctx->queueHead = 0;
|
||||
ctx->queueTail = 0;
|
||||
ctx->numThreadsBusy = 0;
|
||||
@@ -143,7 +141,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
}
|
||||
ctx->shutdown = 0;
|
||||
/* Allocate space for the thread handles */
|
||||
ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
||||
ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
||||
ctx->threadCapacity = 0;
|
||||
ctx->customMem = customMem;
|
||||
/* Check for errors */
|
||||
@@ -176,7 +174,7 @@ static void POOL_join(POOL_ctx* ctx) {
|
||||
/* Join all of the threads */
|
||||
{ size_t i;
|
||||
for (i = 0; i < ctx->threadCapacity; ++i) {
|
||||
ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
|
||||
ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */
|
||||
} }
|
||||
}
|
||||
|
||||
@@ -191,6 +189,17 @@ void POOL_free(POOL_ctx *ctx) {
|
||||
ZSTD_customFree(ctx, ctx->customMem);
|
||||
}
|
||||
|
||||
/*! POOL_joinJobs() :
|
||||
* Waits for all queued jobs to finish executing.
|
||||
*/
|
||||
void POOL_joinJobs(POOL_ctx* ctx) {
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) {
|
||||
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
}
|
||||
|
||||
void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
|
||||
POOL_free (pool);
|
||||
}
|
||||
@@ -212,10 +221,10 @@ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
|
||||
return 0;
|
||||
}
|
||||
/* numThreads > threadCapacity */
|
||||
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
||||
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
||||
if (!threadPool) return 1;
|
||||
/* replace existing thread pool */
|
||||
ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
||||
ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(ZSTD_pthread_t));
|
||||
ZSTD_customFree(ctx->threads, ctx->customMem);
|
||||
ctx->threads = threadPool;
|
||||
/* Initialize additional threads */
|
||||
@@ -263,7 +272,9 @@ static int isQueueFull(POOL_ctx const* ctx) {
|
||||
static void
|
||||
POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
||||
{
|
||||
POOL_job const job = {function, opaque};
|
||||
POOL_job job;
|
||||
job.function = function;
|
||||
job.opaque = opaque;
|
||||
assert(ctx != NULL);
|
||||
if (ctx->shutdown) return;
|
||||
|
||||
@@ -331,6 +342,11 @@ void POOL_free(POOL_ctx* ctx) {
|
||||
(void)ctx;
|
||||
}
|
||||
|
||||
void POOL_joinJobs(POOL_ctx* ctx){
|
||||
assert(!ctx || ctx == &g_poolCtx);
|
||||
(void)ctx;
|
||||
}
|
||||
|
||||
int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
|
||||
(void)ctx; (void)numThreads;
|
||||
return 0;
|
||||
|
||||
19
vendor/github.com/DataDog/zstd/pool.h
generated
vendored
19
vendor/github.com/DataDog/zstd/pool.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,10 +12,6 @@
|
||||
#ifndef POOL_H
|
||||
#define POOL_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
#include "zstd_deps.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
|
||||
@@ -39,10 +35,16 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
*/
|
||||
void POOL_free(POOL_ctx* ctx);
|
||||
|
||||
|
||||
/*! POOL_joinJobs() :
|
||||
* Waits for all queued jobs to finish executing.
|
||||
*/
|
||||
void POOL_joinJobs(POOL_ctx* ctx);
|
||||
|
||||
/*! POOL_resize() :
|
||||
* Expands or shrinks pool's number of threads.
|
||||
* This is more efficient than releasing + creating a new context,
|
||||
* since it tries to preserve and re-use existing threads.
|
||||
* since it tries to preserve and reuse existing threads.
|
||||
* `numThreads` must be at least 1.
|
||||
* @return : 0 when resize was successful,
|
||||
* !0 (typically 1) if there is an error.
|
||||
@@ -77,11 +79,6 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
*/
|
||||
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
56
vendor/github.com/DataDog/zstd/portability_macros.h
generated
vendored
56
vendor/github.com/DataDog/zstd/portability_macros.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -13,7 +13,7 @@
|
||||
#define ZSTD_PORTABILITY_MACROS_H
|
||||
|
||||
/**
|
||||
* This header file contains macro defintions to support portability.
|
||||
* This header file contains macro definitions to support portability.
|
||||
* This header is shared between C and ASM code, so it MUST only
|
||||
* contain macro definitions. It MUST not contain any C code.
|
||||
*
|
||||
@@ -69,30 +69,45 @@
|
||||
/* Mark the internal assembly functions as hidden */
|
||||
#ifdef __ELF__
|
||||
# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
|
||||
#elif defined(__APPLE__)
|
||||
# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func
|
||||
#else
|
||||
# define ZSTD_HIDE_ASM_FUNCTION(func)
|
||||
#endif
|
||||
|
||||
/* Compile time determination of BMI2 support */
|
||||
#ifndef STATIC_BMI2
|
||||
# if defined(__BMI2__)
|
||||
# define STATIC_BMI2 1
|
||||
# elif defined(_MSC_VER) && defined(__AVX2__)
|
||||
# define STATIC_BMI2 1 /* MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifndef STATIC_BMI2
|
||||
# define STATIC_BMI2 0
|
||||
#endif
|
||||
|
||||
/* Enable runtime BMI2 dispatch based on the CPU.
|
||||
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
||||
*/
|
||||
#ifndef DYNAMIC_BMI2
|
||||
#if ((defined(__clang__) && __has_attribute(__target__)) \
|
||||
# if ((defined(__clang__) && __has_attribute(__target__)) \
|
||||
|| (defined(__GNUC__) \
|
||||
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
|
||||
&& (defined(__x86_64__) || defined(_M_X64)) \
|
||||
&& (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)) \
|
||||
&& !defined(__BMI2__)
|
||||
# define DYNAMIC_BMI2 1
|
||||
#else
|
||||
# define DYNAMIC_BMI2 0
|
||||
#endif
|
||||
# define DYNAMIC_BMI2 1
|
||||
# else
|
||||
# define DYNAMIC_BMI2 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Only enable assembly for GNUC comptabile compilers,
|
||||
* Only enable assembly for GNU C compatible compilers,
|
||||
* because other platforms may not support GAS assembly syntax.
|
||||
*
|
||||
* Only enable assembly for Linux / MacOS, other platforms may
|
||||
* Only enable assembly for Linux / MacOS / Win32, other platforms may
|
||||
* work, but they haven't been tested. This could likely be
|
||||
* extended to BSD systems.
|
||||
*
|
||||
@@ -100,7 +115,7 @@
|
||||
* 100% of code to be instrumented to work.
|
||||
*/
|
||||
#if defined(__GNUC__)
|
||||
# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
|
||||
# if defined(__linux__) || defined(__linux) || defined(__APPLE__) || defined(_WIN32)
|
||||
# if ZSTD_MEMORY_SANITIZER
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
# elif ZSTD_DATAFLOW_SANITIZER
|
||||
@@ -135,6 +150,25 @@
|
||||
# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
* For x86 ELF targets, add .note.gnu.property section for Intel CET in
|
||||
* assembly sources when CET is enabled.
|
||||
*
|
||||
* Additionally, any function that may be called indirectly must begin
|
||||
* with ZSTD_CET_ENDBRANCH.
|
||||
*/
|
||||
#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
|
||||
&& defined(__has_include)
|
||||
# if __has_include(<cet.h>)
|
||||
# include <cet.h>
|
||||
# define ZSTD_CET_ENDBRANCH _CET_ENDBR
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_CET_ENDBRANCH
|
||||
# define ZSTD_CET_ENDBRANCH
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_PORTABILITY_MACROS_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
88
vendor/github.com/DataDog/zstd/threading.c
generated
vendored
88
vendor/github.com/DataDog/zstd/threading.c
generated
vendored
@@ -24,8 +24,7 @@ int g_ZSTD_threading_useless_symbol;
|
||||
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
|
||||
|
||||
/**
|
||||
* Windows minimalist Pthread Wrapper, based on :
|
||||
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
|
||||
* Windows minimalist Pthread Wrapper
|
||||
*/
|
||||
|
||||
|
||||
@@ -36,37 +35,94 @@ int g_ZSTD_threading_useless_symbol;
|
||||
|
||||
/* === Implementation === */
|
||||
|
||||
typedef struct {
|
||||
void* (*start_routine)(void*);
|
||||
void* arg;
|
||||
int initialized;
|
||||
ZSTD_pthread_cond_t initialized_cond;
|
||||
ZSTD_pthread_mutex_t initialized_mutex;
|
||||
} ZSTD_thread_params_t;
|
||||
|
||||
static unsigned __stdcall worker(void *arg)
|
||||
{
|
||||
ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
|
||||
thread->arg = thread->start_routine(thread->arg);
|
||||
void* (*start_routine)(void*);
|
||||
void* thread_arg;
|
||||
|
||||
/* Initialized thread_arg and start_routine and signal main thread that we don't need it
|
||||
* to wait any longer.
|
||||
*/
|
||||
{
|
||||
ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg;
|
||||
thread_arg = thread_param->arg;
|
||||
start_routine = thread_param->start_routine;
|
||||
|
||||
/* Signal main thread that we are running and do not depend on its memory anymore */
|
||||
ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex);
|
||||
thread_param->initialized = 1;
|
||||
ZSTD_pthread_cond_signal(&thread_param->initialized_cond);
|
||||
ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex);
|
||||
}
|
||||
|
||||
start_routine(thread_arg);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg)
|
||||
{
|
||||
ZSTD_thread_params_t thread_param;
|
||||
(void)unused;
|
||||
thread->arg = arg;
|
||||
thread->start_routine = start_routine;
|
||||
thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
|
||||
|
||||
if (!thread->handle)
|
||||
if (thread==NULL) return -1;
|
||||
*thread = NULL;
|
||||
|
||||
thread_param.start_routine = start_routine;
|
||||
thread_param.arg = arg;
|
||||
thread_param.initialized = 0;
|
||||
|
||||
/* Setup thread initialization synchronization */
|
||||
if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) {
|
||||
/* Should never happen on Windows */
|
||||
return -1;
|
||||
}
|
||||
if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) {
|
||||
/* Should never happen on Windows */
|
||||
ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Spawn thread */
|
||||
*thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL);
|
||||
if (*thread==NULL) {
|
||||
ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
|
||||
ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
|
||||
return errno;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Wait for thread to be initialized */
|
||||
ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex);
|
||||
while(!thread_param.initialized) {
|
||||
ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex);
|
||||
ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
|
||||
ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread)
|
||||
{
|
||||
DWORD result;
|
||||
|
||||
if (!thread.handle) return 0;
|
||||
if (!thread) return 0;
|
||||
|
||||
result = WaitForSingleObject(thread, INFINITE);
|
||||
CloseHandle(thread);
|
||||
|
||||
result = WaitForSingleObject(thread.handle, INFINITE);
|
||||
switch (result) {
|
||||
case WAIT_OBJECT_0:
|
||||
if (value_ptr) *value_ptr = thread.arg;
|
||||
return 0;
|
||||
case WAIT_ABANDONED:
|
||||
return EINVAL;
|
||||
@@ -84,6 +140,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
||||
|
||||
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
|
||||
{
|
||||
assert(mutex != NULL);
|
||||
*mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t));
|
||||
if (!*mutex)
|
||||
return 1;
|
||||
@@ -92,6 +149,7 @@ int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t con
|
||||
|
||||
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
|
||||
{
|
||||
assert(mutex != NULL);
|
||||
if (!*mutex)
|
||||
return 0;
|
||||
{
|
||||
@@ -103,6 +161,7 @@ int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
|
||||
|
||||
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
|
||||
{
|
||||
assert(cond != NULL);
|
||||
*cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t));
|
||||
if (!*cond)
|
||||
return 1;
|
||||
@@ -111,6 +170,7 @@ int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const*
|
||||
|
||||
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
|
||||
{
|
||||
assert(cond != NULL);
|
||||
if (!*cond)
|
||||
return 0;
|
||||
{
|
||||
|
||||
23
vendor/github.com/DataDog/zstd/threading.h
generated
vendored
23
vendor/github.com/DataDog/zstd/threading.h
generated
vendored
@@ -17,15 +17,10 @@
|
||||
|
||||
#include "debug.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
|
||||
|
||||
/**
|
||||
* Windows minimalist Pthread Wrapper, based on :
|
||||
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
|
||||
* Windows minimalist Pthread Wrapper
|
||||
*/
|
||||
#ifdef WINVER
|
||||
# undef WINVER
|
||||
@@ -63,22 +58,17 @@ extern "C" {
|
||||
#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
|
||||
|
||||
/* ZSTD_pthread_create() and ZSTD_pthread_join() */
|
||||
typedef struct {
|
||||
HANDLE handle;
|
||||
void* (*start_routine)(void*);
|
||||
void* arg;
|
||||
} ZSTD_pthread_t;
|
||||
typedef HANDLE ZSTD_pthread_t;
|
||||
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg);
|
||||
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread);
|
||||
|
||||
/**
|
||||
* add here more wrappers as required
|
||||
*/
|
||||
|
||||
|
||||
#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
|
||||
/* === POSIX Systems === */
|
||||
# include <pthread.h>
|
||||
@@ -100,7 +90,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
|
||||
|
||||
#define ZSTD_pthread_t pthread_t
|
||||
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
|
||||
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
|
||||
#define ZSTD_pthread_join(a) pthread_join((a),NULL)
|
||||
|
||||
#else /* DEBUGLEVEL >= 1 */
|
||||
|
||||
@@ -125,7 +115,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
|
||||
|
||||
#define ZSTD_pthread_t pthread_t
|
||||
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
|
||||
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
|
||||
#define ZSTD_pthread_join(a) pthread_join((a),NULL)
|
||||
|
||||
#endif
|
||||
|
||||
@@ -149,9 +139,6 @@ typedef int ZSTD_pthread_cond_t;
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* THREADING_H_938743 */
|
||||
|
||||
|
||||
16
vendor/github.com/DataDog/zstd/xxhash.c
generated
vendored
16
vendor/github.com/DataDog/zstd/xxhash.c
generated
vendored
@@ -1,26 +1,20 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* xxHash - Fast Hash algorithm
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - xxHash homepage: http://www.xxhash.com
|
||||
* - xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
* xxHash - Extremely Fast Hash algorithm
|
||||
* Copyright (c) Yann Collet - Meta Platforms, Inc
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
*/
|
||||
|
||||
/*
|
||||
* xxhash.c instantiates functions defined in xxhash.h
|
||||
*/
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
|
||||
#define XXH_IMPLEMENTATION /* access definitions */
|
||||
#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
|
||||
#define XXH_IMPLEMENTATION /* access definitions */
|
||||
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
3424
vendor/github.com/DataDog/zstd/xxhash.h
generated
vendored
3424
vendor/github.com/DataDog/zstd/xxhash.h
generated
vendored
File diff suppressed because it is too large
Load Diff
2
vendor/github.com/DataDog/zstd/zbuff.h
generated
vendored
2
vendor/github.com/DataDog/zstd/zbuff.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/zbuff_common.c
generated
vendored
2
vendor/github.com/DataDog/zstd/zbuff_common.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/zbuff_compress.c
generated
vendored
2
vendor/github.com/DataDog/zstd/zbuff_compress.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
4
vendor/github.com/DataDog/zstd/zbuff_decompress.c
generated
vendored
4
vendor/github.com/DataDog/zstd/zbuff_decompress.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -14,6 +14,8 @@
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* suppress warning on ZSTD_initDStream_usingDict */
|
||||
#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h"
|
||||
|
||||
|
||||
122
vendor/github.com/DataDog/zstd/zdict.c
generated
vendored
122
vendor/github.com/DataDog/zstd/zdict.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -45,7 +45,6 @@
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
# define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
|
||||
#include "mem.h" /* read */
|
||||
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
|
||||
@@ -55,6 +54,7 @@
|
||||
#include "zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
|
||||
#include "zdict.h"
|
||||
#include "divsufsort.h"
|
||||
#include "bits.h" /* ZSTD_NbCommonBytes */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@@ -75,9 +75,9 @@ static const U32 g_selectivity_default = 9;
|
||||
* Console display
|
||||
***************************************/
|
||||
#undef DISPLAY
|
||||
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
|
||||
#define DISPLAY(...) do { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } while (0)
|
||||
#undef DISPLAYLEVEL
|
||||
#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||
#define DISPLAYLEVEL(l, ...) do { if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } } while (0) /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||
|
||||
static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
|
||||
|
||||
@@ -131,85 +131,6 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
|
||||
/*-********************************************************
|
||||
* Dictionary training functions
|
||||
**********************************************************/
|
||||
static unsigned ZDICT_NbCommonBytes (size_t val)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward64(&r, (U64)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_ctzll((U64)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
|
||||
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, (U32)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_ctz((U32)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
|
||||
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
||||
# endif
|
||||
}
|
||||
} else { /* Big Endian CPU */
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse64(&r, val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_clzll(val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
|
||||
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
|
||||
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, (unsigned long)val);
|
||||
return (unsigned)(r >> 3);
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (unsigned)(__builtin_clz((U32)val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
/*! ZDICT_count() :
|
||||
Count the nb of common bytes between 2 pointers.
|
||||
Note : this function presumes end of buffer followed by noisy guard band.
|
||||
@@ -224,7 +145,7 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch)
|
||||
pMatch = (const char*)pMatch+sizeof(size_t);
|
||||
continue;
|
||||
}
|
||||
pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);
|
||||
pIn = (const char*)pIn+ZSTD_NbCommonBytes(diff);
|
||||
return (size_t)((const char*)pIn - pStart);
|
||||
}
|
||||
}
|
||||
@@ -452,7 +373,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
|
||||
elt = table[u];
|
||||
/* sort : improve rank */
|
||||
while ((u>1) && (table[u-1].savings < elt.savings))
|
||||
table[u] = table[u-1], u--;
|
||||
table[u] = table[u-1], u--;
|
||||
table[u] = elt;
|
||||
return u;
|
||||
} }
|
||||
@@ -557,10 +478,16 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
|
||||
clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
|
||||
|
||||
# undef DISPLAYUPDATE
|
||||
# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
|
||||
if (ZDICT_clockSpan(displayClock) > refreshRate) \
|
||||
{ displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (notificationLevel>=4) fflush(stderr); } }
|
||||
# define DISPLAYUPDATE(l, ...) \
|
||||
do { \
|
||||
if (notificationLevel>=l) { \
|
||||
if (ZDICT_clockSpan(displayClock) > refreshRate) { \
|
||||
displayClock = clock(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} \
|
||||
if (notificationLevel>=4) fflush(stderr); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* init */
|
||||
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
|
||||
@@ -603,7 +530,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
|
||||
if (solution.length==0) { cursor++; continue; }
|
||||
ZDICT_insertDictItem(dictList, dictListSize, solution, buffer);
|
||||
cursor += solution.length;
|
||||
DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100);
|
||||
DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0);
|
||||
} }
|
||||
|
||||
_cleanup:
|
||||
@@ -646,15 +573,15 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
|
||||
size_t cSize;
|
||||
|
||||
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
|
||||
{ size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict);
|
||||
{ size_t const errorCode = ZSTD_compressBegin_usingCDict_deprecated(esr.zc, esr.dict);
|
||||
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; }
|
||||
|
||||
}
|
||||
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
|
||||
cSize = ZSTD_compressBlock_deprecated(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
|
||||
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
|
||||
|
||||
if (cSize) { /* if == 0; block is not compressible */
|
||||
const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
|
||||
const SeqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
|
||||
|
||||
/* literals stats */
|
||||
{ const BYTE* bytePtr;
|
||||
@@ -682,7 +609,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
|
||||
}
|
||||
|
||||
if (nbSeq >= 2) { /* rep offsets */
|
||||
const seqDef* const seq = seqStorePtr->sequencesStart;
|
||||
const SeqDef* const seq = seqStorePtr->sequencesStart;
|
||||
U32 offset1 = seq[0].offBase - ZSTD_REP_NUM;
|
||||
U32 offset2 = seq[1].offBase - ZSTD_REP_NUM;
|
||||
if (offset1 >= MAXREPOFFSET) offset1 = 0;
|
||||
@@ -755,6 +682,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
|
||||
size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles);
|
||||
BYTE* dstPtr = (BYTE*)dstBuffer;
|
||||
U32 wksp[HUF_CTABLE_WORKSPACE_SIZE_U32];
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(4, "ZDICT_analyzeEntropy");
|
||||
@@ -795,7 +723,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
} }
|
||||
|
||||
/* analyze, build stats, starting with literals */
|
||||
{ size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
|
||||
{ size_t maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
|
||||
if (HUF_isError(maxNbBits)) {
|
||||
eSize = maxNbBits;
|
||||
DISPLAYLEVEL(1, " HUF_buildCTable error \n");
|
||||
@@ -804,7 +732,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
if (maxNbBits==8) { /* not compressible : will fail on HUF_writeCTable() */
|
||||
DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n");
|
||||
ZDICT_flatLit(countLit); /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */
|
||||
maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
|
||||
maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
|
||||
assert(maxNbBits==9);
|
||||
}
|
||||
huffLog = (U32)maxNbBits;
|
||||
@@ -845,7 +773,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
llLog = (U32)errorCode;
|
||||
|
||||
/* write result to buffer */
|
||||
{ size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
|
||||
{ size_t const hhSize = HUF_writeCTable_wksp(dstPtr, maxDstSize, hufTable, 255, huffLog, wksp, sizeof(wksp));
|
||||
if (HUF_isError(hhSize)) {
|
||||
eSize = hhSize;
|
||||
DISPLAYLEVEL(1, "HUF_writeCTable error \n");
|
||||
|
||||
101
vendor/github.com/DataDog/zstd/zdict.h
generated
vendored
101
vendor/github.com/DataDog/zstd/zdict.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -9,32 +9,43 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef DICTBUILDER_H_001
|
||||
#define DICTBUILDER_H_001
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
#ifndef ZSTD_ZDICT_H
|
||||
#define ZSTD_ZDICT_H
|
||||
|
||||
|
||||
/*====== Dependencies ======*/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ===== ZDICTLIB_API : control library symbols visibility ===== */
|
||||
#ifndef ZDICTLIB_VISIBILITY
|
||||
# if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
#ifndef ZDICTLIB_VISIBLE
|
||||
/* Backwards compatibility with old macro name */
|
||||
# ifdef ZDICTLIB_VISIBILITY
|
||||
# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
|
||||
# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
|
||||
# else
|
||||
# define ZDICTLIB_VISIBILITY
|
||||
# define ZDICTLIB_VISIBLE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifndef ZDICTLIB_HIDDEN
|
||||
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
|
||||
# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
|
||||
# else
|
||||
# define ZDICTLIB_HIDDEN
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
|
||||
# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
|
||||
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define ZDICTLIB_API ZDICTLIB_VISIBILITY
|
||||
# define ZDICTLIB_API ZDICTLIB_VISIBLE
|
||||
#endif
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -111,8 +122,8 @@ extern "C" {
|
||||
* The zstd CLI defaults to a 110KB dictionary. You likely don't need a
|
||||
* dictionary larger than that. But, most use cases can get away with a
|
||||
* smaller dictionary. The advanced dictionary builders can automatically
|
||||
* shrink the dictionary for you, and select a the smallest size that
|
||||
* doesn't hurt compression ratio too much. See the `shrinkDict` parameter.
|
||||
* shrink the dictionary for you, and select the smallest size that doesn't
|
||||
* hurt compression ratio too much. See the `shrinkDict` parameter.
|
||||
* A smaller dictionary can save memory, and potentially speed up
|
||||
* compression.
|
||||
*
|
||||
@@ -202,9 +213,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
|
||||
const size_t* samplesSizes, unsigned nbSamples);
|
||||
|
||||
typedef struct {
|
||||
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
|
||||
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value)
|
||||
int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */
|
||||
unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||
unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value)
|
||||
* NOTE: The zstd format reserves some dictionary IDs for future use.
|
||||
* You may use them in private settings, but be warned that they
|
||||
* may be used by zstd in a public dictionary registry in the future.
|
||||
@@ -238,7 +249,7 @@ typedef struct {
|
||||
* is presumed that the most profitable content is at the end of the dictionary,
|
||||
* since that is the cheapest to reference.
|
||||
*
|
||||
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
|
||||
* `maxDictSize` must be >= max(dictContentSize, ZDICT_DICTSIZE_MIN).
|
||||
*
|
||||
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
|
||||
* or an error code, which can be tested by ZDICT_isError().
|
||||
@@ -261,9 +272,29 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS
|
||||
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
|
||||
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_ZDICT_H */
|
||||
|
||||
#ifdef ZDICT_STATIC_LINKING_ONLY
|
||||
#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
|
||||
#define ZSTD_ZDICT_H_STATIC
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* This can be overridden externally to hide static symbols. */
|
||||
#ifndef ZDICTLIB_STATIC_API
|
||||
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
|
||||
# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
|
||||
# else
|
||||
# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
@@ -319,7 +350,7 @@ typedef struct {
|
||||
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
||||
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
||||
ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
|
||||
void *dictBuffer, size_t dictBufferCapacity,
|
||||
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_cover_params_t parameters);
|
||||
@@ -341,7 +372,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
||||
* See ZDICT_trainFromBuffer() for details on failure modes.
|
||||
* Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_cover_params_t* parameters);
|
||||
@@ -362,7 +393,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
||||
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
|
||||
ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
|
||||
size_t dictBufferCapacity, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_fastCover_params_t parameters);
|
||||
@@ -385,7 +416,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
|
||||
* See ZDICT_trainFromBuffer() for details on failure modes.
|
||||
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
||||
ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
||||
size_t dictBufferCapacity, const void* samplesBuffer,
|
||||
const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_fastCover_params_t* parameters);
|
||||
@@ -410,7 +441,7 @@ typedef struct {
|
||||
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
||||
* Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
||||
ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
|
||||
void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t parameters);
|
||||
@@ -422,34 +453,32 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
||||
or _CRT_SECURE_NO_WARNINGS in Visual.
|
||||
Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
|
||||
#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */
|
||||
# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
|
||||
#else
|
||||
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
|
||||
# define ZDICT_DEPRECATED(message) [[deprecated(message)]]
|
||||
# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
|
||||
# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
|
||||
# elif (ZDICT_GCC_VERSION >= 301)
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
|
||||
# define ZDICT_DEPRECATED(message) __attribute__((deprecated))
|
||||
# elif defined(_MSC_VER)
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
|
||||
# define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
|
||||
# else
|
||||
# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API
|
||||
# define ZDICT_DEPRECATED(message)
|
||||
# endif
|
||||
#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
|
||||
|
||||
ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
|
||||
ZDICTLIB_STATIC_API
|
||||
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
||||
|
||||
|
||||
#endif /* ZDICT_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* DICTBUILDER_H_001 */
|
||||
#endif /* ZSTD_ZDICT_H_STATIC */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
36
vendor/github.com/DataDog/zstd/zstd.go
generated
vendored
36
vendor/github.com/DataDog/zstd/zstd.go
generated
vendored
@@ -4,7 +4,7 @@ package zstd
|
||||
// support decoding of "legacy" zstd payloads from versions [0.4, 0.8], matching the
|
||||
// default configuration of the zstd command line tool:
|
||||
// https://github.com/facebook/zstd/blob/dev/programs/README.md
|
||||
#cgo CFLAGS: -DZSTD_LEGACY_SUPPORT=4
|
||||
#cgo CFLAGS: -DZSTD_LEGACY_SUPPORT=4 -DZSTD_MULTITHREAD=1
|
||||
|
||||
#include "zstd.h"
|
||||
*/
|
||||
@@ -36,7 +36,7 @@ const (
|
||||
// decompressed, err := zstd.Decompress(dst, src)
|
||||
decompressSizeBufferLimit = 1000 * 1000
|
||||
|
||||
zstdFrameHeaderSizeMax = 18 // From zstd.h. Since it's experimental API, hardcoding it
|
||||
zstdFrameHeaderSizeMin = 2 // From zstd.h. Since it's experimental API, hardcoding it
|
||||
)
|
||||
|
||||
// CompressBound returns the worst case size needed for a destination buffer,
|
||||
@@ -60,18 +60,21 @@ func cCompressBound(srcSize int) int {
|
||||
// decompressSizeHint tries to give a hint on how much of the output buffer size we should have
|
||||
// based on zstd frame descriptors. To prevent DOS from maliciously-created payloads, limit the size
|
||||
func decompressSizeHint(src []byte) int {
|
||||
// 1 MB or 10x input size
|
||||
upperBound := 10 * len(src)
|
||||
// 1 MB or 50x input size
|
||||
upperBound := 50 * len(src)
|
||||
if upperBound < decompressSizeBufferLimit {
|
||||
upperBound = decompressSizeBufferLimit
|
||||
}
|
||||
|
||||
hint := upperBound
|
||||
if len(src) >= zstdFrameHeaderSizeMax {
|
||||
if len(src) >= zstdFrameHeaderSizeMin {
|
||||
hint = int(C.ZSTD_getFrameContentSize(unsafe.Pointer(&src[0]), C.size_t(len(src))))
|
||||
if hint < 0 { // On error, just use upperBound
|
||||
hint = upperBound
|
||||
}
|
||||
if hint == 0 { // When compressing the empty slice, we need an output of at least 1 to pass down to the C lib
|
||||
hint = 1
|
||||
}
|
||||
}
|
||||
|
||||
// Take the minimum of both
|
||||
@@ -140,12 +143,7 @@ func Decompress(dst, src []byte) ([]byte, error) {
|
||||
dst = make([]byte, bound)
|
||||
}
|
||||
|
||||
written := int(C.ZSTD_decompress(
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src))))
|
||||
err := getError(written)
|
||||
written, err := DecompressInto(dst, src)
|
||||
if err == nil {
|
||||
return dst[:written], nil
|
||||
}
|
||||
@@ -158,3 +156,19 @@ func Decompress(dst, src []byte) ([]byte, error) {
|
||||
defer r.Close()
|
||||
return ioutil.ReadAll(r)
|
||||
}
|
||||
|
||||
// DecompressInto decompresses src into dst. Unlike Decompress, DecompressInto
|
||||
// requires that dst be sufficiently large to hold the decompressed payload.
|
||||
// DecompressInto may be used when the caller knows the size of the decompressed
|
||||
// payload before attempting decompression.
|
||||
//
|
||||
// It returns the number of bytes copied and an error if any is encountered. If
|
||||
// dst is too small, DecompressInto errors.
|
||||
func DecompressInto(dst, src []byte) (int, error) {
|
||||
written := int(C.ZSTD_decompress(
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src))))
|
||||
return written, getError(written)
|
||||
}
|
||||
|
||||
1155
vendor/github.com/DataDog/zstd/zstd.h
generated
vendored
1155
vendor/github.com/DataDog/zstd/zstd.h
generated
vendored
File diff suppressed because it is too large
Load Diff
6
vendor/github.com/DataDog/zstd/zstd_bulk.go
generated
vendored
6
vendor/github.com/DataDog/zstd/zstd_bulk.go
generated
vendored
@@ -112,12 +112,12 @@ func (p *BulkProcessor) Decompress(dst, src []byte) ([]byte, error) {
|
||||
|
||||
contentSize := decompressSizeHint(src)
|
||||
if cap(dst) >= contentSize {
|
||||
dst = dst[0:contentSize]
|
||||
dst = dst[0:cap(dst)]
|
||||
} else {
|
||||
dst = make([]byte, contentSize)
|
||||
}
|
||||
|
||||
if contentSize == 0 {
|
||||
if len(dst) == 0 {
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -125,7 +125,7 @@ func (p *BulkProcessor) Decompress(dst, src []byte) ([]byte, error) {
|
||||
cWritten := C.ZSTD_decompress_usingDDict(
|
||||
dctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(contentSize),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src)),
|
||||
p.dDict,
|
||||
|
||||
37
vendor/github.com/DataDog/zstd/zstd_common.c
generated
vendored
37
vendor/github.com/DataDog/zstd/zstd_common.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -15,7 +15,6 @@
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h"
|
||||
|
||||
@@ -49,38 +48,4 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
* provides error code string from enum */
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
|
||||
|
||||
|
||||
|
||||
/*=**************************************************************
|
||||
* Custom allocator
|
||||
****************************************************************/
|
||||
void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc)
|
||||
return customMem.customAlloc(customMem.opaque, size);
|
||||
return ZSTD_malloc(size);
|
||||
}
|
||||
|
||||
void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
|
||||
{
|
||||
if (customMem.customAlloc) {
|
||||
/* calloc implemented as malloc+memset;
|
||||
* not as efficient as calloc, but next best guess for custom malloc */
|
||||
void* const ptr = customMem.customAlloc(customMem.opaque, size);
|
||||
ZSTD_memset(ptr, 0, size);
|
||||
return ptr;
|
||||
}
|
||||
return ZSTD_calloc(1, size);
|
||||
}
|
||||
|
||||
void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
|
||||
{
|
||||
if (ptr!=NULL) {
|
||||
if (customMem.customFree)
|
||||
customMem.customFree(customMem.opaque, ptr);
|
||||
else
|
||||
ZSTD_free(ptr);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
3310
vendor/github.com/DataDog/zstd/zstd_compress.c
generated
vendored
3310
vendor/github.com/DataDog/zstd/zstd_compress.c
generated
vendored
File diff suppressed because it is too large
Load Diff
676
vendor/github.com/DataDog/zstd/zstd_compress_internal.h
generated
vendored
676
vendor/github.com/DataDog/zstd/zstd_compress_internal.h
generated
vendored
File diff suppressed because it is too large
Load Diff
156
vendor/github.com/DataDog/zstd/zstd_compress_literals.c
generated
vendored
156
vendor/github.com/DataDog/zstd/zstd_compress_literals.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -14,11 +14,36 @@
|
||||
***************************************/
|
||||
#include "zstd_compress_literals.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Debug Traces
|
||||
****************************************************************/
|
||||
#if DEBUGLEVEL >= 2
|
||||
|
||||
static size_t showHexa(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* const ip = (const BYTE*)src;
|
||||
size_t u;
|
||||
for (u=0; u<srcSize; u++) {
|
||||
RAWLOG(5, " %02X", ip[u]); (void)ip;
|
||||
}
|
||||
RAWLOG(5, " \n");
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Literals compression - special cases
|
||||
****************************************************************/
|
||||
size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
||||
|
||||
DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity);
|
||||
|
||||
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
|
||||
|
||||
switch(flSize)
|
||||
@@ -37,16 +62,30 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
|
||||
}
|
||||
|
||||
ZSTD_memcpy(ostart + flSize, src, srcSize);
|
||||
DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
|
||||
DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
|
||||
return srcSize + flSize;
|
||||
}
|
||||
|
||||
static int allBytesIdentical(const void* src, size_t srcSize)
|
||||
{
|
||||
assert(srcSize >= 1);
|
||||
assert(src != NULL);
|
||||
{ const BYTE b = ((const BYTE*)src)[0];
|
||||
size_t p;
|
||||
for (p=1; p<srcSize; p++) {
|
||||
if (((const BYTE*)src)[p] != b) return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
||||
|
||||
(void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
|
||||
assert(dstCapacity >= 4); (void)dstCapacity;
|
||||
assert(allBytesIdentical(src, srcSize));
|
||||
|
||||
switch(flSize)
|
||||
{
|
||||
@@ -64,28 +103,51 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
|
||||
}
|
||||
|
||||
ostart[flSize] = *(const BYTE*)src;
|
||||
DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
|
||||
DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
|
||||
return flSize+1;
|
||||
}
|
||||
|
||||
size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
ZSTD_hufCTables_t* nextHuf,
|
||||
ZSTD_strategy strategy, int disableLiteralCompression,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
||||
const int bmi2,
|
||||
unsigned suspectUncompressible)
|
||||
/* ZSTD_minLiteralsToCompress() :
|
||||
* returns minimal amount of literals
|
||||
* for literal compression to even be attempted.
|
||||
* Minimum is made tighter as compression strategy increases.
|
||||
*/
|
||||
static size_t
|
||||
ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
|
||||
{
|
||||
assert((int)strategy >= 0);
|
||||
assert((int)strategy <= 9);
|
||||
/* btultra2 : min 8 bytes;
|
||||
* then 2x larger for each successive compression strategy
|
||||
* max threshold 64 bytes */
|
||||
{ int const shift = MIN(9-(int)strategy, 3);
|
||||
size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
|
||||
DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
|
||||
return mintc;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_compressLiterals (
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
||||
const ZSTD_hufCTables_t* prevHuf,
|
||||
ZSTD_hufCTables_t* nextHuf,
|
||||
ZSTD_strategy strategy,
|
||||
int disableLiteralCompression,
|
||||
int suspectUncompressible,
|
||||
int bmi2)
|
||||
{
|
||||
size_t const minGain = ZSTD_minGain(srcSize, strategy);
|
||||
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
U32 singleStream = srcSize < 256;
|
||||
symbolEncodingType_e hType = set_compressed;
|
||||
SymbolEncodingType_e hType = set_compressed;
|
||||
size_t cLitSize;
|
||||
|
||||
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
|
||||
disableLiteralCompression, (U32)srcSize);
|
||||
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
|
||||
disableLiteralCompression, (U32)srcSize, dstCapacity);
|
||||
|
||||
DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize));
|
||||
|
||||
/* Prepare nextEntropy assuming reusing the existing table */
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
@@ -93,40 +155,51 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
if (disableLiteralCompression)
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
|
||||
/* small ? don't even attempt compression (speed opt) */
|
||||
# define COMPRESS_LITERALS_SIZE_MIN 63
|
||||
{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
|
||||
if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
/* if too small, don't even attempt compression (speed opt) */
|
||||
if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
|
||||
RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
|
||||
{ HUF_repeat repeat = prevHuf->repeatMode;
|
||||
int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
|
||||
int const flags = 0
|
||||
| (bmi2 ? HUF_flags_bmi2 : 0)
|
||||
| (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
|
||||
| (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
|
||||
| (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
|
||||
|
||||
typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
|
||||
huf_compress_f huf_compress;
|
||||
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
|
||||
cLitSize = singleStream ?
|
||||
HUF_compress1X_repeat(
|
||||
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
||||
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) :
|
||||
HUF_compress4X_repeat(
|
||||
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
||||
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible);
|
||||
huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
|
||||
cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize,
|
||||
src, srcSize,
|
||||
HUF_SYMBOLVALUE_MAX, LitHufLog,
|
||||
entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable,
|
||||
&repeat, flags);
|
||||
DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
|
||||
if (repeat != HUF_repeat_none) {
|
||||
/* reused the existing table */
|
||||
DEBUGLOG(5, "Reusing previous huffman table");
|
||||
DEBUGLOG(5, "reusing statistics from previous huffman block");
|
||||
hType = set_repeat;
|
||||
}
|
||||
}
|
||||
|
||||
if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
{ size_t const minGain = ZSTD_minGain(srcSize, strategy);
|
||||
if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
} }
|
||||
if (cLitSize==1) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
/* A return value of 1 signals that the alphabet consists of a single symbol.
|
||||
* However, in some rare circumstances, it could be the compressed size (a single byte).
|
||||
* For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
|
||||
* (it's also necessary to not generate statistics).
|
||||
* Therefore, in such a case, actively check that all bytes are identical. */
|
||||
if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
|
||||
} }
|
||||
|
||||
if (hType == set_compressed) {
|
||||
/* using a newly constructed table */
|
||||
@@ -137,16 +210,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
switch(lhSize)
|
||||
{
|
||||
case 3: /* 2 - 2 - 10 - 10 */
|
||||
{ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
|
||||
if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
{ U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
|
||||
MEM_writeLE24(ostart, lhc);
|
||||
break;
|
||||
}
|
||||
case 4: /* 2 - 2 - 14 - 14 */
|
||||
assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
{ U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
|
||||
MEM_writeLE32(ostart, lhc);
|
||||
break;
|
||||
}
|
||||
case 5: /* 2 - 2 - 18 - 18 */
|
||||
assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
{ U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
|
||||
MEM_writeLE32(ostart, lhc);
|
||||
ostart[4] = (BYTE)(cLitSize >> 10);
|
||||
|
||||
24
vendor/github.com/DataDog/zstd/zstd_compress_literals.h
generated
vendored
24
vendor/github.com/DataDog/zstd/zstd_compress_literals.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -17,17 +17,25 @@
|
||||
|
||||
size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/* ZSTD_compressRleLiteralsBlock() :
|
||||
* Conditions :
|
||||
* - All bytes in @src are identical
|
||||
* - dstCapacity >= 4 */
|
||||
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
|
||||
size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
||||
ZSTD_hufCTables_t* nextHuf,
|
||||
ZSTD_strategy strategy, int disableLiteralCompression,
|
||||
void* dst, size_t dstCapacity,
|
||||
/* ZSTD_compressLiterals():
|
||||
* @entropyWorkspace: must be aligned on 4-bytes boundaries
|
||||
* @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
|
||||
* @suspectUncompressible: sampling checks, to potentially skip huffman coding
|
||||
*/
|
||||
size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
||||
const int bmi2,
|
||||
unsigned suspectUncompressible);
|
||||
const ZSTD_hufCTables_t* prevHuf,
|
||||
ZSTD_hufCTables_t* nextHuf,
|
||||
ZSTD_strategy strategy, int disableLiteralCompression,
|
||||
int suspectUncompressible,
|
||||
int bmi2);
|
||||
|
||||
#endif /* ZSTD_COMPRESS_LITERALS_H */
|
||||
|
||||
|
||||
20
vendor/github.com/DataDog/zstd/zstd_compress_sequences.c
generated
vendored
20
vendor/github.com/DataDog/zstd/zstd_compress_sequences.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -59,7 +59,7 @@ static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
|
||||
{
|
||||
/* Heuristic: This should cover most blocks <= 16K and
|
||||
* start to fade out after 16K to about 32K depending on
|
||||
* comprssibility.
|
||||
* compressibility.
|
||||
*/
|
||||
return nbSeq >= 2048;
|
||||
}
|
||||
@@ -154,20 +154,20 @@ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
|
||||
return cost >> 8;
|
||||
}
|
||||
|
||||
symbolEncodingType_e
|
||||
SymbolEncodingType_e
|
||||
ZSTD_selectEncodingType(
|
||||
FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
|
||||
size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
|
||||
FSE_CTable const* prevCTable,
|
||||
short const* defaultNorm, U32 defaultNormLog,
|
||||
ZSTD_defaultPolicy_e const isDefaultAllowed,
|
||||
ZSTD_DefaultPolicy_e const isDefaultAllowed,
|
||||
ZSTD_strategy const strategy)
|
||||
{
|
||||
ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0);
|
||||
if (mostFrequent == nbSeq) {
|
||||
*repeatMode = FSE_repeat_none;
|
||||
if (isDefaultAllowed && nbSeq <= 2) {
|
||||
/* Prefer set_basic over set_rle when there are 2 or less symbols,
|
||||
/* Prefer set_basic over set_rle when there are 2 or fewer symbols,
|
||||
* since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol.
|
||||
* If basic encoding isn't possible, always choose RLE.
|
||||
*/
|
||||
@@ -242,7 +242,7 @@ typedef struct {
|
||||
|
||||
size_t
|
||||
ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
||||
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
|
||||
FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type,
|
||||
unsigned* count, U32 max,
|
||||
const BYTE* codeTable, size_t nbSeq,
|
||||
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
|
||||
@@ -294,7 +294,7 @@ ZSTD_encodeSequences_body(
|
||||
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
|
||||
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
|
||||
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
|
||||
seqDef const* sequences, size_t nbSeq, int longOffsets)
|
||||
SeqDef const* sequences, size_t nbSeq, int longOffsets)
|
||||
{
|
||||
BIT_CStream_t blockStream;
|
||||
FSE_CState_t stateMatchLength;
|
||||
@@ -388,7 +388,7 @@ ZSTD_encodeSequences_default(
|
||||
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
|
||||
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
|
||||
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
|
||||
seqDef const* sequences, size_t nbSeq, int longOffsets)
|
||||
SeqDef const* sequences, size_t nbSeq, int longOffsets)
|
||||
{
|
||||
return ZSTD_encodeSequences_body(dst, dstCapacity,
|
||||
CTable_MatchLength, mlCodeTable,
|
||||
@@ -406,7 +406,7 @@ ZSTD_encodeSequences_bmi2(
|
||||
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
|
||||
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
|
||||
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
|
||||
seqDef const* sequences, size_t nbSeq, int longOffsets)
|
||||
SeqDef const* sequences, size_t nbSeq, int longOffsets)
|
||||
{
|
||||
return ZSTD_encodeSequences_body(dst, dstCapacity,
|
||||
CTable_MatchLength, mlCodeTable,
|
||||
@@ -422,7 +422,7 @@ size_t ZSTD_encodeSequences(
|
||||
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
|
||||
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
|
||||
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
|
||||
seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
|
||||
SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity);
|
||||
#if DYNAMIC_BMI2
|
||||
|
||||
15
vendor/github.com/DataDog/zstd/zstd_compress_sequences.h
generated
vendored
15
vendor/github.com/DataDog/zstd/zstd_compress_sequences.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,26 +12,27 @@
|
||||
#ifndef ZSTD_COMPRESS_SEQUENCES_H
|
||||
#define ZSTD_COMPRESS_SEQUENCES_H
|
||||
|
||||
#include "zstd_compress_internal.h" /* SeqDef */
|
||||
#include "fse.h" /* FSE_repeat, FSE_CTable */
|
||||
#include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
|
||||
#include "zstd_internal.h" /* SymbolEncodingType_e, ZSTD_strategy */
|
||||
|
||||
typedef enum {
|
||||
ZSTD_defaultDisallowed = 0,
|
||||
ZSTD_defaultAllowed = 1
|
||||
} ZSTD_defaultPolicy_e;
|
||||
} ZSTD_DefaultPolicy_e;
|
||||
|
||||
symbolEncodingType_e
|
||||
SymbolEncodingType_e
|
||||
ZSTD_selectEncodingType(
|
||||
FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
|
||||
size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
|
||||
FSE_CTable const* prevCTable,
|
||||
short const* defaultNorm, U32 defaultNormLog,
|
||||
ZSTD_defaultPolicy_e const isDefaultAllowed,
|
||||
ZSTD_DefaultPolicy_e const isDefaultAllowed,
|
||||
ZSTD_strategy const strategy);
|
||||
|
||||
size_t
|
||||
ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
||||
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
|
||||
FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type,
|
||||
unsigned* count, U32 max,
|
||||
const BYTE* codeTable, size_t nbSeq,
|
||||
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
|
||||
@@ -43,7 +44,7 @@ size_t ZSTD_encodeSequences(
|
||||
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
|
||||
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
|
||||
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
|
||||
seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
|
||||
SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
|
||||
|
||||
size_t ZSTD_fseBitCost(
|
||||
FSE_CTable const* ctable,
|
||||
|
||||
389
vendor/github.com/DataDog/zstd/zstd_compress_superblock.c
generated
vendored
389
vendor/github.com/DataDog/zstd/zstd_compress_superblock.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -37,13 +37,14 @@
|
||||
* If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
|
||||
* and the following sub-blocks' literals sections will be Treeless_Literals_Block.
|
||||
* @return : compressed size of literals section of a sub-block
|
||||
* Or 0 if it unable to compress.
|
||||
* Or 0 if unable to compress.
|
||||
* Or error code */
|
||||
static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
||||
const ZSTD_hufCTablesMetadata_t* hufMetadata,
|
||||
const BYTE* literals, size_t litSize,
|
||||
void* dst, size_t dstSize,
|
||||
const int bmi2, int writeEntropy, int* entropyWritten)
|
||||
static size_t
|
||||
ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
||||
const ZSTD_hufCTablesMetadata_t* hufMetadata,
|
||||
const BYTE* literals, size_t litSize,
|
||||
void* dst, size_t dstSize,
|
||||
const int bmi2, int writeEntropy, int* entropyWritten)
|
||||
{
|
||||
size_t const header = writeEntropy ? 200 : 0;
|
||||
size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header));
|
||||
@@ -51,11 +52,9 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart + lhSize;
|
||||
U32 const singleStream = lhSize == 3;
|
||||
symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
|
||||
SymbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
|
||||
size_t cLitSize = 0;
|
||||
|
||||
(void)bmi2; /* TODO bmi2... */
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
|
||||
|
||||
*entropyWritten = 0;
|
||||
@@ -77,9 +76,9 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
|
||||
}
|
||||
|
||||
/* TODO bmi2 */
|
||||
{ const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable)
|
||||
: HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable);
|
||||
{ int const flags = bmi2 ? HUF_flags_bmi2 : 0;
|
||||
const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags)
|
||||
: HUF_compress4X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags);
|
||||
op += cSize;
|
||||
cLitSize += cSize;
|
||||
if (cSize == 0 || ERR_isError(cSize)) {
|
||||
@@ -104,7 +103,7 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
||||
switch(lhSize)
|
||||
{
|
||||
case 3: /* 2 - 2 - 10 - 10 */
|
||||
{ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14);
|
||||
{ U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14);
|
||||
MEM_writeLE24(ostart, lhc);
|
||||
break;
|
||||
}
|
||||
@@ -124,26 +123,30 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
||||
}
|
||||
*entropyWritten = 1;
|
||||
DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart));
|
||||
return op-ostart;
|
||||
return (size_t)(op-ostart);
|
||||
}
|
||||
|
||||
static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) {
|
||||
const seqDef* const sstart = sequences;
|
||||
const seqDef* const send = sequences + nbSeq;
|
||||
const seqDef* sp = sstart;
|
||||
static size_t
|
||||
ZSTD_seqDecompressedSize(SeqStore_t const* seqStore,
|
||||
const SeqDef* sequences, size_t nbSeqs,
|
||||
size_t litSize, int lastSubBlock)
|
||||
{
|
||||
size_t matchLengthSum = 0;
|
||||
size_t litLengthSum = 0;
|
||||
(void)(litLengthSum); /* suppress unused variable warning on some environments */
|
||||
while (send-sp > 0) {
|
||||
ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
|
||||
size_t n;
|
||||
for (n=0; n<nbSeqs; n++) {
|
||||
const ZSTD_SequenceLength seqLen = ZSTD_getSequenceLength(seqStore, sequences+n);
|
||||
litLengthSum += seqLen.litLength;
|
||||
matchLengthSum += seqLen.matchLength;
|
||||
sp++;
|
||||
}
|
||||
assert(litLengthSum <= litSize);
|
||||
if (!lastSequence) {
|
||||
DEBUGLOG(5, "ZSTD_seqDecompressedSize: %u sequences from %p: %u literals + %u matchlength",
|
||||
(unsigned)nbSeqs, (const void*)sequences,
|
||||
(unsigned)litLengthSum, (unsigned)matchLengthSum);
|
||||
if (!lastSubBlock)
|
||||
assert(litLengthSum == litSize);
|
||||
}
|
||||
else
|
||||
assert(litLengthSum <= litSize);
|
||||
(void)litLengthSum;
|
||||
return matchLengthSum + litSize;
|
||||
}
|
||||
|
||||
@@ -157,13 +160,14 @@ static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef*
|
||||
* @return : compressed size of sequences section of a sub-block
|
||||
* Or 0 if it is unable to compress
|
||||
* Or error code. */
|
||||
static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
|
||||
const ZSTD_fseCTablesMetadata_t* fseMetadata,
|
||||
const seqDef* sequences, size_t nbSeq,
|
||||
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
void* dst, size_t dstCapacity,
|
||||
const int bmi2, int writeEntropy, int* entropyWritten)
|
||||
static size_t
|
||||
ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
|
||||
const ZSTD_fseCTablesMetadata_t* fseMetadata,
|
||||
const SeqDef* sequences, size_t nbSeq,
|
||||
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
void* dst, size_t dstCapacity,
|
||||
const int bmi2, int writeEntropy, int* entropyWritten)
|
||||
{
|
||||
const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
@@ -177,14 +181,14 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
|
||||
/* Sequences Header */
|
||||
RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
|
||||
dstSize_tooSmall, "");
|
||||
if (nbSeq < 0x7F)
|
||||
if (nbSeq < 128)
|
||||
*op++ = (BYTE)nbSeq;
|
||||
else if (nbSeq < LONGNBSEQ)
|
||||
op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
|
||||
else
|
||||
op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
|
||||
if (nbSeq==0) {
|
||||
return op - ostart;
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
/* seqHead : flags for FSE encoding type */
|
||||
@@ -206,7 +210,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
|
||||
}
|
||||
|
||||
{ size_t const bitstreamSize = ZSTD_encodeSequences(
|
||||
op, oend - op,
|
||||
op, (size_t)(oend - op),
|
||||
fseTables->matchlengthCTable, mlCode,
|
||||
fseTables->offcodeCTable, ofCode,
|
||||
fseTables->litlengthCTable, llCode,
|
||||
@@ -250,7 +254,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
|
||||
#endif
|
||||
|
||||
*entropyWritten = 1;
|
||||
return op - ostart;
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
/** ZSTD_compressSubBlock() :
|
||||
@@ -259,7 +263,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
|
||||
* Or 0 if it failed to compress. */
|
||||
static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
const seqDef* sequences, size_t nbSeq,
|
||||
const SeqDef* sequences, size_t nbSeq,
|
||||
const BYTE* literals, size_t litSize,
|
||||
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
@@ -276,7 +280,8 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
|
||||
litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock);
|
||||
{ size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable,
|
||||
&entropyMetadata->hufMetadata, literals, litSize,
|
||||
op, oend-op, bmi2, writeLitEntropy, litEntropyWritten);
|
||||
op, (size_t)(oend-op),
|
||||
bmi2, writeLitEntropy, litEntropyWritten);
|
||||
FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed");
|
||||
if (cLitSize == 0) return 0;
|
||||
op += cLitSize;
|
||||
@@ -286,18 +291,18 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
|
||||
sequences, nbSeq,
|
||||
llCode, mlCode, ofCode,
|
||||
cctxParams,
|
||||
op, oend-op,
|
||||
op, (size_t)(oend-op),
|
||||
bmi2, writeSeqEntropy, seqEntropyWritten);
|
||||
FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed");
|
||||
if (cSeqSize == 0) return 0;
|
||||
op += cSeqSize;
|
||||
}
|
||||
/* Write block header */
|
||||
{ size_t cSize = (op-ostart)-ZSTD_blockHeaderSize;
|
||||
{ size_t cSize = (size_t)(op-ostart) - ZSTD_blockHeaderSize;
|
||||
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
|
||||
MEM_writeLE24(ostart, cBlockHeader24);
|
||||
}
|
||||
return op-ostart;
|
||||
return (size_t)(op-ostart);
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize,
|
||||
@@ -323,7 +328,7 @@ static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t lit
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
|
||||
static size_t ZSTD_estimateSubBlockSize_symbolType(SymbolEncodingType_e type,
|
||||
const BYTE* codeTable, unsigned maxCode,
|
||||
size_t nbSeq, const FSE_CTable* fseCTable,
|
||||
const U8* additionalBits,
|
||||
@@ -386,7 +391,11 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
|
||||
return cSeqSizeEstimate + sequencesSectionHeaderSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
|
||||
typedef struct {
|
||||
size_t estLitSize;
|
||||
size_t estBlockSize;
|
||||
} EstimatedBlockSize;
|
||||
static EstimatedBlockSize ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
|
||||
const BYTE* ofCodeTable,
|
||||
const BYTE* llCodeTable,
|
||||
const BYTE* mlCodeTable,
|
||||
@@ -394,15 +403,17 @@ static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
|
||||
const ZSTD_entropyCTables_t* entropy,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize,
|
||||
int writeLitEntropy, int writeSeqEntropy) {
|
||||
size_t cSizeEstimate = 0;
|
||||
cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize,
|
||||
&entropy->huf, &entropyMetadata->hufMetadata,
|
||||
workspace, wkspSize, writeLitEntropy);
|
||||
cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
|
||||
int writeLitEntropy, int writeSeqEntropy)
|
||||
{
|
||||
EstimatedBlockSize ebs;
|
||||
ebs.estLitSize = ZSTD_estimateSubBlockSize_literal(literals, litSize,
|
||||
&entropy->huf, &entropyMetadata->hufMetadata,
|
||||
workspace, wkspSize, writeLitEntropy);
|
||||
ebs.estBlockSize = ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
|
||||
nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
|
||||
workspace, wkspSize, writeSeqEntropy);
|
||||
return cSizeEstimate + ZSTD_blockHeaderSize;
|
||||
ebs.estBlockSize += ebs.estLitSize + ZSTD_blockHeaderSize;
|
||||
return ebs;
|
||||
}
|
||||
|
||||
static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata)
|
||||
@@ -416,14 +427,57 @@ static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMe
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t countLiterals(SeqStore_t const* seqStore, const SeqDef* sp, size_t seqCount)
|
||||
{
|
||||
size_t n, total = 0;
|
||||
assert(sp != NULL);
|
||||
for (n=0; n<seqCount; n++) {
|
||||
total += ZSTD_getSequenceLength(seqStore, sp+n).litLength;
|
||||
}
|
||||
DEBUGLOG(6, "countLiterals for %zu sequences from %p => %zu bytes", seqCount, (const void*)sp, total);
|
||||
return total;
|
||||
}
|
||||
|
||||
#define BYTESCALE 256
|
||||
|
||||
static size_t sizeBlockSequences(const SeqDef* sp, size_t nbSeqs,
|
||||
size_t targetBudget, size_t avgLitCost, size_t avgSeqCost,
|
||||
int firstSubBlock)
|
||||
{
|
||||
size_t n, budget = 0, inSize=0;
|
||||
/* entropy headers */
|
||||
size_t const headerSize = (size_t)firstSubBlock * 120 * BYTESCALE; /* generous estimate */
|
||||
assert(firstSubBlock==0 || firstSubBlock==1);
|
||||
budget += headerSize;
|
||||
|
||||
/* first sequence => at least one sequence*/
|
||||
budget += sp[0].litLength * avgLitCost + avgSeqCost;
|
||||
if (budget > targetBudget) return 1;
|
||||
inSize = sp[0].litLength + (sp[0].mlBase+MINMATCH);
|
||||
|
||||
/* loop over sequences */
|
||||
for (n=1; n<nbSeqs; n++) {
|
||||
size_t currentCost = sp[n].litLength * avgLitCost + avgSeqCost;
|
||||
budget += currentCost;
|
||||
inSize += sp[n].litLength + (sp[n].mlBase+MINMATCH);
|
||||
/* stop when sub-block budget is reached */
|
||||
if ( (budget > targetBudget)
|
||||
/* though continue to expand until the sub-block is deemed compressible */
|
||||
&& (budget < inSize * BYTESCALE) )
|
||||
break;
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
/** ZSTD_compressSubBlock_multi() :
|
||||
* Breaks super-block into multiple sub-blocks and compresses them.
|
||||
* Entropy will be written to the first block.
|
||||
* The following blocks will use repeat mode to compress.
|
||||
* All sub-blocks are compressed blocks (no raw or rle blocks).
|
||||
* @return : compressed size of the super block (which is multiple ZSTD blocks)
|
||||
* Or 0 if it failed to compress. */
|
||||
static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
|
||||
* Entropy will be written into the first block.
|
||||
* The following blocks use repeat_mode to compress.
|
||||
* Sub-blocks are all compressed, except the last one when beneficial.
|
||||
* @return : compressed size of the super block (which features multiple ZSTD blocks)
|
||||
* or 0 if it failed to compress. */
|
||||
static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr,
|
||||
const ZSTD_compressedBlockState_t* prevCBlock,
|
||||
ZSTD_compressedBlockState_t* nextCBlock,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
@@ -433,12 +487,14 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
|
||||
const int bmi2, U32 lastBlock,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
const seqDef* const sstart = seqStorePtr->sequencesStart;
|
||||
const seqDef* const send = seqStorePtr->sequences;
|
||||
const seqDef* sp = sstart;
|
||||
const SeqDef* const sstart = seqStorePtr->sequencesStart;
|
||||
const SeqDef* const send = seqStorePtr->sequences;
|
||||
const SeqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */
|
||||
size_t const nbSeqs = (size_t)(send - sstart);
|
||||
const BYTE* const lstart = seqStorePtr->litStart;
|
||||
const BYTE* const lend = seqStorePtr->lit;
|
||||
const BYTE* lp = lstart;
|
||||
size_t const nbLiterals = (size_t)(lend - lstart);
|
||||
BYTE const* ip = (BYTE const*)src;
|
||||
BYTE const* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
@@ -447,112 +503,171 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
|
||||
const BYTE* llCodePtr = seqStorePtr->llCode;
|
||||
const BYTE* mlCodePtr = seqStorePtr->mlCode;
|
||||
const BYTE* ofCodePtr = seqStorePtr->ofCode;
|
||||
size_t targetCBlockSize = cctxParams->targetCBlockSize;
|
||||
size_t litSize, seqCount;
|
||||
int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed;
|
||||
size_t const minTarget = ZSTD_TARGETCBLOCKSIZE_MIN; /* enforce minimum size, to reduce undesirable side effects */
|
||||
size_t const targetCBlockSize = MAX(minTarget, cctxParams->targetCBlockSize);
|
||||
int writeLitEntropy = (entropyMetadata->hufMetadata.hType == set_compressed);
|
||||
int writeSeqEntropy = 1;
|
||||
int lastSequence = 0;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)",
|
||||
(unsigned)(lend-lp), (unsigned)(send-sstart));
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi (srcSize=%u, litSize=%u, nbSeq=%u)",
|
||||
(unsigned)srcSize, (unsigned)(lend-lstart), (unsigned)(send-sstart));
|
||||
|
||||
litSize = 0;
|
||||
seqCount = 0;
|
||||
do {
|
||||
size_t cBlockSizeEstimate = 0;
|
||||
if (sstart == send) {
|
||||
lastSequence = 1;
|
||||
} else {
|
||||
const seqDef* const sequence = sp + seqCount;
|
||||
lastSequence = sequence == send - 1;
|
||||
litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength;
|
||||
seqCount++;
|
||||
/* let's start by a general estimation for the full block */
|
||||
if (nbSeqs > 0) {
|
||||
EstimatedBlockSize const ebs =
|
||||
ZSTD_estimateSubBlockSize(lp, nbLiterals,
|
||||
ofCodePtr, llCodePtr, mlCodePtr, nbSeqs,
|
||||
&nextCBlock->entropy, entropyMetadata,
|
||||
workspace, wkspSize,
|
||||
writeLitEntropy, writeSeqEntropy);
|
||||
/* quick estimation */
|
||||
size_t const avgLitCost = nbLiterals ? (ebs.estLitSize * BYTESCALE) / nbLiterals : BYTESCALE;
|
||||
size_t const avgSeqCost = ((ebs.estBlockSize - ebs.estLitSize) * BYTESCALE) / nbSeqs;
|
||||
const size_t nbSubBlocks = MAX((ebs.estBlockSize + (targetCBlockSize/2)) / targetCBlockSize, 1);
|
||||
size_t n, avgBlockBudget, blockBudgetSupp=0;
|
||||
avgBlockBudget = (ebs.estBlockSize * BYTESCALE) / nbSubBlocks;
|
||||
DEBUGLOG(5, "estimated fullblock size=%u bytes ; avgLitCost=%.2f ; avgSeqCost=%.2f ; targetCBlockSize=%u, nbSubBlocks=%u ; avgBlockBudget=%.0f bytes",
|
||||
(unsigned)ebs.estBlockSize, (double)avgLitCost/BYTESCALE, (double)avgSeqCost/BYTESCALE,
|
||||
(unsigned)targetCBlockSize, (unsigned)nbSubBlocks, (double)avgBlockBudget/BYTESCALE);
|
||||
/* simplification: if estimates states that the full superblock doesn't compress, just bail out immediately
|
||||
* this will result in the production of a single uncompressed block covering @srcSize.*/
|
||||
if (ebs.estBlockSize > srcSize) return 0;
|
||||
|
||||
/* compress and write sub-blocks */
|
||||
assert(nbSubBlocks>0);
|
||||
for (n=0; n < nbSubBlocks-1; n++) {
|
||||
/* determine nb of sequences for current sub-block + nbLiterals from next sequence */
|
||||
size_t const seqCount = sizeBlockSequences(sp, (size_t)(send-sp),
|
||||
avgBlockBudget + blockBudgetSupp, avgLitCost, avgSeqCost, n==0);
|
||||
/* if reached last sequence : break to last sub-block (simplification) */
|
||||
assert(seqCount <= (size_t)(send-sp));
|
||||
if (sp + seqCount == send) break;
|
||||
assert(seqCount > 0);
|
||||
/* compress sub-block */
|
||||
{ int litEntropyWritten = 0;
|
||||
int seqEntropyWritten = 0;
|
||||
size_t litSize = countLiterals(seqStorePtr, sp, seqCount);
|
||||
const size_t decompressedSize =
|
||||
ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 0);
|
||||
size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
|
||||
sp, seqCount,
|
||||
lp, litSize,
|
||||
llCodePtr, mlCodePtr, ofCodePtr,
|
||||
cctxParams,
|
||||
op, (size_t)(oend-op),
|
||||
bmi2, writeLitEntropy, writeSeqEntropy,
|
||||
&litEntropyWritten, &seqEntropyWritten,
|
||||
0);
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
|
||||
|
||||
/* check compressibility, update state components */
|
||||
if (cSize > 0 && cSize < decompressedSize) {
|
||||
DEBUGLOG(5, "Committed sub-block compressing %u bytes => %u bytes",
|
||||
(unsigned)decompressedSize, (unsigned)cSize);
|
||||
assert(ip + decompressedSize <= iend);
|
||||
ip += decompressedSize;
|
||||
lp += litSize;
|
||||
op += cSize;
|
||||
llCodePtr += seqCount;
|
||||
mlCodePtr += seqCount;
|
||||
ofCodePtr += seqCount;
|
||||
/* Entropy only needs to be written once */
|
||||
if (litEntropyWritten) {
|
||||
writeLitEntropy = 0;
|
||||
}
|
||||
if (seqEntropyWritten) {
|
||||
writeSeqEntropy = 0;
|
||||
}
|
||||
sp += seqCount;
|
||||
blockBudgetSupp = 0;
|
||||
} }
|
||||
/* otherwise : do not compress yet, coalesce current sub-block with following one */
|
||||
}
|
||||
if (lastSequence) {
|
||||
assert(lp <= lend);
|
||||
assert(litSize <= (size_t)(lend - lp));
|
||||
litSize = (size_t)(lend - lp);
|
||||
}
|
||||
/* I think there is an optimization opportunity here.
|
||||
* Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful
|
||||
* since it recalculates estimate from scratch.
|
||||
* For example, it would recount literal distribution and symbol codes every time.
|
||||
*/
|
||||
cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount,
|
||||
&nextCBlock->entropy, entropyMetadata,
|
||||
workspace, wkspSize, writeLitEntropy, writeSeqEntropy);
|
||||
if (cBlockSizeEstimate > targetCBlockSize || lastSequence) {
|
||||
int litEntropyWritten = 0;
|
||||
int seqEntropyWritten = 0;
|
||||
const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence);
|
||||
const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
|
||||
sp, seqCount,
|
||||
lp, litSize,
|
||||
llCodePtr, mlCodePtr, ofCodePtr,
|
||||
cctxParams,
|
||||
op, oend-op,
|
||||
bmi2, writeLitEntropy, writeSeqEntropy,
|
||||
&litEntropyWritten, &seqEntropyWritten,
|
||||
lastBlock && lastSequence);
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
|
||||
if (cSize > 0 && cSize < decompressedSize) {
|
||||
DEBUGLOG(5, "Committed the sub-block");
|
||||
assert(ip + decompressedSize <= iend);
|
||||
ip += decompressedSize;
|
||||
sp += seqCount;
|
||||
lp += litSize;
|
||||
op += cSize;
|
||||
llCodePtr += seqCount;
|
||||
mlCodePtr += seqCount;
|
||||
ofCodePtr += seqCount;
|
||||
litSize = 0;
|
||||
seqCount = 0;
|
||||
/* Entropy only needs to be written once */
|
||||
if (litEntropyWritten) {
|
||||
writeLitEntropy = 0;
|
||||
}
|
||||
if (seqEntropyWritten) {
|
||||
writeSeqEntropy = 0;
|
||||
}
|
||||
} /* if (nbSeqs > 0) */
|
||||
|
||||
/* write last block */
|
||||
DEBUGLOG(5, "Generate last sub-block: %u sequences remaining", (unsigned)(send - sp));
|
||||
{ int litEntropyWritten = 0;
|
||||
int seqEntropyWritten = 0;
|
||||
size_t litSize = (size_t)(lend - lp);
|
||||
size_t seqCount = (size_t)(send - sp);
|
||||
const size_t decompressedSize =
|
||||
ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 1);
|
||||
size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
|
||||
sp, seqCount,
|
||||
lp, litSize,
|
||||
llCodePtr, mlCodePtr, ofCodePtr,
|
||||
cctxParams,
|
||||
op, (size_t)(oend-op),
|
||||
bmi2, writeLitEntropy, writeSeqEntropy,
|
||||
&litEntropyWritten, &seqEntropyWritten,
|
||||
lastBlock);
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
|
||||
|
||||
/* update pointers, the nb of literals borrowed from next sequence must be preserved */
|
||||
if (cSize > 0 && cSize < decompressedSize) {
|
||||
DEBUGLOG(5, "Last sub-block compressed %u bytes => %u bytes",
|
||||
(unsigned)decompressedSize, (unsigned)cSize);
|
||||
assert(ip + decompressedSize <= iend);
|
||||
ip += decompressedSize;
|
||||
lp += litSize;
|
||||
op += cSize;
|
||||
llCodePtr += seqCount;
|
||||
mlCodePtr += seqCount;
|
||||
ofCodePtr += seqCount;
|
||||
/* Entropy only needs to be written once */
|
||||
if (litEntropyWritten) {
|
||||
writeLitEntropy = 0;
|
||||
}
|
||||
if (seqEntropyWritten) {
|
||||
writeSeqEntropy = 0;
|
||||
}
|
||||
sp += seqCount;
|
||||
}
|
||||
} while (!lastSequence);
|
||||
}
|
||||
|
||||
|
||||
if (writeLitEntropy) {
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
|
||||
DEBUGLOG(5, "Literal entropy tables were never written");
|
||||
ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
|
||||
}
|
||||
if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
|
||||
/* If we haven't written our entropy tables, then we've violated our contract and
|
||||
* must emit an uncompressed block.
|
||||
*/
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten");
|
||||
DEBUGLOG(5, "Sequence entropy tables were never written => cancel, emit an uncompressed block");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (ip < iend) {
|
||||
size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock);
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip));
|
||||
/* some data left : last part of the block sent uncompressed */
|
||||
size_t const rSize = (size_t)((iend - ip));
|
||||
size_t const cSize = ZSTD_noCompressBlock(op, (size_t)(oend - op), ip, rSize, lastBlock);
|
||||
DEBUGLOG(5, "Generate last uncompressed sub-block of %u bytes", (unsigned)(rSize));
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
|
||||
assert(cSize != 0);
|
||||
op += cSize;
|
||||
/* We have to regenerate the repcodes because we've skipped some sequences */
|
||||
if (sp < send) {
|
||||
seqDef const* seq;
|
||||
repcodes_t rep;
|
||||
const SeqDef* seq;
|
||||
Repcodes_t rep;
|
||||
ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
|
||||
for (seq = sstart; seq < sp; ++seq) {
|
||||
ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
|
||||
ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
|
||||
}
|
||||
ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
|
||||
}
|
||||
}
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
|
||||
return op-ostart;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed all subBlocks: total compressed size = %u",
|
||||
(unsigned)(op-ostart));
|
||||
return (size_t)(op-ostart);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
||||
void* dst, size_t dstCapacity,
|
||||
void const* src, size_t srcSize,
|
||||
unsigned lastBlock) {
|
||||
const void* src, size_t srcSize,
|
||||
unsigned lastBlock)
|
||||
{
|
||||
ZSTD_entropyCTablesMetadata_t entropyMetadata;
|
||||
|
||||
FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
|
||||
@@ -560,7 +675,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
||||
&zc->blockState.nextCBlock->entropy,
|
||||
&zc->appliedParams,
|
||||
&entropyMetadata,
|
||||
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
|
||||
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), "");
|
||||
|
||||
return ZSTD_compressSubBlock_multi(&zc->seqStore,
|
||||
zc->blockState.prevCBlock,
|
||||
@@ -570,7 +685,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
||||
dst, dstCapacity,
|
||||
src, srcSize,
|
||||
zc->bmi2, lastBlock,
|
||||
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
|
||||
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/zstd_compress_superblock.h
generated
vendored
2
vendor/github.com/DataDog/zstd/zstd_compress_superblock.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
29
vendor/github.com/DataDog/zstd/zstd_ctx.go
generated
vendored
29
vendor/github.com/DataDog/zstd/zstd_ctx.go
generated
vendored
@@ -24,6 +24,15 @@ type Ctx interface {
|
||||
// prevent allocation. If it is too small, or if nil is passed, a new buffer
|
||||
// will be allocated and returned.
|
||||
Decompress(dst, src []byte) ([]byte, error)
|
||||
|
||||
// DecompressInto decompresses src into dst. Unlike Decompress, DecompressInto
|
||||
// requires that dst be sufficiently large to hold the decompressed payload.
|
||||
// DecompressInto may be used when the caller knows the size of the decompressed
|
||||
// payload before attempting decompression.
|
||||
//
|
||||
// It returns the number of bytes copied and an error if any is encountered. If
|
||||
// dst is too small, DecompressInto errors.
|
||||
DecompressInto(dst, src []byte) (int, error)
|
||||
}
|
||||
|
||||
type ctx struct {
|
||||
@@ -104,14 +113,7 @@ func (c *ctx) Decompress(dst, src []byte) ([]byte, error) {
|
||||
dst = make([]byte, bound)
|
||||
}
|
||||
|
||||
written := int(C.ZSTD_decompressDCtx(
|
||||
c.dctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src))))
|
||||
|
||||
err := getError(written)
|
||||
written, err := c.DecompressInto(dst, src)
|
||||
if err == nil {
|
||||
return dst[:written], nil
|
||||
}
|
||||
@@ -125,6 +127,17 @@ func (c *ctx) Decompress(dst, src []byte) ([]byte, error) {
|
||||
return ioutil.ReadAll(r)
|
||||
}
|
||||
|
||||
func (c *ctx) DecompressInto(dst, src []byte) (int, error) {
|
||||
written := int(C.ZSTD_decompressDCtx(
|
||||
c.dctx,
|
||||
unsafe.Pointer(&dst[0]),
|
||||
C.size_t(len(dst)),
|
||||
unsafe.Pointer(&src[0]),
|
||||
C.size_t(len(src))))
|
||||
err := getError(written)
|
||||
return written, err
|
||||
}
|
||||
|
||||
func finalizeCtx(c *ctx) {
|
||||
C.ZSTD_freeCCtx(c.cctx)
|
||||
C.ZSTD_freeDCtx(c.dctx)
|
||||
|
||||
279
vendor/github.com/DataDog/zstd/zstd_cwksp.h
generated
vendored
279
vendor/github.com/DataDog/zstd/zstd_cwksp.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -15,11 +15,10 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */
|
||||
#include "zstd_internal.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
#include "portability_macros.h"
|
||||
#include "compiler.h" /* ZS2_isPower2 */
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
@@ -45,8 +44,9 @@ extern "C" {
|
||||
***************************************/
|
||||
typedef enum {
|
||||
ZSTD_cwksp_alloc_objects,
|
||||
ZSTD_cwksp_alloc_buffers,
|
||||
ZSTD_cwksp_alloc_aligned
|
||||
ZSTD_cwksp_alloc_aligned_init_once,
|
||||
ZSTD_cwksp_alloc_aligned,
|
||||
ZSTD_cwksp_alloc_buffers
|
||||
} ZSTD_cwksp_alloc_phase_e;
|
||||
|
||||
/**
|
||||
@@ -99,8 +99,8 @@ typedef enum {
|
||||
*
|
||||
* Workspace Layout:
|
||||
*
|
||||
* [ ... workspace ... ]
|
||||
* [objects][tables ... ->] free space [<- ... aligned][<- ... buffers]
|
||||
* [ ... workspace ... ]
|
||||
* [objects][tables ->] free space [<- buffers][<- aligned][<- init once]
|
||||
*
|
||||
* The various objects that live in the workspace are divided into the
|
||||
* following categories, and are allocated separately:
|
||||
@@ -124,9 +124,18 @@ typedef enum {
|
||||
* uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
|
||||
* Their sizes depend on the cparams. These tables are 64-byte aligned.
|
||||
*
|
||||
* - Aligned: these buffers are used for various purposes that require 4 byte
|
||||
* alignment, but don't require any initialization before they're used. These
|
||||
* buffers are each aligned to 64 bytes.
|
||||
* - Init once: these buffers require to be initialized at least once before
|
||||
* use. They should be used when we want to skip memory initialization
|
||||
* while not triggering memory checkers (like Valgrind) when reading from
|
||||
* from this memory without writing to it first.
|
||||
* These buffers should be used carefully as they might contain data
|
||||
* from previous compressions.
|
||||
* Buffers are aligned to 64 bytes.
|
||||
*
|
||||
* - Aligned: these buffers don't require any initialization before they're
|
||||
* used. The user of the buffer should make sure they write into a buffer
|
||||
* location before reading from it.
|
||||
* Buffers are aligned to 64 bytes.
|
||||
*
|
||||
* - Buffers: these buffers are used for various purposes that don't require
|
||||
* any alignment or initialization before they're used. This means they can
|
||||
@@ -138,8 +147,9 @@ typedef enum {
|
||||
* correctly packed into the workspace buffer. That order is:
|
||||
*
|
||||
* 1. Objects
|
||||
* 2. Buffers
|
||||
* 3. Aligned/Tables
|
||||
* 2. Init once / Tables
|
||||
* 3. Aligned / Tables
|
||||
* 4. Buffers / Tables
|
||||
*
|
||||
* Attempts to reserve objects of different types out of order will fail.
|
||||
*/
|
||||
@@ -151,6 +161,7 @@ typedef struct {
|
||||
void* tableEnd;
|
||||
void* tableValidEnd;
|
||||
void* allocStart;
|
||||
void* initOnceStart;
|
||||
|
||||
BYTE allocFailed;
|
||||
int workspaceOversizedDuration;
|
||||
@@ -163,6 +174,7 @@ typedef struct {
|
||||
***************************************/
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws);
|
||||
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws);
|
||||
|
||||
MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
|
||||
(void)ws;
|
||||
@@ -172,14 +184,29 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
|
||||
assert(ws->tableEnd <= ws->allocStart);
|
||||
assert(ws->tableValidEnd <= ws->allocStart);
|
||||
assert(ws->allocStart <= ws->workspaceEnd);
|
||||
assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws));
|
||||
assert(ws->workspace <= ws->initOnceStart);
|
||||
#if ZSTD_MEMORY_SANITIZER
|
||||
{
|
||||
intptr_t const offset = __msan_test_shadow(ws->initOnceStart,
|
||||
(U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart);
|
||||
(void)offset;
|
||||
#if defined(ZSTD_MSAN_PRINT)
|
||||
if(offset!=-1) {
|
||||
__msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32);
|
||||
}
|
||||
#endif
|
||||
assert(offset==-1);
|
||||
};
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Align must be a power of 2.
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
|
||||
MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t align) {
|
||||
size_t const mask = align - 1;
|
||||
assert((align & mask) == 0);
|
||||
assert(ZSTD_isPower2(align));
|
||||
return (size + mask) & ~mask;
|
||||
}
|
||||
|
||||
@@ -193,7 +220,7 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
|
||||
* to figure out how much space you need for the matchState tables. Everything
|
||||
* else is though.
|
||||
*
|
||||
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size().
|
||||
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned64_alloc_size().
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
|
||||
if (size == 0)
|
||||
@@ -205,12 +232,16 @@ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size, size_t alignment) {
|
||||
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, alignment));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes.
|
||||
* Used to determine the number of bytes required for a given "aligned".
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
|
||||
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES));
|
||||
MEM_STATIC size_t ZSTD_cwksp_aligned64_alloc_size(size_t size) {
|
||||
return ZSTD_cwksp_aligned_alloc_size(size, ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -218,14 +249,10 @@ MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
|
||||
* for internal purposes (currently only alignment).
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
|
||||
/* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes
|
||||
* to align the beginning of tables section, as well as another n_2=[0, 63] bytes
|
||||
* to align the beginning of the aligned section.
|
||||
*
|
||||
* n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and
|
||||
* aligneds being sized in multiples of 64 bytes.
|
||||
/* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES
|
||||
* bytes to align the beginning of tables section and end of buffers;
|
||||
*/
|
||||
size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES;
|
||||
size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2;
|
||||
return slackSpace;
|
||||
}
|
||||
|
||||
@@ -237,11 +264,23 @@ MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
|
||||
MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) {
|
||||
size_t const alignBytesMask = alignBytes - 1;
|
||||
size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
|
||||
assert((alignBytes & alignBytesMask) == 0);
|
||||
assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
assert(ZSTD_isPower2(alignBytes));
|
||||
assert(bytes < alignBytes);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the initial value for allocStart which is used to determine the position from
|
||||
* which we can allocate from the end of the workspace.
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws)
|
||||
{
|
||||
char* endPtr = (char*)ws->workspaceEnd;
|
||||
assert(ZSTD_isPower2(ZSTD_CWKSP_ALIGNMENT_BYTES));
|
||||
endPtr = endPtr - ((size_t)endPtr % ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
return (void*)endPtr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal function. Do not use directly.
|
||||
* Reserves the given number of bytes within the aligned/buffer segment of the wksp,
|
||||
@@ -254,7 +293,7 @@ ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes)
|
||||
{
|
||||
void* const alloc = (BYTE*)ws->allocStart - bytes;
|
||||
void* const bottom = ws->tableEnd;
|
||||
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
|
||||
DEBUGLOG(5, "cwksp: reserving [0x%p]:%zd bytes; %zd bytes remaining",
|
||||
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
assert(alloc >= bottom);
|
||||
@@ -282,27 +321,16 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
|
||||
{
|
||||
assert(phase >= ws->phase);
|
||||
if (phase > ws->phase) {
|
||||
/* Going from allocating objects to allocating buffers */
|
||||
if (ws->phase < ZSTD_cwksp_alloc_buffers &&
|
||||
phase >= ZSTD_cwksp_alloc_buffers) {
|
||||
/* Going from allocating objects to allocating initOnce / tables */
|
||||
if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once &&
|
||||
phase >= ZSTD_cwksp_alloc_aligned_init_once) {
|
||||
ws->tableValidEnd = ws->objectEnd;
|
||||
}
|
||||
ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws);
|
||||
|
||||
/* Going from allocating buffers to allocating aligneds/tables */
|
||||
if (ws->phase < ZSTD_cwksp_alloc_aligned &&
|
||||
phase >= ZSTD_cwksp_alloc_aligned) {
|
||||
{ /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */
|
||||
size_t const bytesToAlign =
|
||||
ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign);
|
||||
ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */
|
||||
RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign),
|
||||
memory_allocation, "aligned phase - alignment initial allocation failed!");
|
||||
}
|
||||
{ /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */
|
||||
void* const alloc = ws->objectEnd;
|
||||
void *const alloc = ws->objectEnd;
|
||||
size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
void* const objectEnd = (BYTE*)alloc + bytesToAlign;
|
||||
void *const objectEnd = (BYTE *) alloc + bytesToAlign;
|
||||
DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign);
|
||||
RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation,
|
||||
"table phase - alignment initial allocation failed!");
|
||||
@@ -310,7 +338,9 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
|
||||
ws->tableEnd = objectEnd; /* table area starts being empty */
|
||||
if (ws->tableValidEnd < ws->tableEnd) {
|
||||
ws->tableValidEnd = ws->tableEnd;
|
||||
} } }
|
||||
}
|
||||
}
|
||||
}
|
||||
ws->phase = phase;
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
}
|
||||
@@ -322,7 +352,7 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
|
||||
*/
|
||||
MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr)
|
||||
{
|
||||
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
|
||||
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -349,7 +379,9 @@ ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase
|
||||
if (alloc) {
|
||||
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
/* We need to keep the redzone poisoned while unpoisoning the bytes that
|
||||
* are actually allocated. */
|
||||
__asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -367,29 +399,64 @@ MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes)
|
||||
|
||||
/**
|
||||
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
|
||||
* This memory has been initialized at least once in the past.
|
||||
* This doesn't mean it has been initialized this time, and it might contain data from previous
|
||||
* operations.
|
||||
* The main usage is for algorithms that might need read access into uninitialized memory.
|
||||
* The algorithm must maintain safety under these conditions and must make sure it doesn't
|
||||
* leak any of the past data (directly or in side channels).
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes)
|
||||
{
|
||||
void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
|
||||
ZSTD_cwksp_alloc_aligned);
|
||||
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
|
||||
size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES);
|
||||
void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once);
|
||||
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
|
||||
if(ptr && ptr < ws->initOnceStart) {
|
||||
/* We assume the memory following the current allocation is either:
|
||||
* 1. Not usable as initOnce memory (end of workspace)
|
||||
* 2. Another initOnce buffer that has been allocated before (and so was previously memset)
|
||||
* 3. An ASAN redzone, in which case we don't want to write on it
|
||||
* For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart.
|
||||
* Note that we assume here that MSAN and ASAN cannot run in the same time. */
|
||||
ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes));
|
||||
ws->initOnceStart = ptr;
|
||||
}
|
||||
#if ZSTD_MEMORY_SANITIZER
|
||||
assert(__msan_test_shadow(ptr, bytes) == -1);
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_aligned64(ZSTD_cwksp* ws, size_t bytes)
|
||||
{
|
||||
void* const ptr = ZSTD_cwksp_reserve_internal(ws,
|
||||
ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
|
||||
ZSTD_cwksp_alloc_aligned);
|
||||
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aligned on 64 bytes. These buffers have the special property that
|
||||
* their values remain constrained, allowing us to re-use them without
|
||||
* their values remain constrained, allowing us to reuse them without
|
||||
* memset()-ing them.
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
|
||||
{
|
||||
const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
|
||||
const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once;
|
||||
void* alloc;
|
||||
void* end;
|
||||
void* top;
|
||||
|
||||
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
|
||||
return NULL;
|
||||
/* We can only start allocating tables after we are done reserving space for objects at the
|
||||
* start of the workspace */
|
||||
if(ws->phase < phase) {
|
||||
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
alloc = ws->tableEnd;
|
||||
end = (BYTE *)alloc + bytes;
|
||||
@@ -414,7 +481,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
|
||||
#endif
|
||||
|
||||
assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
|
||||
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
|
||||
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
|
||||
return alloc;
|
||||
}
|
||||
|
||||
@@ -460,19 +527,41 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes)
|
||||
|
||||
return alloc;
|
||||
}
|
||||
/**
|
||||
* with alignment control
|
||||
* Note : should happen only once, at workspace first initialization
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_object_aligned(ZSTD_cwksp* ws, size_t byteSize, size_t alignment)
|
||||
{
|
||||
size_t const mask = alignment - 1;
|
||||
size_t const surplus = (alignment > sizeof(void*)) ? alignment - sizeof(void*) : 0;
|
||||
void* const start = ZSTD_cwksp_reserve_object(ws, byteSize + surplus);
|
||||
if (start == NULL) return NULL;
|
||||
if (surplus == 0) return start;
|
||||
assert(ZSTD_isPower2(alignment));
|
||||
return (void*)(((size_t)start + surplus) & ~mask);
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws)
|
||||
{
|
||||
DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty");
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
/* To validate that the table re-use logic is sound, and that we don't
|
||||
/* To validate that the table reuse logic is sound, and that we don't
|
||||
* access table space that we haven't cleaned, we re-"poison" the table
|
||||
* space every time we mark it dirty. */
|
||||
* space every time we mark it dirty.
|
||||
* Since tableValidEnd space and initOnce space may overlap we don't poison
|
||||
* the initOnce portion as it break its promise. This means that this poisoning
|
||||
* check isn't always applied fully. */
|
||||
{
|
||||
size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
|
||||
assert(__msan_test_shadow(ws->objectEnd, size) == -1);
|
||||
__msan_poison(ws->objectEnd, size);
|
||||
if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) {
|
||||
__msan_poison(ws->objectEnd, size);
|
||||
} else {
|
||||
assert(ws->initOnceStart >= ws->objectEnd);
|
||||
__msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -500,7 +589,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
|
||||
assert(ws->tableValidEnd >= ws->objectEnd);
|
||||
assert(ws->tableValidEnd <= ws->allocStart);
|
||||
if (ws->tableValidEnd < ws->tableEnd) {
|
||||
ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd);
|
||||
ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd));
|
||||
}
|
||||
ZSTD_cwksp_mark_tables_clean(ws);
|
||||
}
|
||||
@@ -509,7 +598,8 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
|
||||
* Invalidates table allocations.
|
||||
* All other allocations remain valid.
|
||||
*/
|
||||
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
|
||||
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws)
|
||||
{
|
||||
DEBUGLOG(4, "cwksp: clearing tables!");
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
@@ -535,13 +625,16 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
|
||||
DEBUGLOG(4, "cwksp: clearing!");
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
/* To validate that the context re-use logic is sound, and that we don't
|
||||
/* To validate that the context reuse logic is sound, and that we don't
|
||||
* access stuff that this compression hasn't initialized, we re-"poison"
|
||||
* the workspace (or at least the non-static, non-table parts of it)
|
||||
* every time we start a new compression. */
|
||||
* the workspace except for the areas in which we expect memory reuse
|
||||
* without initialization (objects, valid tables area and init once
|
||||
* memory). */
|
||||
{
|
||||
size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd;
|
||||
__msan_poison(ws->tableValidEnd, size);
|
||||
if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) {
|
||||
size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd;
|
||||
__msan_poison(ws->tableValidEnd, size);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -557,14 +650,23 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
|
||||
#endif
|
||||
|
||||
ws->tableEnd = ws->objectEnd;
|
||||
ws->allocStart = ws->workspaceEnd;
|
||||
ws->allocStart = ZSTD_cwksp_initialAllocStart(ws);
|
||||
ws->allocFailed = 0;
|
||||
if (ws->phase > ZSTD_cwksp_alloc_buffers) {
|
||||
ws->phase = ZSTD_cwksp_alloc_buffers;
|
||||
if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) {
|
||||
ws->phase = ZSTD_cwksp_alloc_aligned_init_once;
|
||||
}
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
|
||||
return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) {
|
||||
return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace)
|
||||
+ (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart);
|
||||
}
|
||||
|
||||
/**
|
||||
* The provided workspace takes ownership of the buffer [start, start+size).
|
||||
* Any existing values in the workspace are ignored (the previously managed
|
||||
@@ -577,6 +679,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_c
|
||||
ws->workspaceEnd = (BYTE*)start + size;
|
||||
ws->objectEnd = ws->workspace;
|
||||
ws->tableValidEnd = ws->objectEnd;
|
||||
ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws);
|
||||
ws->phase = ZSTD_cwksp_alloc_objects;
|
||||
ws->isStatic = isStatic;
|
||||
ZSTD_cwksp_clear(ws);
|
||||
@@ -595,6 +698,11 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem
|
||||
MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
|
||||
void *ptr = ws->workspace;
|
||||
DEBUGLOG(4, "cwksp: freeing workspace");
|
||||
#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
if (ptr != NULL && customMem.customFree != NULL) {
|
||||
__msan_unpoison(ptr, ZSTD_cwksp_sizeof(ws));
|
||||
}
|
||||
#endif
|
||||
ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp));
|
||||
ZSTD_customFree(ptr, customMem);
|
||||
}
|
||||
@@ -608,15 +716,6 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
|
||||
ZSTD_memset(src, 0, sizeof(ZSTD_cwksp));
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
|
||||
return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) {
|
||||
return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace)
|
||||
+ (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart);
|
||||
}
|
||||
|
||||
MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
|
||||
return ws->allocFailed;
|
||||
}
|
||||
@@ -629,17 +728,11 @@ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
|
||||
* Returns if the estimated space needed for a wksp is within an acceptable limit of the
|
||||
* actual amount of space used.
|
||||
*/
|
||||
MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws,
|
||||
size_t const estimatedSpace, int resizedWorkspace) {
|
||||
if (resizedWorkspace) {
|
||||
/* Resized/newly allocated wksp should have exact bounds */
|
||||
return ZSTD_cwksp_used(ws) == estimatedSpace;
|
||||
} else {
|
||||
/* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes
|
||||
* than estimatedSpace. See the comments in zstd_cwksp.h for details.
|
||||
*/
|
||||
return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63);
|
||||
}
|
||||
MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) {
|
||||
/* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice
|
||||
* the alignment bytes difference between estimation and actual usage */
|
||||
return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) &&
|
||||
ZSTD_cwksp_used(ws) <= estimatedSpace;
|
||||
}
|
||||
|
||||
|
||||
@@ -670,10 +763,6 @@ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
|
||||
}
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_CWKSP_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
8
vendor/github.com/DataDog/zstd/zstd_ddict.c
generated
vendored
8
vendor/github.com/DataDog/zstd/zstd_ddict.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -15,12 +15,12 @@
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */
|
||||
#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
|
||||
#include "cpu.h" /* bmi2 */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "zstd_decompress_internal.h"
|
||||
#include "zstd_ddict.h"
|
||||
@@ -135,7 +135,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
|
||||
ZSTD_memcpy(internalBuffer, dict, dictSize);
|
||||
}
|
||||
ddict->dictSize = dictSize;
|
||||
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
||||
ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */
|
||||
|
||||
/* parse dictionary content */
|
||||
FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
|
||||
@@ -241,7 +241,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
|
||||
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
|
||||
{
|
||||
if (ddict==NULL) return 0;
|
||||
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
|
||||
return ddict->dictID;
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/zstd_ddict.h
generated
vendored
2
vendor/github.com/DataDog/zstd/zstd_ddict.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
408
vendor/github.com/DataDog/zstd/zstd_decompress.c
generated
vendored
408
vendor/github.com/DataDog/zstd/zstd_decompress.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -57,13 +57,15 @@
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
|
||||
#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h" /* blockProperties_t */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#include "bits.h" /* ZSTD_highbit32 */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
|
||||
#include "zstd_internal.h" /* blockProperties_t */
|
||||
#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
|
||||
#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
|
||||
#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */
|
||||
@@ -79,11 +81,11 @@
|
||||
*************************************/
|
||||
|
||||
#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
|
||||
#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
|
||||
* Currently, that means a 0.75 load factor.
|
||||
* So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
|
||||
* the load factor of the ddict hash set.
|
||||
*/
|
||||
#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
|
||||
* Currently, that means a 0.75 load factor.
|
||||
* So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
|
||||
* the load factor of the ddict hash set.
|
||||
*/
|
||||
|
||||
#define DDICT_HASHSET_TABLE_BASE_SIZE 64
|
||||
#define DDICT_HASHSET_RESIZE_FACTOR 2
|
||||
@@ -244,6 +246,8 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
|
||||
dctx->outBufferMode = ZSTD_bm_buffered;
|
||||
dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
|
||||
dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
|
||||
dctx->disableHufAsm = 0;
|
||||
dctx->maxBlockSizeParam = 0;
|
||||
}
|
||||
|
||||
static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
|
||||
@@ -264,6 +268,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
|
||||
#endif
|
||||
dctx->noForwardProgress = 0;
|
||||
dctx->oversizedDuration = 0;
|
||||
dctx->isFrameDecompression = 1;
|
||||
#if DYNAMIC_BMI2
|
||||
dctx->bmi2 = ZSTD_cpuSupportsBmi2();
|
||||
#endif
|
||||
@@ -439,16 +444,40 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
||||
* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
|
||||
* @return : 0, `zfhPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
|
||||
** or an error code, which can be tested using ZSTD_isError() */
|
||||
size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t const minInputSize = ZSTD_startingInputLength(format);
|
||||
|
||||
ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
|
||||
if (srcSize < minInputSize) return minInputSize;
|
||||
RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter");
|
||||
DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize);
|
||||
|
||||
if (srcSize > 0) {
|
||||
/* note : technically could be considered an assert(), since it's an invalid entry */
|
||||
RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0");
|
||||
}
|
||||
if (srcSize < minInputSize) {
|
||||
if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) {
|
||||
/* when receiving less than @minInputSize bytes,
|
||||
* control these bytes at least correspond to a supported magic number
|
||||
* in order to error out early if they don't.
|
||||
**/
|
||||
size_t const toCopy = MIN(4, srcSize);
|
||||
unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER);
|
||||
assert(src != NULL);
|
||||
ZSTD_memcpy(hbuf, src, toCopy);
|
||||
if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) {
|
||||
/* not a zstd frame : let's check if it's a skippable frame */
|
||||
MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
ZSTD_memcpy(hbuf, src, toCopy);
|
||||
if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
RETURN_ERROR(prefix_unknown,
|
||||
"first bytes don't correspond to any supported magic number");
|
||||
} } }
|
||||
return minInputSize;
|
||||
}
|
||||
|
||||
ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */
|
||||
if ( (format != ZSTD_f_zstd1_magicless)
|
||||
&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
|
||||
if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
@@ -456,8 +485,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
||||
if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
|
||||
return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
|
||||
ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
|
||||
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
|
||||
zfhPtr->frameType = ZSTD_skippableFrame;
|
||||
zfhPtr->dictID = MEM_readLE32(src) - ZSTD_MAGIC_SKIPPABLE_START;
|
||||
zfhPtr->headerSize = ZSTD_SKIPPABLEHEADERSIZE;
|
||||
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
|
||||
return 0;
|
||||
}
|
||||
RETURN_ERROR(prefix_unknown, "");
|
||||
@@ -526,7 +557,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
||||
* @return : 0, `zfhPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
|
||||
size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
|
||||
}
|
||||
@@ -544,7 +575,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
|
||||
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
|
||||
}
|
||||
#endif
|
||||
{ ZSTD_frameHeader zfh;
|
||||
{ ZSTD_FrameHeader zfh;
|
||||
if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
if (zfh.frameType == ZSTD_skippableFrame) {
|
||||
@@ -564,49 +595,52 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
|
||||
sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
|
||||
RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
|
||||
frameParameter_unsupported, "");
|
||||
{
|
||||
size_t const skippableSize = skippableHeaderSize + sizeU32;
|
||||
{ size_t const skippableSize = skippableHeaderSize + sizeU32;
|
||||
RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
|
||||
return skippableSize;
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTD_readSkippableFrame() :
|
||||
* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
|
||||
* Retrieves content of a skippable frame, and writes it to dst buffer.
|
||||
*
|
||||
* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
|
||||
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
|
||||
* in the magicVariant.
|
||||
*
|
||||
* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
|
||||
* Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame.
|
||||
*
|
||||
* @return : number of bytes written or a ZSTD error.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
|
||||
const void* src, size_t srcSize)
|
||||
size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
|
||||
unsigned* magicVariant, /* optional, can be NULL */
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 const magicNumber = MEM_readLE32(src);
|
||||
size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
|
||||
size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
|
||||
RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
|
||||
|
||||
/* check input validity */
|
||||
RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
|
||||
RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
|
||||
RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
|
||||
{ U32 const magicNumber = MEM_readLE32(src);
|
||||
size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
|
||||
size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
|
||||
|
||||
/* deliver payload */
|
||||
if (skippableContentSize > 0 && dst != NULL)
|
||||
ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
|
||||
if (magicVariant != NULL)
|
||||
*magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
|
||||
return skippableContentSize;
|
||||
/* check input validity */
|
||||
RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
|
||||
RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
|
||||
RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
|
||||
|
||||
/* deliver payload */
|
||||
if (skippableContentSize > 0 && dst != NULL)
|
||||
ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
|
||||
if (magicVariant != NULL)
|
||||
*magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
|
||||
return skippableContentSize;
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_findDecompressedSize() :
|
||||
* compatible with legacy mode
|
||||
* `srcSize` must be the exact length of some number of ZSTD compressed and/or
|
||||
* skippable frames
|
||||
* @return : decompressed size of the frames contained */
|
||||
* note: compatible with legacy mode
|
||||
* @return : decompressed size of the frames contained */
|
||||
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
unsigned long long totalDstSize = 0;
|
||||
@@ -616,9 +650,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
||||
|
||||
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
size_t const skippableSize = readSkippableFrameSize(src, srcSize);
|
||||
if (ZSTD_isError(skippableSize)) {
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR;
|
||||
assert(skippableSize <= srcSize);
|
||||
|
||||
src = (const BYTE *)src + skippableSize;
|
||||
@@ -626,17 +658,17 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
||||
continue;
|
||||
}
|
||||
|
||||
{ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
|
||||
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
|
||||
{ unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize);
|
||||
if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs;
|
||||
|
||||
/* check for overflow */
|
||||
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
|
||||
totalDstSize += ret;
|
||||
if (totalDstSize + fcs < totalDstSize)
|
||||
return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */
|
||||
totalDstSize += fcs;
|
||||
}
|
||||
/* skip to next frame */
|
||||
{ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
|
||||
if (ZSTD_isError(frameSrcSize)) {
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR;
|
||||
assert(frameSrcSize <= srcSize);
|
||||
|
||||
src = (const BYTE *)src + frameSrcSize;
|
||||
srcSize -= frameSrcSize;
|
||||
@@ -700,17 +732,17 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
|
||||
return frameSizeInfo;
|
||||
}
|
||||
|
||||
static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
|
||||
static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format)
|
||||
{
|
||||
ZSTD_frameSizeInfo frameSizeInfo;
|
||||
ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
||||
if (ZSTD_isLegacy(src, srcSize))
|
||||
if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize))
|
||||
return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
|
||||
#endif
|
||||
|
||||
if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
|
||||
if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
|
||||
&& (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
|
||||
assert(ZSTD_isError(frameSizeInfo.compressedSize) ||
|
||||
@@ -721,10 +753,10 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
|
||||
const BYTE* const ipstart = ip;
|
||||
size_t remainingSize = srcSize;
|
||||
size_t nbBlocks = 0;
|
||||
ZSTD_frameHeader zfh;
|
||||
ZSTD_FrameHeader zfh;
|
||||
|
||||
/* Extract Frame Header */
|
||||
{ size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
|
||||
{ size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format);
|
||||
if (ZSTD_isError(ret))
|
||||
return ZSTD_errorFrameSizeInfo(ret);
|
||||
if (ret > 0)
|
||||
@@ -758,28 +790,31 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
|
||||
ip += 4;
|
||||
}
|
||||
|
||||
frameSizeInfo.nbBlocks = nbBlocks;
|
||||
frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
|
||||
frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
|
||||
? zfh.frameContentSize
|
||||
: nbBlocks * zfh.blockSizeMax;
|
||||
: (unsigned long long)nbBlocks * zfh.blockSizeMax;
|
||||
return frameSizeInfo;
|
||||
}
|
||||
}
|
||||
|
||||
static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) {
|
||||
ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format);
|
||||
return frameSizeInfo.compressedSize;
|
||||
}
|
||||
|
||||
/** ZSTD_findFrameCompressedSize() :
|
||||
* compatible with legacy mode
|
||||
* `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
|
||||
* `srcSize` must be at least as large as the frame contained
|
||||
* @return : the compressed size of the frame starting at `src` */
|
||||
* See docs in zstd.h
|
||||
* Note: compatible with legacy mode */
|
||||
size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
||||
{
|
||||
ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
|
||||
return frameSizeInfo.compressedSize;
|
||||
return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1);
|
||||
}
|
||||
|
||||
/** ZSTD_decompressBound() :
|
||||
* compatible with legacy mode
|
||||
* `src` must point to the start of a ZSTD frame or a skippeable frame
|
||||
* `src` must point to the start of a ZSTD frame or a skippable frame
|
||||
* `srcSize` must be at least as large as the frame contained
|
||||
* @return : the maximum decompressed size of the compressed source
|
||||
*/
|
||||
@@ -788,7 +823,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
|
||||
unsigned long long bound = 0;
|
||||
/* Iterate over each frame */
|
||||
while (srcSize > 0) {
|
||||
ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
|
||||
ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1);
|
||||
size_t const compressedSize = frameSizeInfo.compressedSize;
|
||||
unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
|
||||
if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
|
||||
@@ -801,6 +836,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
|
||||
return bound;
|
||||
}
|
||||
|
||||
size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
|
||||
{
|
||||
size_t margin = 0;
|
||||
unsigned maxBlockSize = 0;
|
||||
|
||||
/* Iterate over each frame */
|
||||
while (srcSize > 0) {
|
||||
ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1);
|
||||
size_t const compressedSize = frameSizeInfo.compressedSize;
|
||||
unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
|
||||
ZSTD_FrameHeader zfh;
|
||||
|
||||
FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
|
||||
if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
|
||||
return ERROR(corruption_detected);
|
||||
|
||||
if (zfh.frameType == ZSTD_frame) {
|
||||
/* Add the frame header to our margin */
|
||||
margin += zfh.headerSize;
|
||||
/* Add the checksum to our margin */
|
||||
margin += zfh.checksumFlag ? 4 : 0;
|
||||
/* Add 3 bytes per block */
|
||||
margin += 3 * frameSizeInfo.nbBlocks;
|
||||
|
||||
/* Compute the max block size */
|
||||
maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax);
|
||||
} else {
|
||||
assert(zfh.frameType == ZSTD_skippableFrame);
|
||||
/* Add the entire skippable frame size to our margin. */
|
||||
margin += compressedSize;
|
||||
}
|
||||
|
||||
assert(srcSize >= compressedSize);
|
||||
src = (const BYTE*)src + compressedSize;
|
||||
srcSize -= compressedSize;
|
||||
}
|
||||
|
||||
/* Add the max block size back to the margin. */
|
||||
margin += maxBlockSize;
|
||||
|
||||
return margin;
|
||||
}
|
||||
|
||||
/*-*************************************************************
|
||||
* Frame decoding
|
||||
@@ -826,7 +903,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
|
||||
if (srcSize == 0) return 0;
|
||||
RETURN_ERROR(dstBuffer_null, "");
|
||||
}
|
||||
ZSTD_memcpy(dst, src, srcSize);
|
||||
ZSTD_memmove(dst, src, srcSize);
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
@@ -843,7 +920,7 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
|
||||
return regenSize;
|
||||
}
|
||||
|
||||
static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
|
||||
static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, int streaming)
|
||||
{
|
||||
#if ZSTD_TRACE
|
||||
if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
|
||||
@@ -902,8 +979,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
|
||||
}
|
||||
|
||||
/* Shrink the blockSizeMax if enabled */
|
||||
if (dctx->maxBlockSizeParam != 0)
|
||||
dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam);
|
||||
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
BYTE* oBlockEnd = oend;
|
||||
size_t decodedSize;
|
||||
blockProperties_t blockProperties;
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
|
||||
@@ -913,27 +995,48 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
remainingSrcSize -= ZSTD_blockHeaderSize;
|
||||
RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
|
||||
|
||||
if (ip >= op && ip < oBlockEnd) {
|
||||
/* We are decompressing in-place. Limit the output pointer so that we
|
||||
* don't overwrite the block that we are currently reading. This will
|
||||
* fail decompression if the input & output pointers aren't spaced
|
||||
* far enough apart.
|
||||
*
|
||||
* This is important to set, even when the pointers are far enough
|
||||
* apart, because ZSTD_decompressBlock_internal() can decide to store
|
||||
* literals in the output buffer, after the block it is decompressing.
|
||||
* Since we don't want anything to overwrite our input, we have to tell
|
||||
* ZSTD_decompressBlock_internal to never write past ip.
|
||||
*
|
||||
* See ZSTD_allocateLiteralsBuffer() for reference.
|
||||
*/
|
||||
oBlockEnd = op + (ip - op);
|
||||
}
|
||||
|
||||
switch(blockProperties.blockType)
|
||||
{
|
||||
case bt_compressed:
|
||||
decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
|
||||
assert(dctx->isFrameDecompression == 1);
|
||||
decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming);
|
||||
break;
|
||||
case bt_raw :
|
||||
/* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */
|
||||
decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize);
|
||||
decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize);
|
||||
break;
|
||||
case bt_reserved :
|
||||
default:
|
||||
RETURN_ERROR(corruption_detected, "invalid block type");
|
||||
}
|
||||
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
if (dctx->validateChecksum)
|
||||
FORWARD_IF_ERROR(decodedSize, "Block decompression failure");
|
||||
DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize);
|
||||
if (dctx->validateChecksum) {
|
||||
XXH64_update(&dctx->xxhState, op, decodedSize);
|
||||
if (decodedSize != 0)
|
||||
}
|
||||
if (decodedSize) /* support dst = NULL,0 */ {
|
||||
op += decodedSize;
|
||||
}
|
||||
assert(ip != NULL);
|
||||
ip += cBlockSize;
|
||||
remainingSrcSize -= cBlockSize;
|
||||
@@ -957,12 +1060,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
}
|
||||
ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
|
||||
/* Allow caller to get size read */
|
||||
DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr));
|
||||
*srcPtr = ip;
|
||||
*srcSizePtr = remainingSrcSize;
|
||||
return (size_t)(op-ostart);
|
||||
}
|
||||
|
||||
static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||
static
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
@@ -982,7 +1088,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||
while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) {
|
||||
if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) {
|
||||
size_t decodedSize;
|
||||
size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
||||
if (ZSTD_isError(frameSize)) return frameSize;
|
||||
@@ -992,6 +1098,15 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||
decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
|
||||
{
|
||||
unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize);
|
||||
RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!");
|
||||
if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) {
|
||||
RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected,
|
||||
"Frame header size does not match decoded size!");
|
||||
}
|
||||
}
|
||||
|
||||
assert(decodedSize <= dstCapacity);
|
||||
dst = (BYTE*)dst + decodedSize;
|
||||
dstCapacity -= decodedSize;
|
||||
@@ -1003,17 +1118,18 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||
}
|
||||
#endif
|
||||
|
||||
{ U32 const magicNumber = MEM_readLE32(src);
|
||||
DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
|
||||
(unsigned)magicNumber, ZSTD_MAGICNUMBER);
|
||||
if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) {
|
||||
U32 const magicNumber = MEM_readLE32(src);
|
||||
DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber);
|
||||
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
/* skippable frame detected : skip it */
|
||||
size_t const skippableSize = readSkippableFrameSize(src, srcSize);
|
||||
FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
|
||||
FORWARD_IF_ERROR(skippableSize, "invalid skippable frame");
|
||||
assert(skippableSize <= srcSize);
|
||||
|
||||
src = (const BYTE *)src + skippableSize;
|
||||
srcSize -= skippableSize;
|
||||
continue;
|
||||
continue; /* check next frame */
|
||||
} }
|
||||
|
||||
if (ddict) {
|
||||
@@ -1109,8 +1225,8 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
|
||||
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
|
||||
|
||||
/**
|
||||
* Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed,
|
||||
* we allow taking a partial block as the input. Currently only raw uncompressed blocks can
|
||||
* Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we
|
||||
* allow taking a partial block as the input. Currently only raw uncompressed blocks can
|
||||
* be streamed.
|
||||
*
|
||||
* For blocks that can be streamed, this allows us to reduce the latency until we produce
|
||||
@@ -1229,7 +1345,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
{
|
||||
case bt_compressed:
|
||||
DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
|
||||
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
|
||||
assert(dctx->isFrameDecompression == 1);
|
||||
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming);
|
||||
dctx->expected = 0; /* Streaming not supported */
|
||||
break;
|
||||
case bt_raw :
|
||||
@@ -1298,6 +1415,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
case ZSTDds_decodeSkippableHeader:
|
||||
assert(src != NULL);
|
||||
assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
|
||||
assert(dctx->format != ZSTD_f_zstd1_magicless);
|
||||
ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
|
||||
dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
|
||||
dctx->stage = ZSTDds_skipFrame;
|
||||
@@ -1310,7 +1428,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
|
||||
default:
|
||||
assert(0); /* impossible */
|
||||
RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
|
||||
RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1351,11 +1469,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
||||
/* in minimal huffman, we always use X1 variants */
|
||||
size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
|
||||
dictPtr, dictEnd - dictPtr,
|
||||
workspace, workspaceSize);
|
||||
workspace, workspaceSize, /* flags */ 0);
|
||||
#else
|
||||
size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
|
||||
dictPtr, (size_t)(dictEnd - dictPtr),
|
||||
workspace, workspaceSize);
|
||||
workspace, workspaceSize, /* flags */ 0);
|
||||
#endif
|
||||
RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
|
||||
dictPtr += hSize;
|
||||
@@ -1454,10 +1572,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
||||
dctx->prefixStart = NULL;
|
||||
dctx->virtualStart = NULL;
|
||||
dctx->dictEnd = NULL;
|
||||
dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
||||
dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */
|
||||
dctx->litEntropy = dctx->fseEntropy = 0;
|
||||
dctx->dictID = 0;
|
||||
dctx->bType = bt_reserved;
|
||||
dctx->isFrameDecompression = 1;
|
||||
ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
|
||||
ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
|
||||
dctx->LLTptr = dctx->entropy.LLTable;
|
||||
@@ -1516,7 +1635,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
|
||||
* This could for one of the following reasons :
|
||||
* - The frame does not require a dictionary (most common case).
|
||||
* - The frame was built with dictID intentionally removed.
|
||||
* Needed dictionary is a hidden information.
|
||||
* Needed dictionary is a hidden piece of information.
|
||||
* Note : this use case also happens when using a non-conformant dictionary.
|
||||
* - `srcSize` is too small, and as a result, frame header could not be decoded.
|
||||
* Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
|
||||
@@ -1525,7 +1644,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
|
||||
* ZSTD_getFrameHeader(), which will provide a more precise error code. */
|
||||
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 };
|
||||
ZSTD_FrameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
|
||||
size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
|
||||
if (ZSTD_isError(hError)) return 0;
|
||||
return zfp.dictID;
|
||||
@@ -1632,7 +1751,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
|
||||
size_t ZSTD_initDStream(ZSTD_DStream* zds)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_initDStream");
|
||||
return ZSTD_initDStream_usingDDict(zds, NULL);
|
||||
FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), "");
|
||||
FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), "");
|
||||
return ZSTD_startingInputLength(zds->format);
|
||||
}
|
||||
|
||||
/* ZSTD_initDStream_usingDDict() :
|
||||
@@ -1640,6 +1761,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
|
||||
* this function cannot fail */
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_initDStream_usingDDict");
|
||||
FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
|
||||
FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
|
||||
return ZSTD_startingInputLength(dctx->format);
|
||||
@@ -1650,6 +1772,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
|
||||
* this function cannot fail */
|
||||
size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_resetDStream");
|
||||
FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
|
||||
return ZSTD_startingInputLength(dctx->format);
|
||||
}
|
||||
@@ -1721,6 +1844,15 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
|
||||
bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
|
||||
bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
|
||||
return bounds;
|
||||
case ZSTD_d_disableHuffmanAssembly:
|
||||
bounds.lowerBound = 0;
|
||||
bounds.upperBound = 1;
|
||||
return bounds;
|
||||
case ZSTD_d_maxBlockSize:
|
||||
bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
|
||||
bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
|
||||
return bounds;
|
||||
|
||||
default:;
|
||||
}
|
||||
bounds.error = ERROR(parameter_unsupported);
|
||||
@@ -1761,6 +1893,12 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value
|
||||
case ZSTD_d_refMultipleDDicts:
|
||||
*value = (int)dctx->refMultipleDDicts;
|
||||
return 0;
|
||||
case ZSTD_d_disableHuffmanAssembly:
|
||||
*value = (int)dctx->disableHufAsm;
|
||||
return 0;
|
||||
case ZSTD_d_maxBlockSize:
|
||||
*value = dctx->maxBlockSizeParam;
|
||||
return 0;
|
||||
default:;
|
||||
}
|
||||
RETURN_ERROR(parameter_unsupported, "");
|
||||
@@ -1794,6 +1932,14 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
|
||||
}
|
||||
dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
|
||||
return 0;
|
||||
case ZSTD_d_disableHuffmanAssembly:
|
||||
CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value);
|
||||
dctx->disableHufAsm = value != 0;
|
||||
return 0;
|
||||
case ZSTD_d_maxBlockSize:
|
||||
if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value);
|
||||
dctx->maxBlockSizeParam = value;
|
||||
return 0;
|
||||
default:;
|
||||
}
|
||||
RETURN_ERROR(parameter_unsupported, "");
|
||||
@@ -1805,6 +1951,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
|
||||
|| (reset == ZSTD_reset_session_and_parameters) ) {
|
||||
dctx->streamStage = zdss_init;
|
||||
dctx->noForwardProgress = 0;
|
||||
dctx->isFrameDecompression = 1;
|
||||
}
|
||||
if ( (reset == ZSTD_reset_parameters)
|
||||
|| (reset == ZSTD_reset_session_and_parameters) ) {
|
||||
@@ -1821,11 +1968,17 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
|
||||
return ZSTD_sizeof_DCtx(dctx);
|
||||
}
|
||||
|
||||
size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
|
||||
static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax)
|
||||
{
|
||||
size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
|
||||
/* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
|
||||
unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
|
||||
size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax);
|
||||
/* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block
|
||||
* ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing
|
||||
* the block at the beginning of the output buffer, and maintain a full window.
|
||||
*
|
||||
* We need another blockSize worth of buffer so that we can store split
|
||||
* literals at the end of the block without overwriting the extDict window.
|
||||
*/
|
||||
unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2);
|
||||
unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
|
||||
size_t const minRBSize = (size_t) neededSize;
|
||||
RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
|
||||
@@ -1833,6 +1986,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
return minRBSize;
|
||||
}
|
||||
|
||||
size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
|
||||
{
|
||||
return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX);
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateDStreamSize(size_t windowSize)
|
||||
{
|
||||
size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
|
||||
@@ -1844,7 +2002,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
|
||||
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
||||
{
|
||||
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
|
||||
ZSTD_frameHeader zfh;
|
||||
ZSTD_FrameHeader zfh;
|
||||
size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
|
||||
if (ZSTD_isError(err)) return err;
|
||||
RETURN_ERROR_IF(err>0, srcSize_wrong, "");
|
||||
@@ -1939,6 +2097,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
U32 someMoreWork = 1;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_decompressStream");
|
||||
assert(zds != NULL);
|
||||
RETURN_ERROR_IF(
|
||||
input->pos > input->size,
|
||||
srcSize_wrong,
|
||||
@@ -1981,7 +2140,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
if (zds->refMultipleDDicts && zds->ddictSet) {
|
||||
ZSTD_DCtx_selectFrameDDict(zds);
|
||||
}
|
||||
DEBUGLOG(5, "header size : %u", (U32)hSize);
|
||||
if (ZSTD_isError(hSize)) {
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||
U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
|
||||
@@ -2013,6 +2171,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
zds->lhSize += remainingInput;
|
||||
}
|
||||
input->pos = input->size;
|
||||
/* check first few bytes */
|
||||
FORWARD_IF_ERROR(
|
||||
ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format),
|
||||
"First few bytes detected incorrect" );
|
||||
/* return hint input size */
|
||||
return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
|
||||
}
|
||||
assert(ip != NULL);
|
||||
@@ -2024,14 +2187,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
|
||||
&& zds->fParams.frameType != ZSTD_skippableFrame
|
||||
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
|
||||
size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart));
|
||||
size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format);
|
||||
if (cSize <= (size_t)(iend-istart)) {
|
||||
/* shortcut : using single-pass mode */
|
||||
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
|
||||
if (ZSTD_isError(decompressedSize)) return decompressedSize;
|
||||
DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
|
||||
DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()");
|
||||
assert(istart != NULL);
|
||||
ip = istart + cSize;
|
||||
op += decompressedSize;
|
||||
op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
|
||||
zds->expected = 0;
|
||||
zds->streamStage = zdss_init;
|
||||
someMoreWork = 0;
|
||||
@@ -2050,7 +2214,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
DEBUGLOG(4, "Consume header");
|
||||
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
|
||||
|
||||
if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
||||
if (zds->format == ZSTD_f_zstd1
|
||||
&& (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
||||
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
|
||||
zds->stage = ZSTDds_skipFrame;
|
||||
} else {
|
||||
@@ -2066,11 +2231,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
||||
RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
|
||||
frameParameter_windowTooLarge, "");
|
||||
if (zds->maxBlockSizeParam != 0)
|
||||
zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam);
|
||||
|
||||
/* Adapt buffer sizes to frame header instructions */
|
||||
{ size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
|
||||
size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
|
||||
? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
|
||||
? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax)
|
||||
: 0;
|
||||
|
||||
ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
|
||||
@@ -2115,6 +2282,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
}
|
||||
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
||||
FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
|
||||
assert(ip != NULL);
|
||||
ip += neededInSize;
|
||||
/* Function modifies the stage so we must break */
|
||||
break;
|
||||
@@ -2129,7 +2297,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
int const isSkipFrame = ZSTD_isSkipFrame(zds);
|
||||
size_t loadedSize;
|
||||
/* At this point we shouldn't be decompressing a block that we can stream. */
|
||||
assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
|
||||
assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)));
|
||||
if (isSkipFrame) {
|
||||
loadedSize = MIN(toLoad, (size_t)(iend-ip));
|
||||
} else {
|
||||
@@ -2138,8 +2306,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
"should never happen");
|
||||
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
|
||||
}
|
||||
ip += loadedSize;
|
||||
zds->inPos += loadedSize;
|
||||
if (loadedSize != 0) {
|
||||
/* ip may be NULL */
|
||||
ip += loadedSize;
|
||||
zds->inPos += loadedSize;
|
||||
}
|
||||
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
|
||||
|
||||
/* decode loaded input */
|
||||
@@ -2149,14 +2320,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
break;
|
||||
}
|
||||
case zdss_flush:
|
||||
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
|
||||
{
|
||||
size_t const toFlushSize = zds->outEnd - zds->outStart;
|
||||
size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
|
||||
op = op ? op + flushedSize : op;
|
||||
|
||||
zds->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) { /* flush completed */
|
||||
zds->streamStage = zdss_read;
|
||||
if ( (zds->outBuffSize < zds->fParams.frameContentSize)
|
||||
&& (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
|
||||
&& (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
|
||||
DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
|
||||
(int)(zds->outBuffSize - zds->outStart),
|
||||
(U32)zds->fParams.blockSizeMax);
|
||||
@@ -2170,7 +2344,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
|
||||
default:
|
||||
assert(0); /* impossible */
|
||||
RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
|
||||
RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */
|
||||
} }
|
||||
|
||||
/* result */
|
||||
@@ -2183,8 +2357,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
if ((ip==istart) && (op==ostart)) { /* no forward progress */
|
||||
zds->noForwardProgress ++;
|
||||
if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
|
||||
RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
|
||||
RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
|
||||
RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, "");
|
||||
RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, "");
|
||||
assert(0);
|
||||
}
|
||||
} else {
|
||||
@@ -2221,13 +2395,19 @@ size_t ZSTD_decompressStream_simpleArgs (
|
||||
void* dst, size_t dstCapacity, size_t* dstPos,
|
||||
const void* src, size_t srcSize, size_t* srcPos)
|
||||
{
|
||||
ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
|
||||
ZSTD_inBuffer input = { src, srcSize, *srcPos };
|
||||
/* ZSTD_compress_generic() will check validity of dstPos and srcPos */
|
||||
size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
|
||||
*dstPos = output.pos;
|
||||
*srcPos = input.pos;
|
||||
return cErr;
|
||||
ZSTD_outBuffer output;
|
||||
ZSTD_inBuffer input;
|
||||
output.dst = dst;
|
||||
output.size = dstCapacity;
|
||||
output.pos = *dstPos;
|
||||
input.src = src;
|
||||
input.size = srcSize;
|
||||
input.pos = *srcPos;
|
||||
{ size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
|
||||
*dstPos = output.pos;
|
||||
*srcPos = input.pos;
|
||||
return cErr;
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
721
vendor/github.com/DataDog/zstd/zstd_decompress_block.c
generated
vendored
721
vendor/github.com/DataDog/zstd/zstd_decompress_block.c
generated
vendored
File diff suppressed because it is too large
Load Diff
9
vendor/github.com/DataDog/zstd/zstd_decompress_block.h
generated
vendored
9
vendor/github.com/DataDog/zstd/zstd_decompress_block.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -48,7 +48,7 @@ typedef enum {
|
||||
*/
|
||||
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize, const int frame, const streaming_operation streaming);
|
||||
const void* src, size_t srcSize, const streaming_operation streaming);
|
||||
|
||||
/* ZSTD_buildFSETable() :
|
||||
* generate FSE decoding table for one symbol (ll, ml or off)
|
||||
@@ -65,6 +65,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
|
||||
unsigned tableLog, void* wksp, size_t wkspSize,
|
||||
int bmi2);
|
||||
|
||||
/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */
|
||||
size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
|
||||
#endif /* ZSTD_DEC_BLOCK_H */
|
||||
|
||||
|
||||
18
vendor/github.com/DataDog/zstd/zstd_decompress_internal.h
generated
vendored
18
vendor/github.com/DataDog/zstd/zstd_decompress_internal.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -76,12 +76,13 @@ static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
|
||||
|
||||
#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
|
||||
#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
|
||||
#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
|
||||
|
||||
typedef struct {
|
||||
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
|
||||
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
|
||||
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
|
||||
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
|
||||
HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
|
||||
} ZSTD_entropyDTables_t;
|
||||
@@ -136,7 +137,7 @@ struct ZSTD_DCtx_s
|
||||
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
|
||||
const void* dictEnd; /* end of previous segment */
|
||||
size_t expected;
|
||||
ZSTD_frameHeader fParams;
|
||||
ZSTD_FrameHeader fParams;
|
||||
U64 processedCSize;
|
||||
U64 decodedSize;
|
||||
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
|
||||
@@ -153,7 +154,8 @@ struct ZSTD_DCtx_s
|
||||
size_t litSize;
|
||||
size_t rleSize;
|
||||
size_t staticSize;
|
||||
#if DYNAMIC_BMI2 != 0
|
||||
int isFrameDecompression;
|
||||
#if DYNAMIC_BMI2
|
||||
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
|
||||
#endif
|
||||
|
||||
@@ -165,6 +167,8 @@ struct ZSTD_DCtx_s
|
||||
ZSTD_dictUses_e dictUses;
|
||||
ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */
|
||||
ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
|
||||
int disableHufAsm;
|
||||
int maxBlockSizeParam;
|
||||
|
||||
/* streaming */
|
||||
ZSTD_dStreamStage streamStage;
|
||||
@@ -208,11 +212,11 @@ struct ZSTD_DCtx_s
|
||||
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
|
||||
|
||||
MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
|
||||
#if DYNAMIC_BMI2 != 0
|
||||
return dctx->bmi2;
|
||||
#if DYNAMIC_BMI2
|
||||
return dctx->bmi2;
|
||||
#else
|
||||
(void)dctx;
|
||||
return 0;
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
14
vendor/github.com/DataDog/zstd/zstd_deps.h
generated
vendored
14
vendor/github.com/DataDog/zstd/zstd_deps.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -25,6 +25,18 @@
|
||||
#ifndef ZSTD_DEPS_COMMON
|
||||
#define ZSTD_DEPS_COMMON
|
||||
|
||||
/* Even though we use qsort_r only for the dictionary builder, the macro
|
||||
* _GNU_SOURCE has to be declared *before* the inclusion of any standard
|
||||
* header and the script 'combine.sh' combines the whole zstd source code
|
||||
* in a single file.
|
||||
*/
|
||||
#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
|
||||
defined(__CYGWIN__) || defined(__MSYS__)
|
||||
#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
|
||||
#define _GNU_SOURCE
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include <limits.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
244
vendor/github.com/DataDog/zstd/zstd_double_fast.c
generated
vendored
244
vendor/github.com/DataDog/zstd/zstd_double_fast.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,8 +12,49 @@
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "zstd_double_fast.h"
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
|
||||
|
||||
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
static
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32* const hashLarge = ms->hashTable;
|
||||
U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
||||
U32 const mls = cParams->minMatch;
|
||||
U32* const hashSmall = ms->chainTable;
|
||||
U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* ip = base + ms->nextToUpdate;
|
||||
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
|
||||
const U32 fastHashFillStep = 3;
|
||||
|
||||
/* Always insert every fastHashFillStep position into the hash tables.
|
||||
* Insert the other positions into the large hash table if their entry
|
||||
* is empty.
|
||||
*/
|
||||
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
|
||||
U32 const curr = (U32)(ip - base);
|
||||
U32 i;
|
||||
for (i = 0; i < fastHashFillStep; ++i) {
|
||||
size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls);
|
||||
size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8);
|
||||
if (i == 0) {
|
||||
ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i);
|
||||
}
|
||||
if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) {
|
||||
ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i);
|
||||
}
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
if (dtlm == ZSTD_dtlm_fast)
|
||||
break;
|
||||
} }
|
||||
}
|
||||
|
||||
static
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
@@ -44,13 +85,26 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
if (dtlm == ZSTD_dtlm_fast)
|
||||
break;
|
||||
} }
|
||||
} }
|
||||
}
|
||||
|
||||
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
|
||||
const void* const end,
|
||||
ZSTD_dictTableLoadMethod_e dtlm,
|
||||
ZSTD_tableFillPurpose_e tfp)
|
||||
{
|
||||
if (tfp == ZSTD_tfp_forCDict) {
|
||||
ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm);
|
||||
} else {
|
||||
ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize, U32 const mls /* template */)
|
||||
{
|
||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
||||
@@ -68,7 +122,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
U32 offsetSaved = 0;
|
||||
U32 offsetSaved1 = 0, offsetSaved2 = 0;
|
||||
|
||||
size_t mLength;
|
||||
U32 offset;
|
||||
@@ -89,9 +143,14 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
||||
const BYTE* matchl0; /* the long match for ip */
|
||||
const BYTE* matchs0; /* the short match for ip */
|
||||
const BYTE* matchl1; /* the long match for ip1 */
|
||||
const BYTE* matchs0_safe; /* matchs0 or safe address */
|
||||
|
||||
const BYTE* ip = istart; /* the current position */
|
||||
const BYTE* ip1; /* the next position */
|
||||
/* Array of ~random data, should have low probability of matching data
|
||||
* we load from here instead of from tables, if matchl0/matchl1 are
|
||||
* invalid indices. Used to avoid unpredictable branches. */
|
||||
const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4};
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
|
||||
|
||||
@@ -101,8 +160,8 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
||||
U32 const current = (U32)(ip - base);
|
||||
U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog);
|
||||
U32 const maxRep = current - windowLow;
|
||||
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
|
||||
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
|
||||
if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
|
||||
if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
|
||||
}
|
||||
|
||||
/* Outer Loop: one iteration per match found and stored */
|
||||
@@ -132,30 +191,35 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
||||
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
|
||||
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
|
||||
goto _match_stored;
|
||||
}
|
||||
|
||||
hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
|
||||
|
||||
if (idxl0 > prefixLowestIndex) {
|
||||
/* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch.
|
||||
* However expression below complies into conditional move. Since
|
||||
* match is unlikely and we only *branch* on idxl0 > prefixLowestIndex
|
||||
* if there is a match, all branches become predictable. */
|
||||
{ const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]);
|
||||
|
||||
/* check prefix long match */
|
||||
if (MEM_read64(matchl0) == MEM_read64(ip)) {
|
||||
if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) {
|
||||
mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
|
||||
offset = (U32)(ip-matchl0);
|
||||
while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
}
|
||||
} }
|
||||
|
||||
idxl1 = hashLong[hl1];
|
||||
matchl1 = base + idxl1;
|
||||
|
||||
if (idxs0 > prefixLowestIndex) {
|
||||
/* check prefix short match */
|
||||
if (MEM_read32(matchs0) == MEM_read32(ip)) {
|
||||
goto _search_next_long;
|
||||
}
|
||||
/* Same optimization as matchl0 above */
|
||||
matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]);
|
||||
|
||||
/* check prefix short match */
|
||||
if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) {
|
||||
goto _search_next_long;
|
||||
}
|
||||
|
||||
if (ip1 >= nextStep) {
|
||||
@@ -176,30 +240,36 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
||||
} while (ip1 <= ilimit);
|
||||
|
||||
_cleanup:
|
||||
/* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
|
||||
* rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
|
||||
offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
|
||||
|
||||
/* save reps for next block */
|
||||
rep[0] = offset_1 ? offset_1 : offsetSaved;
|
||||
rep[1] = offset_2 ? offset_2 : offsetSaved;
|
||||
rep[0] = offset_1 ? offset_1 : offsetSaved1;
|
||||
rep[1] = offset_2 ? offset_2 : offsetSaved2;
|
||||
|
||||
/* Return the last literals size */
|
||||
return (size_t)(iend - anchor);
|
||||
|
||||
_search_next_long:
|
||||
|
||||
/* check prefix long +1 match */
|
||||
if (idxl1 > prefixLowestIndex) {
|
||||
if (MEM_read64(matchl1) == MEM_read64(ip1)) {
|
||||
/* short match found: let's check for a longer one */
|
||||
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
|
||||
offset = (U32)(ip - matchs0);
|
||||
|
||||
/* check long match at +1 position */
|
||||
if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) {
|
||||
size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8;
|
||||
if (l1len > mLength) {
|
||||
/* use the long match instead */
|
||||
ip = ip1;
|
||||
mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8;
|
||||
mLength = l1len;
|
||||
offset = (U32)(ip-matchl1);
|
||||
while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
matchs0 = matchl1;
|
||||
}
|
||||
}
|
||||
|
||||
/* if no long +1 match, explore the short match we found */
|
||||
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
|
||||
offset = (U32)(ip - matchs0);
|
||||
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */
|
||||
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */
|
||||
|
||||
/* fall-through */
|
||||
|
||||
@@ -218,7 +288,7 @@ _match_found: /* requires ip, offset, mLength */
|
||||
hashLong[hl1] = (U32)(ip1 - base);
|
||||
}
|
||||
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
||||
|
||||
_match_stored:
|
||||
/* match found */
|
||||
@@ -244,7 +314,7 @@ _match_stored:
|
||||
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, rLength);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
|
||||
ip += rLength;
|
||||
anchor = ip;
|
||||
continue; /* faster when present ... (?) */
|
||||
@@ -255,8 +325,9 @@ _match_stored:
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize,
|
||||
U32 const mls /* template */)
|
||||
{
|
||||
@@ -276,9 +347,8 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
||||
U32 offset_1=rep[0], offset_2=rep[1];
|
||||
U32 offsetSaved = 0;
|
||||
|
||||
const ZSTD_matchState_t* const dms = ms->dictMatchState;
|
||||
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
|
||||
const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
|
||||
const U32* const dictHashLong = dms->hashTable;
|
||||
const U32* const dictHashSmall = dms->chainTable;
|
||||
@@ -287,8 +357,8 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
const BYTE* const dictStart = dictBase + dictStartIndex;
|
||||
const BYTE* const dictEnd = dms->window.nextSrc;
|
||||
const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase);
|
||||
const U32 dictHBitsL = dictCParams->hashLog;
|
||||
const U32 dictHBitsS = dictCParams->chainLog;
|
||||
const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
||||
const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
||||
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic");
|
||||
@@ -296,6 +366,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
/* if a dictionary is attached, it must be within window range */
|
||||
assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex);
|
||||
|
||||
if (ms->prefetchCDictTables) {
|
||||
size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
|
||||
size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32);
|
||||
PREFETCH_AREA(dictHashLong, hashTableBytes);
|
||||
PREFETCH_AREA(dictHashSmall, chainTableBytes);
|
||||
}
|
||||
|
||||
/* init */
|
||||
ip += (dictAndPrefixLength == 0);
|
||||
|
||||
@@ -310,8 +387,12 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
U32 offset;
|
||||
size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
|
||||
size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
|
||||
size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8);
|
||||
size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls);
|
||||
size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8);
|
||||
size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls);
|
||||
U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS];
|
||||
U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS];
|
||||
int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL);
|
||||
int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS);
|
||||
U32 const curr = (U32)(ip-base);
|
||||
U32 const matchIndexL = hashLong[h2];
|
||||
U32 matchIndexS = hashSmall[h];
|
||||
@@ -324,26 +405,24 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
|
||||
|
||||
/* check repcode */
|
||||
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
|
||||
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
|
||||
goto _match_stored;
|
||||
}
|
||||
|
||||
if (matchIndexL > prefixLowestIndex) {
|
||||
if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
|
||||
/* check prefix long match */
|
||||
if (MEM_read64(matchLong) == MEM_read64(ip)) {
|
||||
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
|
||||
offset = (U32)(ip-matchLong);
|
||||
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
} else {
|
||||
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
|
||||
offset = (U32)(ip-matchLong);
|
||||
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
} else if (dictTagsMatchL) {
|
||||
/* check dictMatchState long match */
|
||||
U32 const dictMatchIndexL = dictHashLong[dictHL];
|
||||
U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
|
||||
const BYTE* dictMatchL = dictBase + dictMatchIndexL;
|
||||
assert(dictMatchL < dictEnd);
|
||||
|
||||
@@ -355,13 +434,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
} }
|
||||
|
||||
if (matchIndexS > prefixLowestIndex) {
|
||||
/* check prefix short match */
|
||||
/* short match candidate */
|
||||
if (MEM_read32(match) == MEM_read32(ip)) {
|
||||
goto _search_next_long;
|
||||
}
|
||||
} else {
|
||||
} else if (dictTagsMatchS) {
|
||||
/* check dictMatchState short match */
|
||||
U32 const dictMatchIndexS = dictHashSmall[dictHS];
|
||||
U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS;
|
||||
match = dictBase + dictMatchIndexS;
|
||||
matchIndexS = dictMatchIndexS + dictIndexDelta;
|
||||
|
||||
@@ -376,25 +455,24 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
||||
continue;
|
||||
|
||||
_search_next_long:
|
||||
|
||||
{ size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
|
||||
size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
|
||||
size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
|
||||
U32 const matchIndexL3 = hashLong[hl3];
|
||||
U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS];
|
||||
int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3);
|
||||
const BYTE* matchL3 = base + matchIndexL3;
|
||||
hashLong[hl3] = curr + 1;
|
||||
|
||||
/* check prefix long +1 match */
|
||||
if (matchIndexL3 > prefixLowestIndex) {
|
||||
if (MEM_read64(matchL3) == MEM_read64(ip+1)) {
|
||||
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
|
||||
ip++;
|
||||
offset = (U32)(ip-matchL3);
|
||||
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
}
|
||||
} else {
|
||||
if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) {
|
||||
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
|
||||
ip++;
|
||||
offset = (U32)(ip-matchL3);
|
||||
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
|
||||
goto _match_found;
|
||||
} else if (dictTagsMatchL3) {
|
||||
/* check dict long +1 match */
|
||||
U32 const dictMatchIndexL3 = dictHashLong[dictHLNext];
|
||||
U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
|
||||
const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
|
||||
assert(dictMatchL3 < dictEnd);
|
||||
if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
|
||||
@@ -420,7 +498,7 @@ _match_found:
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
||||
|
||||
_match_stored:
|
||||
/* match found */
|
||||
@@ -444,12 +522,12 @@ _match_stored:
|
||||
const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
|
||||
dictBase + repIndex2 - dictIndexDelta :
|
||||
base + repIndex2;
|
||||
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
|
||||
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2))
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
||||
ip += repLength2;
|
||||
@@ -462,8 +540,8 @@ _match_stored:
|
||||
} /* while (ip < ilimit) */
|
||||
|
||||
/* save reps for next block */
|
||||
rep[0] = offset_1 ? offset_1 : offsetSaved;
|
||||
rep[1] = offset_2 ? offset_2 : offsetSaved;
|
||||
rep[0] = offset_1;
|
||||
rep[1] = offset_2;
|
||||
|
||||
/* Return the last literals size */
|
||||
return (size_t)(iend - anchor);
|
||||
@@ -471,7 +549,7 @@ _match_stored:
|
||||
|
||||
#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
|
||||
static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
|
||||
void const* src, size_t srcSize) \
|
||||
{ \
|
||||
return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
|
||||
@@ -489,7 +567,7 @@ ZSTD_GEN_DFAST_FN(dictMatchState, 7)
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
const U32 mls = ms->cParams.minMatch;
|
||||
@@ -509,7 +587,7 @@ size_t ZSTD_compressBlock_doubleFast(
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
const U32 mls = ms->cParams.minMatch;
|
||||
@@ -528,8 +606,10 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
static
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize,
|
||||
U32 const mls /* template */)
|
||||
{
|
||||
@@ -580,13 +660,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
size_t mLength;
|
||||
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
|
||||
|
||||
if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
|
||||
if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
|
||||
& (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
|
||||
} else {
|
||||
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
|
||||
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
|
||||
@@ -597,7 +677,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
||||
|
||||
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
|
||||
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
|
||||
@@ -622,7 +702,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
}
|
||||
offset_2 = offset_1;
|
||||
offset_1 = offset;
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
|
||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
||||
|
||||
} else {
|
||||
ip += ((ip-anchor) >> kSearchStrength) + 1;
|
||||
@@ -648,13 +728,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
|
||||
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
|
||||
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
|
||||
& (offset_2 <= current2 - dictStartIndex))
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
||||
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
|
||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
|
||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
||||
ip += repLength2;
|
||||
@@ -678,7 +758,7 @@ ZSTD_GEN_DFAST_FN(extDict, 6)
|
||||
ZSTD_GEN_DFAST_FN(extDict, 7)
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const mls = ms->cParams.minMatch;
|
||||
@@ -696,4 +776,6 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
32
vendor/github.com/DataDog/zstd/zstd_double_fast.h
generated
vendored
32
vendor/github.com/DataDog/zstd/zstd_double_fast.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,29 +12,33 @@
|
||||
#ifndef ZSTD_DOUBLE_FAST_H
|
||||
#define ZSTD_DOUBLE_FAST_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm);
|
||||
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
|
||||
|
||||
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm,
|
||||
ZSTD_tableFillPurpose_e tfp);
|
||||
|
||||
size_t ZSTD_compressBlock_doubleFast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast
|
||||
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE ZSTD_compressBlock_doubleFast_dictMatchState
|
||||
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT ZSTD_compressBlock_doubleFast_extDict
|
||||
#else
|
||||
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST NULL
|
||||
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE NULL
|
||||
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL
|
||||
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
|
||||
|
||||
#endif /* ZSTD_DOUBLE_FAST_H */
|
||||
|
||||
|
||||
44
vendor/github.com/DataDog/zstd/zstd_errors.h
generated
vendored
44
vendor/github.com/DataDog/zstd/zstd_errors.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -16,24 +16,32 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*===== dependency =====*/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
|
||||
#ifndef ZSTDERRORLIB_VISIBILITY
|
||||
# if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
#ifndef ZSTDERRORLIB_VISIBLE
|
||||
/* Backwards compatibility with old macro name */
|
||||
# ifdef ZSTDERRORLIB_VISIBILITY
|
||||
# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
|
||||
# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
|
||||
# else
|
||||
# define ZSTDERRORLIB_VISIBILITY
|
||||
# define ZSTDERRORLIB_VISIBLE
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifndef ZSTDERRORLIB_HIDDEN
|
||||
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
|
||||
# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
|
||||
# else
|
||||
# define ZSTDERRORLIB_HIDDEN
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
|
||||
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
|
||||
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
||||
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
|
||||
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
|
||||
#endif
|
||||
|
||||
/*-*********************************************
|
||||
@@ -59,14 +67,18 @@ typedef enum {
|
||||
ZSTD_error_frameParameter_windowTooLarge = 16,
|
||||
ZSTD_error_corruption_detected = 20,
|
||||
ZSTD_error_checksum_wrong = 22,
|
||||
ZSTD_error_literals_headerWrong = 24,
|
||||
ZSTD_error_dictionary_corrupted = 30,
|
||||
ZSTD_error_dictionary_wrong = 32,
|
||||
ZSTD_error_dictionaryCreation_failed = 34,
|
||||
ZSTD_error_parameter_unsupported = 40,
|
||||
ZSTD_error_parameter_combination_unsupported = 41,
|
||||
ZSTD_error_parameter_outOfBound = 42,
|
||||
ZSTD_error_tableLog_tooLarge = 44,
|
||||
ZSTD_error_maxSymbolValue_tooLarge = 46,
|
||||
ZSTD_error_maxSymbolValue_tooSmall = 48,
|
||||
ZSTD_error_cannotProduce_uncompressedBlock = 49,
|
||||
ZSTD_error_stabilityCondition_notRespected = 50,
|
||||
ZSTD_error_stage_wrong = 60,
|
||||
ZSTD_error_init_missing = 62,
|
||||
ZSTD_error_memory_allocation = 64,
|
||||
@@ -74,18 +86,18 @@ typedef enum {
|
||||
ZSTD_error_dstSize_tooSmall = 70,
|
||||
ZSTD_error_srcSize_wrong = 72,
|
||||
ZSTD_error_dstBuffer_null = 74,
|
||||
ZSTD_error_noForwardProgress_destFull = 80,
|
||||
ZSTD_error_noForwardProgress_inputEmpty = 82,
|
||||
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
|
||||
ZSTD_error_frameIndex_tooLarge = 100,
|
||||
ZSTD_error_seekableIO = 102,
|
||||
ZSTD_error_dstBuffer_wrong = 104,
|
||||
ZSTD_error_srcBuffer_wrong = 105,
|
||||
ZSTD_error_sequenceProducer_failed = 106,
|
||||
ZSTD_error_externalSequences_invalid = 107,
|
||||
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
|
||||
} ZSTD_ErrorCode;
|
||||
|
||||
/*! ZSTD_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
||||
which can be used to compare with enum list published above */
|
||||
ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
|
||||
|
||||
|
||||
|
||||
1136
vendor/github.com/DataDog/zstd/zstd_fast.c
generated
vendored
1136
vendor/github.com/DataDog/zstd/zstd_fast.c
generated
vendored
File diff suppressed because it is too large
Load Diff
21
vendor/github.com/DataDog/zstd/zstd_fast.h
generated
vendored
21
vendor/github.com/DataDog/zstd/zstd_fast.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,29 +12,22 @@
|
||||
#ifndef ZSTD_FAST_H
|
||||
#define ZSTD_FAST_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#include "zstd_compress_internal.h"
|
||||
|
||||
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm);
|
||||
void ZSTD_fillHashTable(ZSTD_MatchState_t* ms,
|
||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm,
|
||||
ZSTD_tableFillPurpose_e tfp);
|
||||
size_t ZSTD_compressBlock_fast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_fast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_fast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_FAST_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
195
vendor/github.com/DataDog/zstd/zstd_internal.h
generated
vendored
195
vendor/github.com/DataDog/zstd/zstd_internal.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -29,7 +29,6 @@
|
||||
#include "zstd.h"
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#ifndef XXH_STATIC_LINKING_ONLY
|
||||
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
@@ -41,10 +40,6 @@
|
||||
# define ZSTD_TRACE 0
|
||||
#endif
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ---- static assert (debug) --- */
|
||||
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
|
||||
#define ZSTD_isError ERR_isError /* for inlining */
|
||||
@@ -94,16 +89,17 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
|
||||
#define ZSTD_FRAMECHECKSUMSIZE 4
|
||||
|
||||
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */
|
||||
#define MIN_LITERALS_FOR_4_STREAMS 6
|
||||
|
||||
#define HufLog 12
|
||||
typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
|
||||
typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e;
|
||||
|
||||
#define LONGNBSEQ 0x7F00
|
||||
|
||||
#define MINMATCH 3
|
||||
|
||||
#define Litbits 8
|
||||
#define LitHufLog 11
|
||||
#define MaxLit ((1<<Litbits) - 1)
|
||||
#define MaxML 52
|
||||
#define MaxLL 35
|
||||
@@ -114,6 +110,8 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
|
||||
#define LLFSELog 9
|
||||
#define OffFSELog 8
|
||||
#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
|
||||
#define MaxMLBits 16
|
||||
#define MaxLLBits 16
|
||||
|
||||
#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
|
||||
/* Each table cannot take more than #symbols * FSELog bits */
|
||||
@@ -177,7 +175,7 @@ static void ZSTD_copy8(void* dst, const void* src) {
|
||||
ZSTD_memcpy(dst, src, 8);
|
||||
#endif
|
||||
}
|
||||
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
||||
#define COPY8(d,s) do { ZSTD_copy8(d,s); d+=8; s+=8; } while (0)
|
||||
|
||||
/* Need to use memmove here since the literal buffer can now be located within
|
||||
the dst buffer. In circumstances where the op "catches up" to where the
|
||||
@@ -197,7 +195,7 @@ static void ZSTD_copy16(void* dst, const void* src) {
|
||||
ZSTD_memcpy(dst, copy16_buf, 16);
|
||||
#endif
|
||||
}
|
||||
#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
|
||||
#define COPY16(d,s) do { ZSTD_copy16(d,s); d+=16; s+=16; } while (0)
|
||||
|
||||
#define WILDCOPY_OVERLENGTH 32
|
||||
#define WILDCOPY_VECLEN 16
|
||||
@@ -226,7 +224,7 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
|
||||
if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
|
||||
/* Handle short offset copies. */
|
||||
do {
|
||||
COPY8(op, ip)
|
||||
COPY8(op, ip);
|
||||
} while (op < oend);
|
||||
} else {
|
||||
assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
|
||||
@@ -236,12 +234,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
|
||||
* one COPY16() in the first call. Then, do two calls per loop since
|
||||
* at that point it is more likely to have a high trip count.
|
||||
*/
|
||||
#ifdef __aarch64__
|
||||
do {
|
||||
COPY16(op, ip);
|
||||
}
|
||||
while (op < oend);
|
||||
#else
|
||||
ZSTD_copy16(op, ip);
|
||||
if (16 >= length) return;
|
||||
op += 16;
|
||||
@@ -251,7 +243,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
|
||||
COPY16(op, ip);
|
||||
}
|
||||
while (op < oend);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,62 +275,6 @@ typedef enum {
|
||||
/*-*******************************************
|
||||
* Private declarations
|
||||
*********************************************/
|
||||
typedef struct seqDef_s {
|
||||
U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
|
||||
U16 litLength;
|
||||
U16 mlBase; /* mlBase == matchLength - MINMATCH */
|
||||
} seqDef;
|
||||
|
||||
/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
|
||||
typedef enum {
|
||||
ZSTD_llt_none = 0, /* no longLengthType */
|
||||
ZSTD_llt_literalLength = 1, /* represents a long literal */
|
||||
ZSTD_llt_matchLength = 2 /* represents a long match */
|
||||
} ZSTD_longLengthType_e;
|
||||
|
||||
typedef struct {
|
||||
seqDef* sequencesStart;
|
||||
seqDef* sequences; /* ptr to end of sequences */
|
||||
BYTE* litStart;
|
||||
BYTE* lit; /* ptr to end of literals */
|
||||
BYTE* llCode;
|
||||
BYTE* mlCode;
|
||||
BYTE* ofCode;
|
||||
size_t maxNbSeq;
|
||||
size_t maxNbLit;
|
||||
|
||||
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
|
||||
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
|
||||
* the existing value of the litLength or matchLength by 0x10000.
|
||||
*/
|
||||
ZSTD_longLengthType_e longLengthType;
|
||||
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
|
||||
} seqStore_t;
|
||||
|
||||
typedef struct {
|
||||
U32 litLength;
|
||||
U32 matchLength;
|
||||
} ZSTD_sequenceLength;
|
||||
|
||||
/**
|
||||
* Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
|
||||
* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
|
||||
*/
|
||||
MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
|
||||
{
|
||||
ZSTD_sequenceLength seqLen;
|
||||
seqLen.litLength = seq->litLength;
|
||||
seqLen.matchLength = seq->mlBase + MINMATCH;
|
||||
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
|
||||
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
|
||||
seqLen.litLength += 0xFFFF;
|
||||
}
|
||||
if (seqStore->longLengthType == ZSTD_llt_matchLength) {
|
||||
seqLen.matchLength += 0xFFFF;
|
||||
}
|
||||
}
|
||||
return seqLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Contains the compressed frame size and an upper-bound for the decompressed frame size.
|
||||
@@ -348,111 +283,11 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
|
||||
* `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
|
||||
*/
|
||||
typedef struct {
|
||||
size_t nbBlocks;
|
||||
size_t compressedSize;
|
||||
unsigned long long decompressedBound;
|
||||
} ZSTD_frameSizeInfo; /* decompress & legacy */
|
||||
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
|
||||
|
||||
/* custom memory allocation functions */
|
||||
void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
|
||||
void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
|
||||
void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
|
||||
|
||||
|
||||
MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
# if STATIC_BMI2 == 1
|
||||
return _lzcnt_u32(val)^31;
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return __builtin_clz (val) ^ 31;
|
||||
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
||||
return 31 - __CLZ(val);
|
||||
# else /* Software version */
|
||||
static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the number of trailing zeros of a `size_t`.
|
||||
* Most compilers should support CTZ as a builtin. A backup
|
||||
* implementation is provided if the builtin isn't supported, but
|
||||
* it may not be terribly efficient.
|
||||
*/
|
||||
MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val)
|
||||
{
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
# if STATIC_BMI2
|
||||
return _tzcnt_u64(val);
|
||||
# else
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward64(&r, (U64)val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# endif
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
return __builtin_ctzll((U64)val);
|
||||
# else
|
||||
static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19,
|
||||
4, 25, 14, 28, 9, 34, 20, 56,
|
||||
5, 17, 26, 54, 15, 41, 29, 43,
|
||||
10, 31, 38, 35, 21, 45, 49, 57,
|
||||
63, 6, 12, 18, 24, 27, 33, 55,
|
||||
16, 53, 40, 42, 30, 37, 44, 48,
|
||||
62, 11, 23, 32, 52, 39, 36, 47,
|
||||
61, 22, 51, 46, 60, 50, 59, 58 };
|
||||
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
if (val != 0) {
|
||||
unsigned long r;
|
||||
_BitScanForward(&r, (U32)val);
|
||||
return (unsigned)r;
|
||||
} else {
|
||||
/* Should not reach this code path */
|
||||
__assume(0);
|
||||
}
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return __builtin_ctz((U32)val);
|
||||
# else
|
||||
static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3,
|
||||
30, 22, 20, 15, 25, 17, 4, 8,
|
||||
31, 27, 13, 23, 21, 19, 16, 7,
|
||||
26, 12, 18, 6, 11, 5, 10, 9 };
|
||||
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ZSTD_invalidateRepCodes() :
|
||||
* ensures next compression will not use repcodes from previous block.
|
||||
* Note : only works with regular variant;
|
||||
@@ -468,13 +303,13 @@ typedef struct {
|
||||
|
||||
/*! ZSTD_getcBlockSize() :
|
||||
* Provides the size of compressed block from block header `src` */
|
||||
/* Used by: decompress, fullbench (does not get its definition from here) */
|
||||
/* Used by: decompress, fullbench */
|
||||
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
||||
blockProperties_t* bpPtr);
|
||||
|
||||
/*! ZSTD_decodeSeqHeaders() :
|
||||
* decode sequence header from src */
|
||||
/* Used by: decompress, fullbench (does not get its definition from here) */
|
||||
/* Used by: zstd_decompress_block, fullbench */
|
||||
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
@@ -487,10 +322,6 @@ MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
|
||||
return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
1165
vendor/github.com/DataDog/zstd/zstd_lazy.c
generated
vendored
1165
vendor/github.com/DataDog/zstd/zstd_lazy.c
generated
vendored
File diff suppressed because it is too large
Load Diff
214
vendor/github.com/DataDog/zstd/zstd_lazy.h
generated
vendored
214
vendor/github.com/DataDog/zstd/zstd_lazy.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,10 +12,6 @@
|
||||
#ifndef ZSTD_LAZY_H
|
||||
#define ZSTD_LAZY_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
|
||||
/**
|
||||
@@ -26,101 +22,173 @@ extern "C" {
|
||||
*/
|
||||
#define ZSTD_LAZY_DDSS_BUCKET_LOG 2
|
||||
|
||||
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
|
||||
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
|
||||
#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */
|
||||
|
||||
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
|
||||
#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
|
||||
|| !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
|
||||
|| !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
|
||||
|| !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR)
|
||||
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip);
|
||||
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip);
|
||||
|
||||
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip);
|
||||
|
||||
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
|
||||
#endif
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_greedy(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_greedy_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_greedy_extDict_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY ZSTD_compressBlock_greedy
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_ROW ZSTD_compressBlock_greedy_row
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE ZSTD_compressBlock_greedy_dictMatchState
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW ZSTD_compressBlock_greedy_dictMatchState_row
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH ZSTD_compressBlock_greedy_dedicatedDictSearch
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_greedy_dedicatedDictSearch_row
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT ZSTD_compressBlock_greedy_extDict
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW ZSTD_compressBlock_greedy_extDict_row
|
||||
#else
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY NULL
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE NULL
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH NULL
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT NULL
|
||||
#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW NULL
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_lazy(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_row(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dictMatchState(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_extDict(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy_extDict_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY ZSTD_compressBlock_lazy
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_ROW ZSTD_compressBlock_lazy_row
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE ZSTD_compressBlock_lazy_dictMatchState
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy_dictMatchState_row
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy_dedicatedDictSearch
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy_dedicatedDictSearch_row
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT ZSTD_compressBlock_lazy_extDict
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW ZSTD_compressBlock_lazy_extDict_row
|
||||
#else
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW NULL
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_lazy2(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_row(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dictMatchState(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_extDict(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_lazy2_extDict_row(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2 ZSTD_compressBlock_lazy2
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_ROW ZSTD_compressBlock_lazy2_row
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE ZSTD_compressBlock_lazy2_dictMatchState
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy2_dictMatchState_row
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy2_dedicatedDictSearch
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy2_dedicatedDictSearch_row
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT ZSTD_compressBlock_lazy2_extDict
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW ZSTD_compressBlock_lazy2_extDict_row
|
||||
#else
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2 NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT NULL
|
||||
#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW NULL
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_btlazy2(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btlazy2_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#define ZSTD_COMPRESSBLOCK_BTLAZY2 ZSTD_compressBlock_btlazy2
|
||||
#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE ZSTD_compressBlock_btlazy2_dictMatchState
|
||||
#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT ZSTD_compressBlock_btlazy2_extDict
|
||||
#else
|
||||
#define ZSTD_COMPRESSBLOCK_BTLAZY2 NULL
|
||||
#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE NULL
|
||||
#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT NULL
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_LAZY_H */
|
||||
|
||||
103
vendor/github.com/DataDog/zstd/zstd_ldm.c
generated
vendored
103
vendor/github.com/DataDog/zstd/zstd_ldm.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -17,7 +17,7 @@
|
||||
#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
|
||||
#include "zstd_ldm_geartab.h"
|
||||
|
||||
#define LDM_BUCKET_SIZE_LOG 3
|
||||
#define LDM_BUCKET_SIZE_LOG 4
|
||||
#define LDM_MIN_MATCH_LENGTH 64
|
||||
#define LDM_HASH_RLOG 7
|
||||
|
||||
@@ -134,21 +134,35 @@ done:
|
||||
}
|
||||
|
||||
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
ZSTD_compressionParameters const* cParams)
|
||||
const ZSTD_compressionParameters* cParams)
|
||||
{
|
||||
params->windowLog = cParams->windowLog;
|
||||
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
|
||||
DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
|
||||
if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
|
||||
if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
|
||||
if (params->hashLog == 0) {
|
||||
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
|
||||
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
|
||||
}
|
||||
if (params->hashRateLog == 0) {
|
||||
params->hashRateLog = params->windowLog < params->hashLog
|
||||
? 0
|
||||
: params->windowLog - params->hashLog;
|
||||
if (params->hashLog > 0) {
|
||||
/* if params->hashLog is set, derive hashRateLog from it */
|
||||
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
|
||||
if (params->windowLog > params->hashLog) {
|
||||
params->hashRateLog = params->windowLog - params->hashLog;
|
||||
}
|
||||
} else {
|
||||
assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9);
|
||||
/* mapping from [fast, rate7] to [btultra2, rate4] */
|
||||
params->hashRateLog = 7 - (cParams->strategy/3);
|
||||
}
|
||||
}
|
||||
if (params->hashLog == 0) {
|
||||
params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX);
|
||||
}
|
||||
if (params->minMatchLength == 0) {
|
||||
params->minMatchLength = LDM_MIN_MATCH_LENGTH;
|
||||
if (cParams->strategy >= ZSTD_btultra)
|
||||
params->minMatchLength /= 2;
|
||||
}
|
||||
if (params->bucketSizeLog==0) {
|
||||
assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9);
|
||||
params->bucketSizeLog = BOUNDED(LDM_BUCKET_SIZE_LOG, (U32)cParams->strategy, ZSTD_LDM_BUCKETSIZELOG_MAX);
|
||||
}
|
||||
params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog);
|
||||
}
|
||||
@@ -171,22 +185,22 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
|
||||
/** ZSTD_ldm_getBucket() :
|
||||
* Returns a pointer to the start of the bucket associated with hash. */
|
||||
static ldmEntry_t* ZSTD_ldm_getBucket(
|
||||
ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams)
|
||||
const ldmState_t* ldmState, size_t hash, U32 const bucketSizeLog)
|
||||
{
|
||||
return ldmState->hashTable + (hash << ldmParams.bucketSizeLog);
|
||||
return ldmState->hashTable + (hash << bucketSizeLog);
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_insertEntry() :
|
||||
* Insert the entry with corresponding hash into the hash table */
|
||||
static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
|
||||
size_t const hash, const ldmEntry_t entry,
|
||||
ldmParams_t const ldmParams)
|
||||
U32 const bucketSizeLog)
|
||||
{
|
||||
BYTE* const pOffset = ldmState->bucketOffsets + hash;
|
||||
unsigned const offset = *pOffset;
|
||||
|
||||
*(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
|
||||
*pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
|
||||
*(ZSTD_ldm_getBucket(ldmState, hash, bucketSizeLog) + offset) = entry;
|
||||
*pOffset = (BYTE)((offset + 1) & ((1u << bucketSizeLog) - 1));
|
||||
|
||||
}
|
||||
|
||||
@@ -235,7 +249,7 @@ static size_t ZSTD_ldm_countBackwardsMatch_2segments(
|
||||
*
|
||||
* The tables for the other strategies are filled within their
|
||||
* block compressors. */
|
||||
static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
|
||||
static size_t ZSTD_ldm_fillFastTables(ZSTD_MatchState_t* ms,
|
||||
void const* end)
|
||||
{
|
||||
const BYTE* const iend = (const BYTE*)end;
|
||||
@@ -243,11 +257,15 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
|
||||
switch(ms->cParams.strategy)
|
||||
{
|
||||
case ZSTD_fast:
|
||||
ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast);
|
||||
ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
|
||||
break;
|
||||
|
||||
case ZSTD_dfast:
|
||||
ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast);
|
||||
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
|
||||
ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
|
||||
#else
|
||||
assert(0); /* shouldn't be called: cparams should've been adjusted. */
|
||||
#endif
|
||||
break;
|
||||
|
||||
case ZSTD_greedy:
|
||||
@@ -270,7 +288,8 @@ void ZSTD_ldm_fillHashTable(
|
||||
const BYTE* iend, ldmParams_t const* params)
|
||||
{
|
||||
U32 const minMatchLength = params->minMatchLength;
|
||||
U32 const hBits = params->hashLog - params->bucketSizeLog;
|
||||
U32 const bucketSizeLog = params->bucketSizeLog;
|
||||
U32 const hBits = params->hashLog - bucketSizeLog;
|
||||
BYTE const* const base = ldmState->window.base;
|
||||
BYTE const* const istart = ip;
|
||||
ldmRollingHashState_t hashState;
|
||||
@@ -285,7 +304,7 @@ void ZSTD_ldm_fillHashTable(
|
||||
unsigned n;
|
||||
|
||||
numSplits = 0;
|
||||
hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
|
||||
hashed = ZSTD_ldm_gear_feed(&hashState, ip, (size_t)(iend - ip), splits, &numSplits);
|
||||
|
||||
for (n = 0; n < numSplits; n++) {
|
||||
if (ip + splits[n] >= istart + minMatchLength) {
|
||||
@@ -296,7 +315,7 @@ void ZSTD_ldm_fillHashTable(
|
||||
|
||||
entry.offset = (U32)(split - base);
|
||||
entry.checksum = (U32)(xxhash >> 32);
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, entry, params->bucketSizeLog);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,7 +329,7 @@ void ZSTD_ldm_fillHashTable(
|
||||
* Sets cctx->nextToUpdate to a position corresponding closer to anchor
|
||||
* if it is far way
|
||||
* (after a long match, only update tables a limited amount). */
|
||||
static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
|
||||
static void ZSTD_ldm_limitTableUpdate(ZSTD_MatchState_t* ms, const BYTE* anchor)
|
||||
{
|
||||
U32 const curr = (U32)(anchor - ms->window.base);
|
||||
if (curr > ms->nextToUpdate + 1024) {
|
||||
@@ -319,8 +338,10 @@ static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
|
||||
}
|
||||
}
|
||||
|
||||
static size_t ZSTD_ldm_generateSequences_internal(
|
||||
ldmState_t* ldmState, rawSeqStore_t* rawSeqStore,
|
||||
static
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
size_t ZSTD_ldm_generateSequences_internal(
|
||||
ldmState_t* ldmState, RawSeqStore_t* rawSeqStore,
|
||||
ldmParams_t const* params, void const* src, size_t srcSize)
|
||||
{
|
||||
/* LDM parameters */
|
||||
@@ -374,7 +395,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
candidates[n].split = split;
|
||||
candidates[n].hash = hash;
|
||||
candidates[n].checksum = (U32)(xxhash >> 32);
|
||||
candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
|
||||
candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, params->bucketSizeLog);
|
||||
PREFETCH_L1(candidates[n].bucket);
|
||||
}
|
||||
|
||||
@@ -397,7 +418,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
* the previous one, we merely register it in the hash table and
|
||||
* move on */
|
||||
if (split < anchor) {
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -444,7 +465,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
/* No match found -- insert an entry into the hash table
|
||||
* and process the next candidate match */
|
||||
if (bestEntry == NULL) {
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -465,7 +486,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
|
||||
/* Insert the current entry into the hash table --- it must be
|
||||
* done after the previous block to avoid clobbering bestEntry */
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
|
||||
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
|
||||
|
||||
anchor = split + forwardMatchLength;
|
||||
|
||||
@@ -504,7 +525,7 @@ static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_generateSequences(
|
||||
ldmState_t* ldmState, rawSeqStore_t* sequences,
|
||||
ldmState_t* ldmState, RawSeqStore_t* sequences,
|
||||
ldmParams_t const* params, void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const maxDist = 1U << params->windowLog;
|
||||
@@ -550,7 +571,7 @@ size_t ZSTD_ldm_generateSequences(
|
||||
* the window through early invalidation.
|
||||
* TODO: * Test the chunk size.
|
||||
* * Try invalidation after the sequence generation and test the
|
||||
* the offset against maxDist directly.
|
||||
* offset against maxDist directly.
|
||||
*
|
||||
* NOTE: Because of dictionaries + sequence splitting we MUST make sure
|
||||
* that any offset used is valid at the END of the sequence, since it may
|
||||
@@ -581,7 +602,7 @@ size_t ZSTD_ldm_generateSequences(
|
||||
}
|
||||
|
||||
void
|
||||
ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
|
||||
ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
|
||||
{
|
||||
while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
|
||||
rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
|
||||
@@ -617,7 +638,7 @@ ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const min
|
||||
* Returns the current sequence to handle, or if the rest of the block should
|
||||
* be literals, it returns a sequence with offset == 0.
|
||||
*/
|
||||
static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
|
||||
static rawSeq maybeSplitSequence(RawSeqStore_t* rawSeqStore,
|
||||
U32 const remaining, U32 const minMatch)
|
||||
{
|
||||
rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos];
|
||||
@@ -641,7 +662,7 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
|
||||
return sequence;
|
||||
}
|
||||
|
||||
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
|
||||
void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) {
|
||||
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
|
||||
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
|
||||
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
|
||||
@@ -658,14 +679,14 @@ void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_paramSwitch_e useRowMatchFinder,
|
||||
size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_ParamSwitch_e useRowMatchFinder,
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
unsigned const minMatch = cParams->minMatch;
|
||||
ZSTD_blockCompressor const blockCompressor =
|
||||
ZSTD_BlockCompressor_f const blockCompressor =
|
||||
ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
|
||||
/* Input bounds */
|
||||
BYTE const* const istart = (BYTE const*)src;
|
||||
@@ -690,7 +711,6 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
/* maybeSplitSequence updates rawSeqStore->pos */
|
||||
rawSeq const sequence = maybeSplitSequence(rawSeqStore,
|
||||
(U32)(iend - ip), minMatch);
|
||||
int i;
|
||||
/* End signal */
|
||||
if (sequence.offset == 0)
|
||||
break;
|
||||
@@ -703,6 +723,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
/* Run the block compressor */
|
||||
DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength);
|
||||
{
|
||||
int i;
|
||||
size_t const newLitLength =
|
||||
blockCompressor(ms, seqStore, rep, ip, sequence.litLength);
|
||||
ip += sequence.litLength;
|
||||
@@ -712,7 +733,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
rep[0] = sequence.offset;
|
||||
/* Store the sequence */
|
||||
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
|
||||
STORE_OFFSET(sequence.offset),
|
||||
OFFSET_TO_OFFBASE(sequence.offset),
|
||||
sequence.matchLength);
|
||||
ip += sequence.matchLength;
|
||||
}
|
||||
|
||||
22
vendor/github.com/DataDog/zstd/zstd_ldm.h
generated
vendored
22
vendor/github.com/DataDog/zstd/zstd_ldm.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,10 +12,6 @@
|
||||
#ifndef ZSTD_LDM_H
|
||||
#define ZSTD_LDM_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_compress_internal.h" /* ldmParams_t, U32 */
|
||||
#include "zstd.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
@@ -44,7 +40,7 @@ void ZSTD_ldm_fillHashTable(
|
||||
* sequences.
|
||||
*/
|
||||
size_t ZSTD_ldm_generateSequences(
|
||||
ldmState_t* ldms, rawSeqStore_t* sequences,
|
||||
ldmState_t* ldms, RawSeqStore_t* sequences,
|
||||
ldmParams_t const* params, void const* src, size_t srcSize);
|
||||
|
||||
/**
|
||||
@@ -65,9 +61,9 @@ size_t ZSTD_ldm_generateSequences(
|
||||
* two. We handle that case correctly, and update `rawSeqStore` appropriately.
|
||||
* NOTE: This function does not return any errors.
|
||||
*/
|
||||
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_paramSwitch_e useRowMatchFinder,
|
||||
size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_ParamSwitch_e useRowMatchFinder,
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
/**
|
||||
@@ -77,7 +73,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
* Avoids emitting matches less than `minMatch` bytes.
|
||||
* Must be called for data that is not passed to ZSTD_ldm_blockCompress().
|
||||
*/
|
||||
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
|
||||
void ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize,
|
||||
U32 const minMatch);
|
||||
|
||||
/* ZSTD_ldm_skipRawSeqStoreBytes():
|
||||
@@ -85,7 +81,7 @@ void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
|
||||
* Not to be used in conjunction with ZSTD_ldm_skipSequences().
|
||||
* Must be called for data with is not passed to ZSTD_ldm_blockCompress().
|
||||
*/
|
||||
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes);
|
||||
void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes);
|
||||
|
||||
/** ZSTD_ldm_getTableSize() :
|
||||
* Estimate the space needed for long distance matching tables or 0 if LDM is
|
||||
@@ -111,10 +107,6 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
|
||||
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
ZSTD_compressionParameters const* cParams);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_FAST_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h
generated
vendored
2
vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
39
vendor/github.com/DataDog/zstd/zstd_legacy.h
generated
vendored
39
vendor/github.com/DataDog/zstd/zstd_legacy.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -125,6 +125,20 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
||||
const void* dict,size_t dictSize)
|
||||
{
|
||||
U32 const version = ZSTD_isLegacy(src, compressedSize);
|
||||
char x;
|
||||
/* Avoid passing NULL to legacy decoding. */
|
||||
if (dst == NULL) {
|
||||
assert(dstCapacity == 0);
|
||||
dst = &x;
|
||||
}
|
||||
if (src == NULL) {
|
||||
assert(compressedSize == 0);
|
||||
src = &x;
|
||||
}
|
||||
if (dict == NULL) {
|
||||
assert(dictSize == 0);
|
||||
dict = &x;
|
||||
}
|
||||
(void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */
|
||||
switch(version)
|
||||
{
|
||||
@@ -243,6 +257,13 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
|
||||
frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
|
||||
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
/* In all cases, decompressedBound == nbBlocks * ZSTD_BLOCKSIZE_MAX.
|
||||
* So we can compute nbBlocks without having to change every function.
|
||||
*/
|
||||
if (frameSizeInfo.decompressedBound != ZSTD_CONTENTSIZE_ERROR) {
|
||||
assert((frameSizeInfo.decompressedBound & (ZSTD_BLOCKSIZE_MAX - 1)) == 0);
|
||||
frameSizeInfo.nbBlocks = (size_t)(frameSizeInfo.decompressedBound / ZSTD_BLOCKSIZE_MAX);
|
||||
}
|
||||
return frameSizeInfo;
|
||||
}
|
||||
|
||||
@@ -281,6 +302,12 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
|
||||
MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion,
|
||||
const void* dict, size_t dictSize)
|
||||
{
|
||||
char x;
|
||||
/* Avoid passing NULL to legacy decoding. */
|
||||
if (dict == NULL) {
|
||||
assert(dictSize == 0);
|
||||
dict = &x;
|
||||
}
|
||||
DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion);
|
||||
if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion);
|
||||
switch(newVersion)
|
||||
@@ -340,6 +367,16 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
|
||||
MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
|
||||
ZSTD_outBuffer* output, ZSTD_inBuffer* input)
|
||||
{
|
||||
static char x;
|
||||
/* Avoid passing NULL to legacy decoding. */
|
||||
if (output->dst == NULL) {
|
||||
assert(output->size == 0);
|
||||
output->dst = &x;
|
||||
}
|
||||
if (input->src == NULL) {
|
||||
assert(input->size == 0);
|
||||
input->src = &x;
|
||||
}
|
||||
DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version);
|
||||
switch(version)
|
||||
{
|
||||
|
||||
572
vendor/github.com/DataDog/zstd/zstd_opt.c
generated
vendored
572
vendor/github.com/DataDog/zstd/zstd_opt.c
generated
vendored
File diff suppressed because it is too large
Load Diff
64
vendor/github.com/DataDog/zstd/zstd_opt.h
generated
vendored
64
vendor/github.com/DataDog/zstd/zstd_opt.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,46 +12,62 @@
|
||||
#ifndef ZSTD_OPT_H
|
||||
#define ZSTD_OPT_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
|
||||
#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
|
||||
|| !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
|
||||
|| !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
|
||||
/* used in ZSTD_loadDictionaryContent() */
|
||||
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend);
|
||||
void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend);
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_btopt(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_btopt_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btopt_extDict(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
#define ZSTD_COMPRESSBLOCK_BTOPT ZSTD_compressBlock_btopt
|
||||
#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE ZSTD_compressBlock_btopt_dictMatchState
|
||||
#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT ZSTD_compressBlock_btopt_extDict
|
||||
#else
|
||||
#define ZSTD_COMPRESSBLOCK_BTOPT NULL
|
||||
#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE NULL
|
||||
#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT NULL
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_btultra(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_compressBlock_btopt_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
/* note : no btultra2 variant for extDict nor dictMatchState,
|
||||
* because btultra2 is not meant to work with dictionaries
|
||||
* and is only specific for the first block (no prefix) */
|
||||
size_t ZSTD_compressBlock_btultra2(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA ZSTD_compressBlock_btultra
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE ZSTD_compressBlock_btultra_dictMatchState
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT ZSTD_compressBlock_btultra_extDict
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA2 ZSTD_compressBlock_btultra2
|
||||
#else
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA NULL
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE NULL
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT NULL
|
||||
#define ZSTD_COMPRESSBLOCK_BTULTRA2 NULL
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_OPT_H */
|
||||
|
||||
241
vendor/github.com/DataDog/zstd/zstd_preSplit.c
generated
vendored
Normal file
241
vendor/github.com/DataDog/zstd/zstd_preSplit.c
generated
vendored
Normal file
@@ -0,0 +1,241 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "compiler.h" /* ZSTD_ALIGNOF */
|
||||
#include "mem.h" /* S64 */
|
||||
#include "zstd_deps.h" /* ZSTD_memset */
|
||||
#include "zstd_internal.h" /* ZSTD_STATIC_ASSERT */
|
||||
#include "hist.h" /* HIST_add */
|
||||
#include "zstd_preSplit.h"
|
||||
|
||||
|
||||
#define BLOCKSIZE_MIN 3500
|
||||
#define THRESHOLD_PENALTY_RATE 16
|
||||
#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2)
|
||||
#define THRESHOLD_PENALTY 3
|
||||
|
||||
#define HASHLENGTH 2
|
||||
#define HASHLOG_MAX 10
|
||||
#define HASHTABLESIZE (1 << HASHLOG_MAX)
|
||||
#define HASHMASK (HASHTABLESIZE - 1)
|
||||
#define KNUTH 0x9e3779b9
|
||||
|
||||
/* for hashLog > 8, hash 2 bytes.
|
||||
* for hashLog == 8, just take the byte, no hashing.
|
||||
* The speed of this method relies on compile-time constant propagation */
|
||||
FORCE_INLINE_TEMPLATE unsigned hash2(const void *p, unsigned hashLog)
|
||||
{
|
||||
assert(hashLog >= 8);
|
||||
if (hashLog == 8) return (U32)((const BYTE*)p)[0];
|
||||
assert(hashLog <= HASHLOG_MAX);
|
||||
return (U32)(MEM_read16(p)) * KNUTH >> (32 - hashLog);
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
unsigned events[HASHTABLESIZE];
|
||||
size_t nbEvents;
|
||||
} Fingerprint;
|
||||
typedef struct {
|
||||
Fingerprint pastEvents;
|
||||
Fingerprint newEvents;
|
||||
} FPStats;
|
||||
|
||||
static void initStats(FPStats* fpstats)
|
||||
{
|
||||
ZSTD_memset(fpstats, 0, sizeof(FPStats));
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE void
|
||||
addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
|
||||
{
|
||||
const char* p = (const char*)src;
|
||||
size_t limit = srcSize - HASHLENGTH + 1;
|
||||
size_t n;
|
||||
assert(srcSize >= HASHLENGTH);
|
||||
for (n = 0; n < limit; n+=samplingRate) {
|
||||
fp->events[hash2(p+n, hashLog)]++;
|
||||
}
|
||||
fp->nbEvents += limit/samplingRate;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE void
|
||||
recordFingerprint_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
|
||||
{
|
||||
ZSTD_memset(fp, 0, sizeof(unsigned) * ((size_t)1 << hashLog));
|
||||
fp->nbEvents = 0;
|
||||
addEvents_generic(fp, src, srcSize, samplingRate, hashLog);
|
||||
}
|
||||
|
||||
typedef void (*RecordEvents_f)(Fingerprint* fp, const void* src, size_t srcSize);
|
||||
|
||||
#define FP_RECORD(_rate) ZSTD_recordFingerprint_##_rate
|
||||
|
||||
#define ZSTD_GEN_RECORD_FINGERPRINT(_rate, _hSize) \
|
||||
static void FP_RECORD(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \
|
||||
{ \
|
||||
recordFingerprint_generic(fp, src, srcSize, _rate, _hSize); \
|
||||
}
|
||||
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(1, 10)
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(5, 10)
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(11, 9)
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(43, 8)
|
||||
|
||||
|
||||
static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); }
|
||||
|
||||
static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2, unsigned hashLog)
|
||||
{
|
||||
U64 distance = 0;
|
||||
size_t n;
|
||||
assert(hashLog <= HASHLOG_MAX);
|
||||
for (n = 0; n < ((size_t)1 << hashLog); n++) {
|
||||
distance +=
|
||||
abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents);
|
||||
}
|
||||
return distance;
|
||||
}
|
||||
|
||||
/* Compare newEvents with pastEvents
|
||||
* return 1 when considered "too different"
|
||||
*/
|
||||
static int compareFingerprints(const Fingerprint* ref,
|
||||
const Fingerprint* newfp,
|
||||
int penalty,
|
||||
unsigned hashLog)
|
||||
{
|
||||
assert(ref->nbEvents > 0);
|
||||
assert(newfp->nbEvents > 0);
|
||||
{ U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents;
|
||||
U64 deviation = fpDistance(ref, newfp, hashLog);
|
||||
U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
|
||||
return deviation >= threshold;
|
||||
}
|
||||
}
|
||||
|
||||
static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp)
|
||||
{
|
||||
size_t n;
|
||||
for (n = 0; n < HASHTABLESIZE; n++) {
|
||||
acc->events[n] += newfp->events[n];
|
||||
}
|
||||
acc->nbEvents += newfp->nbEvents;
|
||||
}
|
||||
|
||||
static void flushEvents(FPStats* fpstats)
|
||||
{
|
||||
size_t n;
|
||||
for (n = 0; n < HASHTABLESIZE; n++) {
|
||||
fpstats->pastEvents.events[n] = fpstats->newEvents.events[n];
|
||||
}
|
||||
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents;
|
||||
ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents));
|
||||
}
|
||||
|
||||
static void removeEvents(Fingerprint* acc, const Fingerprint* slice)
|
||||
{
|
||||
size_t n;
|
||||
for (n = 0; n < HASHTABLESIZE; n++) {
|
||||
assert(acc->events[n] >= slice->events[n]);
|
||||
acc->events[n] -= slice->events[n];
|
||||
}
|
||||
acc->nbEvents -= slice->nbEvents;
|
||||
}
|
||||
|
||||
#define CHUNKSIZE (8 << 10)
|
||||
static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize,
|
||||
int level,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
static const RecordEvents_f records_fs[] = {
|
||||
FP_RECORD(43), FP_RECORD(11), FP_RECORD(5), FP_RECORD(1)
|
||||
};
|
||||
static const unsigned hashParams[] = { 8, 9, 10, 10 };
|
||||
const RecordEvents_f record_f = (assert(0<=level && level<=3), records_fs[level]);
|
||||
FPStats* const fpstats = (FPStats*)workspace;
|
||||
const char* p = (const char*)blockStart;
|
||||
int penalty = THRESHOLD_PENALTY;
|
||||
size_t pos = 0;
|
||||
assert(blockSize == (128 << 10));
|
||||
assert(workspace != NULL);
|
||||
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
|
||||
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
|
||||
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
|
||||
|
||||
initStats(fpstats);
|
||||
record_f(&fpstats->pastEvents, p, CHUNKSIZE);
|
||||
for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) {
|
||||
record_f(&fpstats->newEvents, p + pos, CHUNKSIZE);
|
||||
if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty, hashParams[level])) {
|
||||
return pos;
|
||||
} else {
|
||||
mergeEvents(&fpstats->pastEvents, &fpstats->newEvents);
|
||||
if (penalty > 0) penalty--;
|
||||
}
|
||||
}
|
||||
assert(pos == blockSize);
|
||||
return blockSize;
|
||||
(void)flushEvents; (void)removeEvents;
|
||||
}
|
||||
|
||||
/* ZSTD_splitBlock_fromBorders(): very fast strategy :
|
||||
* compare fingerprint from beginning and end of the block,
|
||||
* derive from their difference if it's preferable to split in the middle,
|
||||
* repeat the process a second time, for finer grained decision.
|
||||
* 3 times did not brought improvements, so I stopped at 2.
|
||||
* Benefits are good enough for a cheap heuristic.
|
||||
* More accurate splitting saves more, but speed impact is also more perceptible.
|
||||
* For better accuracy, use more elaborate variant *_byChunks.
|
||||
*/
|
||||
static size_t ZSTD_splitBlock_fromBorders(const void* blockStart, size_t blockSize,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
#define SEGMENT_SIZE 512
|
||||
FPStats* const fpstats = (FPStats*)workspace;
|
||||
Fingerprint* middleEvents = (Fingerprint*)(void*)((char*)workspace + 512 * sizeof(unsigned));
|
||||
assert(blockSize == (128 << 10));
|
||||
assert(workspace != NULL);
|
||||
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
|
||||
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
|
||||
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
|
||||
|
||||
initStats(fpstats);
|
||||
HIST_add(fpstats->pastEvents.events, blockStart, SEGMENT_SIZE);
|
||||
HIST_add(fpstats->newEvents.events, (const char*)blockStart + blockSize - SEGMENT_SIZE, SEGMENT_SIZE);
|
||||
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents = SEGMENT_SIZE;
|
||||
if (!compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, 0, 8))
|
||||
return blockSize;
|
||||
|
||||
HIST_add(middleEvents->events, (const char*)blockStart + blockSize/2 - SEGMENT_SIZE/2, SEGMENT_SIZE);
|
||||
middleEvents->nbEvents = SEGMENT_SIZE;
|
||||
{ U64 const distFromBegin = fpDistance(&fpstats->pastEvents, middleEvents, 8);
|
||||
U64 const distFromEnd = fpDistance(&fpstats->newEvents, middleEvents, 8);
|
||||
U64 const minDistance = SEGMENT_SIZE * SEGMENT_SIZE / 3;
|
||||
if (abs64((S64)distFromBegin - (S64)distFromEnd) < minDistance)
|
||||
return 64 KB;
|
||||
return (distFromBegin > distFromEnd) ? 32 KB : 96 KB;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
|
||||
int level,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
DEBUGLOG(6, "ZSTD_splitBlock (level=%i)", level);
|
||||
assert(0<=level && level<=4);
|
||||
if (level == 0)
|
||||
return ZSTD_splitBlock_fromBorders(blockStart, blockSize, workspace, wkspSize);
|
||||
/* level >= 1*/
|
||||
return ZSTD_splitBlock_byChunks(blockStart, blockSize, level-1, workspace, wkspSize);
|
||||
}
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
36
vendor/github.com/DataDog/zstd/zstd_preSplit.h
generated
vendored
Normal file
36
vendor/github.com/DataDog/zstd/zstd_preSplit.h
generated
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_PRESPLIT_H
|
||||
#define ZSTD_PRESPLIT_H
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
#define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208
|
||||
|
||||
/* ZSTD_splitBlock():
|
||||
* @level must be a value between 0 and 4.
|
||||
* higher levels spend more energy to detect block boundaries.
|
||||
* @workspace must be aligned for size_t.
|
||||
* @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE
|
||||
* note:
|
||||
* For the time being, this function only accepts full 128 KB blocks.
|
||||
* Therefore, @blockSize must be == 128 KB.
|
||||
* While this could be extended to smaller sizes in the future,
|
||||
* it is not yet clear if this would be useful. TBD.
|
||||
*/
|
||||
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
|
||||
int level,
|
||||
void* workspace, size_t wkspSize);
|
||||
|
||||
#endif /* ZSTD_PRESPLIT_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
127
vendor/github.com/DataDog/zstd/zstd_stream.go
generated
vendored
127
vendor/github.com/DataDog/zstd/zstd_stream.go
generated
vendored
@@ -72,6 +72,7 @@ import (
|
||||
|
||||
var errShortRead = errors.New("short read")
|
||||
var errReaderClosed = errors.New("Reader is closed")
|
||||
var ErrNoParallelSupport = errors.New("No parallel support")
|
||||
|
||||
// Writer is an io.WriteCloser that zstd-compresses its input.
|
||||
type Writer struct {
|
||||
@@ -79,7 +80,6 @@ type Writer struct {
|
||||
|
||||
ctx *C.ZSTD_CCtx
|
||||
dict []byte
|
||||
srcBuffer []byte
|
||||
dstBuffer []byte
|
||||
firstError error
|
||||
underlyingWriter io.Writer
|
||||
@@ -137,7 +137,6 @@ func NewWriterLevelDict(w io.Writer, level int, dict []byte) *Writer {
|
||||
CompressionLevel: level,
|
||||
ctx: ctx,
|
||||
dict: dict,
|
||||
srcBuffer: make([]byte, 0),
|
||||
dstBuffer: make([]byte, CompressBound(1024)),
|
||||
firstError: err,
|
||||
underlyingWriter: w,
|
||||
@@ -153,65 +152,59 @@ func (w *Writer) Write(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
total := len(p)
|
||||
// Check if dstBuffer is enough
|
||||
w.dstBuffer = w.dstBuffer[0:cap(w.dstBuffer)]
|
||||
if len(w.dstBuffer) < CompressBound(len(p)) {
|
||||
w.dstBuffer = make([]byte, CompressBound(len(p)))
|
||||
}
|
||||
|
||||
// Do not do an extra memcopy if zstd ingest all input data
|
||||
srcData := p
|
||||
fastPath := len(w.srcBuffer) == 0
|
||||
if !fastPath {
|
||||
w.srcBuffer = append(w.srcBuffer, p...)
|
||||
srcData = w.srcBuffer
|
||||
}
|
||||
dstoff := 0
|
||||
consumed := 0
|
||||
for len(p) > 0 {
|
||||
C.ZSTD_compressStream2_wrapper(
|
||||
w.resultBuffer,
|
||||
w.ctx,
|
||||
unsafe.Pointer(&w.dstBuffer[dstoff]),
|
||||
C.size_t(len(w.dstBuffer[dstoff:])),
|
||||
unsafe.Pointer(&p[0]),
|
||||
C.size_t(len(p)),
|
||||
)
|
||||
ret := int(w.resultBuffer.return_code)
|
||||
if err := getError(ret); err != nil {
|
||||
// The stream is dead after this.
|
||||
w.firstError = err
|
||||
return 0, err
|
||||
}
|
||||
p = p[w.resultBuffer.bytes_consumed:]
|
||||
dstoff += int(w.resultBuffer.bytes_written)
|
||||
consumed += int(w.resultBuffer.bytes_consumed)
|
||||
if len(p) > 0 && dstoff == len(w.dstBuffer) {
|
||||
// We have bytes remaining to compress and our output buffer
|
||||
// filled up. This shouldn't happen since we calculated it
|
||||
// in advance using CompressBound, but we need to handle it
|
||||
// in case there was some miscalculation, or the internal
|
||||
// stream buffer contained enough data from previous writes
|
||||
// to overflow dstBuffer. (it's not clear from the docs
|
||||
// whether this is possible)
|
||||
//
|
||||
// Allocate space for whatever we haven't compressed yet.
|
||||
newbuf := make([]byte, len(w.dstBuffer)+CompressBound(total-consumed))
|
||||
copy(newbuf, w.dstBuffer)
|
||||
w.dstBuffer = newbuf
|
||||
|
||||
if len(srcData) == 0 {
|
||||
// this is technically unnecessary: srcData is p or w.srcBuffer, and len() > 0 checked above
|
||||
// but this ensures the code can change without dereferencing an srcData[0]
|
||||
return 0, nil
|
||||
}
|
||||
C.ZSTD_compressStream2_wrapper(
|
||||
w.resultBuffer,
|
||||
w.ctx,
|
||||
unsafe.Pointer(&w.dstBuffer[0]),
|
||||
C.size_t(len(w.dstBuffer)),
|
||||
unsafe.Pointer(&srcData[0]),
|
||||
C.size_t(len(srcData)),
|
||||
)
|
||||
ret := int(w.resultBuffer.return_code)
|
||||
if err := getError(ret); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
consumed := int(w.resultBuffer.bytes_consumed)
|
||||
if !fastPath {
|
||||
w.srcBuffer = w.srcBuffer[consumed:]
|
||||
} else {
|
||||
remaining := len(p) - consumed
|
||||
if remaining > 0 {
|
||||
// We still have some non-consumed data, copy remaining data to srcBuffer
|
||||
// Try to not reallocate w.srcBuffer if we already have enough space
|
||||
if cap(w.srcBuffer) >= remaining {
|
||||
w.srcBuffer = w.srcBuffer[0:remaining]
|
||||
} else {
|
||||
w.srcBuffer = make([]byte, remaining)
|
||||
}
|
||||
copy(w.srcBuffer, p[consumed:])
|
||||
}
|
||||
}
|
||||
|
||||
written := int(w.resultBuffer.bytes_written)
|
||||
// Write to underlying buffer
|
||||
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
|
||||
_, err := w.underlyingWriter.Write(w.dstBuffer[:dstoff])
|
||||
|
||||
// Same behaviour as zlib, we can't know how much data we wrote, only
|
||||
// if there was an error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(p), err
|
||||
return total, err
|
||||
}
|
||||
|
||||
// Flush writes any unwritten data to the underlying io.Writer.
|
||||
@@ -222,24 +215,18 @@ func (w *Writer) Flush() error {
|
||||
|
||||
ret := 1 // So we loop at least once
|
||||
for ret > 0 {
|
||||
var srcPtr *byte // Do not point anywhere, if src is empty
|
||||
if len(w.srcBuffer) > 0 {
|
||||
srcPtr = &w.srcBuffer[0]
|
||||
}
|
||||
|
||||
C.ZSTD_compressStream2_flush(
|
||||
w.resultBuffer,
|
||||
w.ctx,
|
||||
unsafe.Pointer(&w.dstBuffer[0]),
|
||||
C.size_t(len(w.dstBuffer)),
|
||||
unsafe.Pointer(srcPtr),
|
||||
C.size_t(len(w.srcBuffer)),
|
||||
unsafe.Pointer(uintptr(0)),
|
||||
C.size_t(0),
|
||||
)
|
||||
ret = int(w.resultBuffer.return_code)
|
||||
if err := getError(ret); err != nil {
|
||||
return err
|
||||
}
|
||||
w.srcBuffer = w.srcBuffer[w.resultBuffer.bytes_consumed:]
|
||||
written := int(w.resultBuffer.bytes_written)
|
||||
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
|
||||
if err != nil {
|
||||
@@ -266,24 +253,18 @@ func (w *Writer) Close() error {
|
||||
|
||||
ret := 1 // So we loop at least once
|
||||
for ret > 0 {
|
||||
var srcPtr *byte // Do not point anywhere, if src is empty
|
||||
if len(w.srcBuffer) > 0 {
|
||||
srcPtr = &w.srcBuffer[0]
|
||||
}
|
||||
|
||||
C.ZSTD_compressStream2_finish(
|
||||
w.resultBuffer,
|
||||
w.ctx,
|
||||
unsafe.Pointer(&w.dstBuffer[0]),
|
||||
C.size_t(len(w.dstBuffer)),
|
||||
unsafe.Pointer(srcPtr),
|
||||
C.size_t(len(w.srcBuffer)),
|
||||
unsafe.Pointer(uintptr(0)),
|
||||
C.size_t(0),
|
||||
)
|
||||
ret = int(w.resultBuffer.return_code)
|
||||
if err := getError(ret); err != nil {
|
||||
return err
|
||||
}
|
||||
w.srcBuffer = w.srcBuffer[w.resultBuffer.bytes_consumed:]
|
||||
written := int(w.resultBuffer.bytes_written)
|
||||
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
|
||||
if err != nil {
|
||||
@@ -302,6 +283,28 @@ func (w *Writer) Close() error {
|
||||
return getError(int(C.ZSTD_freeCStream(w.ctx)))
|
||||
}
|
||||
|
||||
// Set the number of workers to run the compression in parallel using multiple threads
|
||||
// If > 1, the Write() call will become asynchronous. This means data will be buffered until processed.
|
||||
// If you call Write() too fast, you might incur a memory buffer up to as large as your input.
|
||||
// Consider calling Flush() periodically if you need to compress a very large file that would not fit all in memory.
|
||||
// By default only one worker is used.
|
||||
func (w *Writer) SetNbWorkers(n int) error {
|
||||
if w.firstError != nil {
|
||||
return w.firstError
|
||||
}
|
||||
if err := getError(int(C.ZSTD_CCtx_setParameter(w.ctx, C.ZSTD_c_nbWorkers, C.int(n)))); err != nil {
|
||||
w.firstError = err
|
||||
// First error case, a shared libary is used, and the library was compiled without parallel support
|
||||
if err.Error() == "Unsupported parameter" {
|
||||
return ErrNoParallelSupport
|
||||
} else {
|
||||
// This could happen if a very large number is passed in, and possibly zstd refuse to create as many threads, or the OS fails to do so
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cSize is the recommended size of reader.compressionBuffer. This func and
|
||||
// invocation allow for a one-time check for validity.
|
||||
var cSize = func() int {
|
||||
@@ -463,7 +466,7 @@ func (r *reader) Read(p []byte) (int, error) {
|
||||
n, err = r.underlyingReader.Read(src[r.compressionLeft:])
|
||||
}
|
||||
if err != nil && err != io.EOF { // Handle underlying reader errors first
|
||||
return 0, fmt.Errorf("failed to read from underlying reader: %s", err)
|
||||
return 0, fmt.Errorf("failed to read from underlying reader: %w", err)
|
||||
}
|
||||
if n == 0 {
|
||||
// Ideally, we'd return with ErrUnexpectedEOF in all cases where the stream was unexpectedly EOF'd
|
||||
@@ -501,7 +504,7 @@ func (r *reader) Read(p []byte) (int, error) {
|
||||
// Keep src here even though we reuse later, the code might be deleted at some point
|
||||
runtime.KeepAlive(src)
|
||||
if err := getError(retCode); err != nil {
|
||||
return 0, fmt.Errorf("failed to decompress: %s", err)
|
||||
return 0, fmt.Errorf("failed to decompress: %w", err)
|
||||
}
|
||||
|
||||
// Put everything in buffer
|
||||
|
||||
19
vendor/github.com/DataDog/zstd/zstd_trace.h
generated
vendored
19
vendor/github.com/DataDog/zstd/zstd_trace.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,23 +12,20 @@
|
||||
#ifndef ZSTD_TRACE_H
|
||||
#define ZSTD_TRACE_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* weak symbol support
|
||||
* For now, enable conservatively:
|
||||
* - Only GNUC
|
||||
* - Only ELF
|
||||
* - Only x86-64 and i386
|
||||
* - Only x86-64, i386, aarch64 and risc-v.
|
||||
* Also, explicitly disable on platforms known not to work so they aren't
|
||||
* forgotten in the future.
|
||||
*/
|
||||
#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
|
||||
defined(__GNUC__) && defined(__ELF__) && \
|
||||
(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) && \
|
||||
(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
|
||||
defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \
|
||||
!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
|
||||
!defined(__CYGWIN__) && !defined(_AIX)
|
||||
# define ZSTD_HAVE_WEAK_SYMBOLS 1
|
||||
@@ -65,7 +62,7 @@ typedef struct {
|
||||
/**
|
||||
* Non-zero if streaming (de)compression is used.
|
||||
*/
|
||||
unsigned streaming;
|
||||
int streaming;
|
||||
/**
|
||||
* The dictionary ID.
|
||||
*/
|
||||
@@ -74,7 +71,7 @@ typedef struct {
|
||||
* Is the dictionary cold?
|
||||
* Only set on decompression.
|
||||
*/
|
||||
unsigned dictionaryIsCold;
|
||||
int dictionaryIsCold;
|
||||
/**
|
||||
* The dictionary size or zero if no dictionary.
|
||||
*/
|
||||
@@ -157,10 +154,6 @@ ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
|
||||
|
||||
#endif /* ZSTD_TRACE */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_TRACE_H */
|
||||
|
||||
#endif /* USE_EXTERNAL_ZSTD */
|
||||
|
||||
69
vendor/github.com/DataDog/zstd/zstd_v01.c
generated
vendored
69
vendor/github.com/DataDog/zstd/zstd_v01.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -15,6 +15,7 @@
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include "zstd_v01.h"
|
||||
#include "compiler.h"
|
||||
#include "error_private.h"
|
||||
|
||||
|
||||
@@ -191,25 +192,6 @@ typedef signed long long S64;
|
||||
/****************************************************************
|
||||
* Memory I/O
|
||||
*****************************************************************/
|
||||
/* FSE_FORCE_MEMORY_ACCESS
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
* Method 2 : direct access. This method is portable but violate C standard.
|
||||
* It can generate buggy code on targets generating assembly depending on alignment.
|
||||
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
||||
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define FSE_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
static unsigned FSE_32bits(void)
|
||||
{
|
||||
@@ -222,24 +204,6 @@ static unsigned FSE_isLittleEndian(void)
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
#if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2)
|
||||
|
||||
static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
||||
static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
||||
static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
||||
|
||||
#elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1)
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
||||
|
||||
static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
||||
static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
||||
static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
||||
|
||||
#else
|
||||
|
||||
static U16 FSE_read16(const void* memPtr)
|
||||
{
|
||||
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
@@ -255,8 +219,6 @@ static U64 FSE_read64(const void* memPtr)
|
||||
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
||||
|
||||
static U16 FSE_readLE16(const void* memPtr)
|
||||
{
|
||||
if (FSE_isLittleEndian())
|
||||
@@ -1191,7 +1153,7 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
|
||||
zstd - standard compression library
|
||||
Copyright (C) 2014-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -1422,7 +1384,7 @@ typedef struct {
|
||||
BYTE* matchLength;
|
||||
BYTE* dumpsStart;
|
||||
BYTE* dumps;
|
||||
} seqStore_t;
|
||||
} SeqStore_t;
|
||||
|
||||
|
||||
typedef struct ZSTD_Cctx_s
|
||||
@@ -1430,7 +1392,7 @@ typedef struct ZSTD_Cctx_s
|
||||
const BYTE* base;
|
||||
U32 current;
|
||||
U32 nextUpdate;
|
||||
seqStore_t seqStore;
|
||||
SeqStore_t seqStore;
|
||||
#ifdef __AVX2__
|
||||
__m256i hashTable[HASH_TABLESIZE>>3];
|
||||
#else
|
||||
@@ -1760,20 +1722,26 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
||||
const BYTE* const ostart = op;
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
const size_t litLength = sequence.litLength;
|
||||
BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
||||
const BYTE* const litEnd = *litPtr + litLength;
|
||||
|
||||
/* check */
|
||||
/* checks */
|
||||
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
||||
|
||||
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
||||
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
||||
/* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
|
||||
if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
|
||||
|
||||
if (endMatch > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit) return ERROR(corruption_detected);
|
||||
if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */
|
||||
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
||||
if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */
|
||||
|
||||
/* copy Literals */
|
||||
if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
|
||||
memmove(op, *litPtr, litLength); /* overwrite risk */
|
||||
else
|
||||
ZSTD_wildcopy(op, *litPtr, litLength);
|
||||
ZSTD_memmove(op, *litPtr, sequence.litLength); /* note : v0.1 seems to allow scenarios where output or input are close to end of buffer */
|
||||
|
||||
op += litLength;
|
||||
*litPtr = litEnd; /* update for next sequence */
|
||||
|
||||
@@ -2152,6 +2120,7 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
||||
}
|
||||
ctx->phase = 1;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
if (ZSTDv01_isError(rSize)) return rSize;
|
||||
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
|
||||
return rSize;
|
||||
}
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/zstd_v01.h
generated
vendored
2
vendor/github.com/DataDog/zstd/zstd_v01.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
107
vendor/github.com/DataDog/zstd/zstd_v02.c
generated
vendored
107
vendor/github.com/DataDog/zstd/zstd_v02.c
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
@@ -12,6 +12,7 @@
|
||||
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include "zstd_v02.h"
|
||||
#include "compiler.h"
|
||||
#include "error_private.h"
|
||||
|
||||
|
||||
@@ -29,7 +30,7 @@
|
||||
low-level memory access routines
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -72,20 +73,6 @@ extern "C" {
|
||||
#include <string.h> /* memcpy */
|
||||
|
||||
|
||||
/******************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
#if defined(__GNUC__)
|
||||
# define MEM_STATIC static __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define MEM_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define MEM_STATIC static __inline
|
||||
#else
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
|
||||
/****************************************************************
|
||||
* Basic Types
|
||||
*****************************************************************/
|
||||
@@ -116,24 +103,6 @@ extern "C" {
|
||||
/****************************************************************
|
||||
* Memory I/O
|
||||
*****************************************************************/
|
||||
/* MEM_FORCE_MEMORY_ACCESS
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
* Method 2 : direct access. This method is portable but violate C standard.
|
||||
* It can generate buggy code on targets generating assembly depending on alignment.
|
||||
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
||||
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define MEM_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
||||
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
||||
@@ -144,33 +113,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
||||
|
||||
/* violates C standard on structure alignment.
|
||||
Only use if no other choice to achieve best performance on target platform */
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
||||
|
||||
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
||||
|
||||
#else
|
||||
|
||||
/* default method, safe and standard.
|
||||
can sometimes prove slower */
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
||||
{
|
||||
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
@@ -191,9 +133,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
||||
|
||||
|
||||
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
@@ -270,7 +209,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
||||
header file (to include)
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -434,7 +373,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
}
|
||||
|
||||
/*! BIT_lookBitsFast :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
* unsafe version; only works if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
@@ -454,7 +393,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
}
|
||||
|
||||
/*!BIT_readBitsFast :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
* unsafe version; only works if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
||||
@@ -511,7 +450,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
||||
Error codes and messages
|
||||
Copyright (C) 2013-2015, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -610,7 +549,7 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor
|
||||
header file for static linking (only)
|
||||
Copyright (C) 2013-2015, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -754,7 +693,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
header file for static linking (only)
|
||||
Copyright (C) 2013-2015, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -823,7 +762,7 @@ static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, si
|
||||
Header File
|
||||
Copyright (C) 2014-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -883,7 +822,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */
|
||||
Header File for static linking only
|
||||
Copyright (C) 2014-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -924,7 +863,7 @@ extern "C" {
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
|
||||
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
||||
typedef struct ZSTDv02_Dctx_s ZSTD_DCtx;
|
||||
|
||||
/*
|
||||
Use above functions alternatively.
|
||||
@@ -947,7 +886,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
||||
FSE : Finite State Entropy coder
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -1451,7 +1390,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
|
||||
Huff0 : Huffman coder, part of New Generation Entropy library
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -2610,7 +2549,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
||||
zstd - standard compression library
|
||||
Copyright (C) 2014-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
@@ -2784,7 +2723,7 @@ typedef struct {
|
||||
BYTE* matchLength;
|
||||
BYTE* dumpsStart;
|
||||
BYTE* dumps;
|
||||
} seqStore_t;
|
||||
} SeqStore_t;
|
||||
|
||||
|
||||
/* *************************************
|
||||
@@ -2799,7 +2738,7 @@ static unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
/* *************************************************************
|
||||
* Decompression section
|
||||
***************************************************************/
|
||||
struct ZSTD_DCtx_s
|
||||
struct ZSTDv02_Dctx_s
|
||||
{
|
||||
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
|
||||
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
||||
@@ -3115,12 +3054,19 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
||||
|
||||
/* checks */
|
||||
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
||||
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
||||
|
||||
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
||||
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
||||
/* Now we know there are no overflow in literal nor match lengths, can use the pointer check */
|
||||
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
|
||||
if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
|
||||
|
||||
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
||||
ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
||||
op = oLitEnd;
|
||||
*litPtr = litEnd; /* update for next sequence */
|
||||
|
||||
@@ -3473,6 +3419,7 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
|
||||
}
|
||||
ctx->phase = 1;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
if (ZSTD_isError(rSize)) return rSize;
|
||||
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
|
||||
return rSize;
|
||||
}
|
||||
|
||||
2
vendor/github.com/DataDog/zstd/zstd_v02.h
generated
vendored
2
vendor/github.com/DataDog/zstd/zstd_v02.h
generated
vendored
@@ -1,6 +1,6 @@
|
||||
#ifndef USE_EXTERNAL_ZSTD
|
||||
/*
|
||||
* Copyright (c) Yann Collet, Facebook, Inc.
|
||||
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user