aleesa-bot/aleesa-webapp-go
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: aleesa-bot:v1.0
Branches Tags
aleesa-bot:master
aleesa-bot:v1.3 aleesa-bot:v1.2 aleesa-bot:v1.1 aleesa-bot:v1.0
..
compare: aleesa-bot:300e7862932ebc411c23fad251ed84a579b3da85
Branches Tags
aleesa-bot:master
aleesa-bot:v1.3 aleesa-bot:v1.2 aleesa-bot:v1.1 aleesa-bot:v1.0

12 Commits
v1.0 ... 300e786293

Author SHA1 Message Date
eleksir
300e786293 celebratoday.go: parse out holidays via another method 2026-03-07 20:42:51 +00:00
eleksir
bf2469a9db Add some holiday string filters to celebratoday.go 2026-03-07 08:36:33 +00:00
eleksir
170d0d0ed4 Update dependencies 2026-02-10 20:54:39 +03:00
eleksir
f2fc168abe Replace prazdnikisegodnyaru with celebratoday, small underhood fixes 
- Replace prazdnikisegodnyaru with celebratoday as first is banned in Russia
- Update .golangci.yml rule set
- Megatons of fixes for linter
- Update README.md
 2026-02-10 20:05:13 +03:00
eleksir
ff57b0af7d Small underhood changes, linter fixes 2025-10-20 21:07:10 +03:00
eleksir
6c00224f64 Update README.md 2025-10-12 16:50:33 +03:00
eleksir
d5c6d5719a Join webbapp core modules into single package 2025-10-12 16:42:16 +03:00
eleksir
3932835d47 Make Makefile more crossplatform 2025-10-12 15:41:13 +03:00
eleksir
5ebefcddf4 Update dependencies 2025-10-12 15:27:41 +03:00
eleksir
1330c512aa Add missing cmd folders, change logrus to slog, update gitignore 2025-10-12 15:25:08 +03:00
eleksir
6cebe40018 Update deps 2025-02-09 15:15:23 +03:00
eleksir
07430d87ed Implement caching previously queried (by chatter) city in weather module 2025-02-09 14:27:01 +03:00
587 changed files with 39209 additions and 56725 deletions
Expand all files Collapse all files

11
.gitignore vendored
View File

@@ -1,4 +1,7 @@
data/config.json
data/cache_db/**
aleesa-webapp-go
flickr_init
/data/config.json
/data/cache_db/**
/aleesa-webapp-go
/flickr_init
/flickr_populate
/flickr_test
/.golangci.bck.yml

764
.golangci.yml
View File

@@ -1,30 +1,17 @@
version: "2"
run:
tests: false
skip-dirs:
- data
- vendor
go: "1.25"
modules-download-mode: readonly
tests: false
allow-parallel-runners: true
go: '1.22'
output:
sort-results: true
linters-settings:
gosec:
excludes:
# Оно не используется для повышения безопасности или каких-то таких секъюрных вещей, оно используется для выборки
# произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью.
- G404 # Insecure random number source (rand)
# Мне не нужны шашечки, мне ехать надо.
- G402 # TLS InsecureSkipVerify set true
issues:
# Maximum issues count per one linter. Set to 0 to disable (default 50)
max-issues-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable (default 3)
max-same-issues: 0
linters:
disable-all: true
default: none
enable:
- errcheck
- gosimple
- govet
- ineffassign
- staticcheck
- unused
- asasalint
- asciicheck
- bodyclose
@@ -32,21 +19,30 @@ linters:
- contextcheck
- decorder
- durationcheck
- errcheck
- errchkjson
- errorlint
- exhaustive
- exportloopref
- exptostd
- ginkgolinter
- gocheckcompilerdirectives
- gocritic
- godoclint
- godot
- gofmt
- goimports
- godox
- goprintffuncname
- gosec
- gosmopolitan
- govet
- grouper
- importas
- ineffassign
- intrange
- ireturn
- makezero
- mirror
- misspell
- modernize
- nakedret
- nilerr
- nilnil
@@ -54,13 +50,727 @@ linters:
- noctx
- nolintlint
- nonamedreturns
- perfsprint
- predeclared
- promlinter
- reassign
- revive
- rowserrcheck
- stylecheck
- staticcheck
- unconvert
- unused
- usestdlibvars
- wastedassign
- whitespace
- wsl
- wsl_v5
settings:
decorder:
dec-order:
- type
- const
- var
- func
gocritic:
disable-all: false
enable-all: true
disabled-checks:
- singleCaseSwitch
- octalLiteral
- regexpSimplify
- paramTypeCombine
- whyNoLint
- hugeParam
godoclint:
default: all
ignore-unexported: false
ignore-exported: false
include-unexported: true
disable:
- max-len
godot:
scope: all
exclude:
# Exclude todo and fixme comments.
- "^fixme:"
- "^todo:"
period: true
capital: false
godox:
keywords:
- FIXME
gosec: # go-fucking-sic
excludes:
# Боже, как же вы затрахали своей крипто-ебучкой, наркоманы хреновы!
- G405 # Detect the usage of DES or RC4
- G406 # Detect the usage of MD4 or RIPEMD160
# Оно не используется для повышения безопасности или каких-то таких секъюрных вещей, оно используется для выборки
# произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью.
- G404 # Insecure random number source (rand)
- G501 # Import blocklist: crypto/md5
- G502 # Import blocklist: crypto/des
- G503 # Import blocklist: crypto/rc4
- G504 # Import blocklist: net/http/cgi
- G505 # Import blocklist: crypto/sha1
- G506 # Import blocklist: golang.org/x/crypto/md4
- G507 # Import blocklist: golang.org/x/crypto/ripemd160
config:
global:
audit: true
# Ваша сраная паранойя при создании файлов и каталогов мешает нормальной работе сервисов. Группы придуманы для
# удобства, группе удобно давать право чтения. Поэтому идите нахуй со своими ограничениями на чтение для
# группы, "безопасные" клоуны.
G301: "0755"
G306: "0644"
govet:
disable-all: false
enable-all: true
disable:
- shadow
grouper:
# Require the use of a single global 'const' declaration only.
# Default: false
const-require-single-const: true
# Require the use of grouped global 'const' declarations.
# Default: false
const-require-grouping: true
# Require the use of a single 'import' declaration only.
# Default: false
import-require-single-import: true
# Require the use of grouped 'import' declarations.
# Default: false
import-require-grouping: true
# Require the use of a single global 'type' declaration only.
# Default: false
type-require-single-type: true
# Require the use of grouped global 'type' declarations.
# Default: false
type-require-grouping: true
# Require the use of a single global 'var' declaration only.
# Default: false
var-require-single-var: true
# Require the use of grouped global 'var' declarations.
# Default: false
var-require-grouping: true
misspell:
# Correct spellings using locale preferences for US or UK.
# Setting locale to US will correct the British spelling of 'colour' to 'color'.
# Default is to use a neutral variety of English.
locale: US
# Typos to ignore.
# Should be in lower case.
# Default: []
ignore-rules:
- someword
# Extra word corrections.
# `typo` and `correction` should only contain letters.
# The words are case-insensitive.
# Default: []
extra-words:
- typo: "iff"
correction: "if"
- typo: "cancelation"
correction: "cancellation"
# Mode of the analysis:
# - default: checks all the file content.
# - restricted: checks only comments.
# Default: ""
mode: restricted
nonamedreturns:
# Report named error if it is assigned inside defer.
# Default: false
report-error-in-defer: true
perfsprint:
integer-format: true
int-conversion: true
error-format: true
err-error: false
errorf: true
string-format: true
sprintf1: true
strconcat: true
bool-format: true
hex-format: true
concat-loop: true
loop-other-ops: false
revive:
enable-all-rules: false
staticcheck:
# https://staticcheck.dev/docs/configuration/options/#dot_import_whitelist
# Default: ["github.com/mmcloughlin/avo/build", "github.com/mmcloughlin/avo/operand", "github.com/mmcloughlin/avo/reg"]
dot-import-whitelist:
- fmt
# https://staticcheck.dev/docs/configuration/options/#initialisms
# Default: ["ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS", "SIP", "RTP", "AMQP", "DB", "TS"]
initialisms: ["ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS", "SIP", "RTP", "AMQP", "DB", "TS"]
# https://staticcheck.dev/docs/configuration/options/#http_status_code_whitelist
# Default: ["200", "400", "404", "500"]
http-status-code-whitelist: ["200", "400", "404", "500"]
# SAxxxx checks in https://staticcheck.dev/docs/configuration/options/#checks
# Example (to disable some checks): [ "all", "-SA1000", "-SA1001"]
# Run `GL_DEBUG=staticcheck golangci-lint run --enable=staticcheck` to see all available checks and enabled by config checks.
# Default: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022"]
checks:
# Invalid regular expression.
# https://staticcheck.dev/docs/checks/#SA1000
- SA1000
# Invalid template.
# https://staticcheck.dev/docs/checks/#SA1001
- SA1001
# Invalid format in 'time.Parse'.
# https://staticcheck.dev/docs/checks/#SA1002
- SA1002
# Unsupported argument to functions in 'encoding/binary'.
# https://staticcheck.dev/docs/checks/#SA1003
- SA1003
# Suspiciously small untyped constant in 'time.Sleep'.
# https://staticcheck.dev/docs/checks/#SA1004
- SA1004
# Invalid first argument to 'exec.Command'.
# https://staticcheck.dev/docs/checks/#SA1005
- SA1005
# 'Printf' with dynamic first argument and no further arguments.
# https://staticcheck.dev/docs/checks/#SA1006
- SA1006
# Invalid URL in 'net/url.Parse'.
# https://staticcheck.dev/docs/checks/#SA1007
- SA1007
# Non-canonical key in 'http.Header' map.
# https://staticcheck.dev/docs/checks/#SA1008
- SA1008
# '(*regexp.Regexp).FindAll' called with 'n == 0', which will always return zero results.
# https://staticcheck.dev/docs/checks/#SA1010
- SA1010
# Various methods in the "strings" package expect valid UTF-8, but invalid input is provided.
# https://staticcheck.dev/docs/checks/#SA1011
- SA1011
# A nil 'context.Context' is being passed to a function, consider using 'context.TODO' instead.
# https://staticcheck.dev/docs/checks/#SA1012
- SA1012
# 'io.Seeker.Seek' is being called with the whence constant as the first argument, but it should be the second.
# https://staticcheck.dev/docs/checks/#SA1013
- SA1013
# Non-pointer value passed to 'Unmarshal' or 'Decode'.
# https://staticcheck.dev/docs/checks/#SA1014
- SA1014
# Using 'time.Tick' in a way that will leak. Consider using 'time.NewTicker', and only use 'time.Tick' in tests, commands and endless functions.
# https://staticcheck.dev/docs/checks/#SA1015
- SA1015
# Trapping a signal that cannot be trapped.
# https://staticcheck.dev/docs/checks/#SA1016
- SA1016
# Channels used with 'os/signal.Notify' should be buffered.
# https://staticcheck.dev/docs/checks/#SA1017
- SA1017
# 'strings.Replace' called with 'n == 0', which does nothing.
# https://staticcheck.dev/docs/checks/#SA1018
- SA1018
# Using a deprecated function, variable, constant or field.
# https://staticcheck.dev/docs/checks/#SA1019
- SA1019
# Using an invalid host:port pair with a 'net.Listen'-related function.
# https://staticcheck.dev/docs/checks/#SA1020
- SA1020
# Using 'bytes.Equal' to compare two 'net.IP'.
# https://staticcheck.dev/docs/checks/#SA1021
- SA1021
# Modifying the buffer in an 'io.Writer' implementation.
# https://staticcheck.dev/docs/checks/#SA1023
- SA1023
# A string cutset contains duplicate characters.
# https://staticcheck.dev/docs/checks/#SA1024
- SA1024
# It is not possible to use '(*time.Timer).Reset''s return value correctly.
# https://staticcheck.dev/docs/checks/#SA1025
- SA1025
# Cannot marshal channels or functions.
# https://staticcheck.dev/docs/checks/#SA1026
- SA1026
# Atomic access to 64-bit variable must be 64-bit aligned.
# https://staticcheck.dev/docs/checks/#SA1027
- SA1027
# 'sort.Slice' can only be used on slices.
# https://staticcheck.dev/docs/checks/#SA1028
- SA1028
# Inappropriate key in call to 'context.WithValue'.
# https://staticcheck.dev/docs/checks/#SA1029
- SA1029
# Invalid argument in call to a 'strconv' function.
# https://staticcheck.dev/docs/checks/#SA1030
- SA1030
# Overlapping byte slices passed to an encoder.
# https://staticcheck.dev/docs/checks/#SA1031
- SA1031
# Wrong order of arguments to 'errors.Is'.
# https://staticcheck.dev/docs/checks/#SA1032
- SA1032
# 'sync.WaitGroup.Add' called inside the goroutine, leading to a race condition.
# https://staticcheck.dev/docs/checks/#SA2000
- SA2000
# Empty critical section, did you mean to defer the unlock?.
# https://staticcheck.dev/docs/checks/#SA2001
- SA2001
# Called 'testing.T.FailNow' or 'SkipNow' in a goroutine, which isn't allowed.
# https://staticcheck.dev/docs/checks/#SA2002
- SA2002
# Deferred 'Lock' right after locking, likely meant to defer 'Unlock' instead.
# https://staticcheck.dev/docs/checks/#SA2003
- SA2003
# 'TestMain' doesn't call 'os.Exit', hiding test failures.
# https://staticcheck.dev/docs/checks/#SA3000
- SA3000
# Assigning to 'b.N' in benchmarks distorts the results.
# https://staticcheck.dev/docs/checks/#SA3001
- SA3001
# Binary operator has identical expressions on both sides.
# https://staticcheck.dev/docs/checks/#SA4000
- SA4000
# '&*x' gets simplified to 'x', it does not copy 'x'.
# https://staticcheck.dev/docs/checks/#SA4001
- SA4001
# Comparing unsigned values against negative values is pointless.
# https://staticcheck.dev/docs/checks/#SA4003
- SA4003
# The loop exits unconditionally after one iteration.
# https://staticcheck.dev/docs/checks/#SA4004
- SA4004
# Field assignment that will never be observed. Did you mean to use a pointer receiver?.
# https://staticcheck.dev/docs/checks/#SA4005
- SA4005
# A value assigned to a variable is never read before being overwritten. Forgotten error check or dead code?.
# https://staticcheck.dev/docs/checks/#SA4006
- SA4006
# The variable in the loop condition never changes, are you incrementing the wrong variable?.
# https://staticcheck.dev/docs/checks/#SA4008
- SA4008
# A function argument is overwritten before its first use.
# https://staticcheck.dev/docs/checks/#SA4009
- SA4009
# The result of 'append' will never be observed anywhere.
# https://staticcheck.dev/docs/checks/#SA4010
- SA4010
# Break statement with no effect. Did you mean to break out of an outer loop?.
# https://staticcheck.dev/docs/checks/#SA4011
- SA4011
# Comparing a value against NaN even though no value is equal to NaN.
# https://staticcheck.dev/docs/checks/#SA4012
- SA4012
# Negating a boolean twice ('!!b') is the same as writing 'b'. This is either redundant, or a typo.
# https://staticcheck.dev/docs/checks/#SA4013
- SA4013
# An if/else if chain has repeated conditions and no side-effects; if the condition didn't match the first time, it won't match the second time, either.
# https://staticcheck.dev/docs/checks/#SA4014
- SA4014
# Calling functions like 'math.Ceil' on floats converted from integers doesn't do anything useful.
# https://staticcheck.dev/docs/checks/#SA4015
- SA4015
# Certain bitwise operations, such as 'x ^ 0', do not do anything useful.
# https://staticcheck.dev/docs/checks/#SA4016
- SA4016
# Discarding the return values of a function without side effects, making the call pointless.
# https://staticcheck.dev/docs/checks/#SA4017
- SA4017
# Self-assignment of variables.
# https://staticcheck.dev/docs/checks/#SA4018
- SA4018
# Multiple, identical build constraints in the same file.
# https://staticcheck.dev/docs/checks/#SA4019
- SA4019
# Unreachable case clause in a type switch.
# https://staticcheck.dev/docs/checks/#SA4020
- SA4020
# "x = append(y)" is equivalent to "x = y".
# https://staticcheck.dev/docs/checks/#SA4021
- SA4021
# Comparing the address of a variable against nil.
# https://staticcheck.dev/docs/checks/#SA4022
- SA4022
# Impossible comparison of interface value with untyped nil.
# https://staticcheck.dev/docs/checks/#SA4023
- SA4023
# Checking for impossible return value from a builtin function.
# https://staticcheck.dev/docs/checks/#SA4024
- SA4024
# Integer division of literals that results in zero.
# https://staticcheck.dev/docs/checks/#SA4025
- SA4025
# Go constants cannot express negative zero.
# https://staticcheck.dev/docs/checks/#SA4026
- SA4026
# '(*net/url.URL).Query' returns a copy, modifying it doesn't change the URL.
# https://staticcheck.dev/docs/checks/#SA4027
- SA4027
# 'x % 1' is always zero.
# https://staticcheck.dev/docs/checks/#SA4028
- SA4028
# Ineffective attempt at sorting slice.
# https://staticcheck.dev/docs/checks/#SA4029
- SA4029
# Ineffective attempt at generating random number.
# https://staticcheck.dev/docs/checks/#SA4030
- SA4030
# Checking never-nil value against nil.
# https://staticcheck.dev/docs/checks/#SA4031
- SA4031
# Comparing 'runtime.GOOS' or 'runtime.GOARCH' against impossible value.
# https://staticcheck.dev/docs/checks/#SA4032
- SA4032
# Assignment to nil map.
# https://staticcheck.dev/docs/checks/#SA5000
- SA5000
# Deferring 'Close' before checking for a possible error.
# https://staticcheck.dev/docs/checks/#SA5001
- SA5001
# The empty for loop ("for {}") spins and can block the scheduler.
# https://staticcheck.dev/docs/checks/#SA5002
- SA5002
# Defers in infinite loops will never execute.
# https://staticcheck.dev/docs/checks/#SA5003
- SA5003
# "for { select { ..." with an empty default branch spins.
# https://staticcheck.dev/docs/checks/#SA5004
- SA5004
# The finalizer references the finalized object, preventing garbage collection.
# https://staticcheck.dev/docs/checks/#SA5005
- SA5005
# Infinite recursive call.
# https://staticcheck.dev/docs/checks/#SA5007
- SA5007
# Invalid struct tag.
# https://staticcheck.dev/docs/checks/#SA5008
- SA5008
# Invalid Printf call.
# https://staticcheck.dev/docs/checks/#SA5009
- SA5009
# Impossible type assertion.
# https://staticcheck.dev/docs/checks/#SA5010
- SA5010
# Possible nil pointer dereference.
# https://staticcheck.dev/docs/checks/#SA5011
- SA5011
# Passing odd-sized slice to function expecting even size.
# https://staticcheck.dev/docs/checks/#SA5012
- SA5012
# Using 'regexp.Match' or related in a loop, should use 'regexp.Compile'.
# https://staticcheck.dev/docs/checks/#SA6000
- SA6000
# Missing an optimization opportunity when indexing maps by byte slices.
# https://staticcheck.dev/docs/checks/#SA6001
- SA6001
# Storing non-pointer values in 'sync.Pool' allocates memory.
# https://staticcheck.dev/docs/checks/#SA6002
- SA6002
# Converting a string to a slice of runes before ranging over it.
# https://staticcheck.dev/docs/checks/#SA6003
- SA6003
# Inefficient string comparison with 'strings.ToLower' or 'strings.ToUpper'.
# https://staticcheck.dev/docs/checks/#SA6005
- SA6005
# Using io.WriteString to write '[]byte'.
# https://staticcheck.dev/docs/checks/#SA6006
- SA6006
# Defers in range loops may not run when you expect them to.
# https://staticcheck.dev/docs/checks/#SA9001
- SA9001
# Using a non-octal 'os.FileMode' that looks like it was meant to be in octal.
# https://staticcheck.dev/docs/checks/#SA9002
- SA9002
# Empty body in an if or else branch.
# https://staticcheck.dev/docs/checks/#SA9003
- SA9003
# Only the first constant has an explicit type.
# https://staticcheck.dev/docs/checks/#SA9004
- SA9004
# Trying to marshal a struct with no public fields nor custom marshaling.
# https://staticcheck.dev/docs/checks/#SA9005
- SA9005
# Dubious bit shifting of a fixed size integer value.
# https://staticcheck.dev/docs/checks/#SA9006
- SA9006
# Deleting a directory that shouldn't be deleted.
# https://staticcheck.dev/docs/checks/#SA9007
- SA9007
# 'else' branch of a type assertion is probably not reading the right value.
# https://staticcheck.dev/docs/checks/#SA9008
- SA9008
# Ineffectual Go compiler directive.
# https://staticcheck.dev/docs/checks/#SA9009
- SA9009
# Incorrect or missing package comment.
# https://staticcheck.dev/docs/checks/#ST1000
- ST1000
# Dot imports are discouraged.
# https://staticcheck.dev/docs/checks/#ST1001
- ST1001
# Poorly chosen identifier.
# https://staticcheck.dev/docs/checks/#ST1003
- ST1003
# Incorrectly formatted error string.
# https://staticcheck.dev/docs/checks/#ST1005
- ST1005
# Poorly chosen receiver name.
# https://staticcheck.dev/docs/checks/#ST1006
- ST1006
# A function's error value should be its last return value.
# https://staticcheck.dev/docs/checks/#ST1008
- ST1008
# Poorly chosen name for variable of type 'time.Duration'.
# https://staticcheck.dev/docs/checks/#ST1011
- ST1011
# Poorly chosen name for error variable.
# https://staticcheck.dev/docs/checks/#ST1012
- ST1012
# Should use constants for HTTP error codes, not magic numbers.
# https://staticcheck.dev/docs/checks/#ST1013
- ST1013
# A switch's default case should be the first or last case.
# https://staticcheck.dev/docs/checks/#ST1015
- ST1015
# Use consistent method receiver names.
# https://staticcheck.dev/docs/checks/#ST1016
- ST1016
# Don't use Yoda conditions.
# https://staticcheck.dev/docs/checks/#ST1017
- ST1017
# Avoid zero-width and control characters in string literals.
# https://staticcheck.dev/docs/checks/#ST1018
- ST1018
# Importing the same package multiple times.
# https://staticcheck.dev/docs/checks/#ST1019
- ST1019
# The documentation of an exported function should start with the function's name.
# https://staticcheck.dev/docs/checks/#ST1020
- ST1020
# The documentation of an exported type should start with type's name.
# https://staticcheck.dev/docs/checks/#ST1021
- ST1021
# The documentation of an exported variable or constant should start with variable's name.
# https://staticcheck.dev/docs/checks/#ST1022
- ST1022
# Redundant type in variable declaration.
# https://staticcheck.dev/docs/checks/#ST1023
- ST1023
# Use plain channel send or receive instead of single-case select.
# https://staticcheck.dev/docs/checks/#S1000
- S1000
# Replace for loop with call to copy.
# https://staticcheck.dev/docs/checks/#S1001
- S1001
# Omit comparison with boolean constant.
# https://staticcheck.dev/docs/checks/#S1002
- S1002
# Replace call to 'strings.Index' with 'strings.Contains'.
# https://staticcheck.dev/docs/checks/#S1003
- S1003
# Replace call to 'bytes.Compare' with 'bytes.Equal'.
# https://staticcheck.dev/docs/checks/#S1004
- S1004
# Drop unnecessary use of the blank identifier.
# https://staticcheck.dev/docs/checks/#S1005
- S1005
# Use "for { ... }" for infinite loops.
# https://staticcheck.dev/docs/checks/#S1006
- S1006
# Simplify regular expression by using raw string literal.
# https://staticcheck.dev/docs/checks/#S1007
- S1007
# Simplify returning boolean expression.
# https://staticcheck.dev/docs/checks/#S1008
- S1008
# Omit redundant nil check on slices, maps, and channels.
# https://staticcheck.dev/docs/checks/#S1009
- S1009
# Omit default slice index.
# https://staticcheck.dev/docs/checks/#S1010
- S1010
# Use a single 'append' to concatenate two slices.
# https://staticcheck.dev/docs/checks/#S1011
- S1011
# Replace 'time.Now().Sub(x)' with 'time.Since(x)'.
# https://staticcheck.dev/docs/checks/#S1012
- S1012
# Use a type conversion instead of manually copying struct fields.
# https://staticcheck.dev/docs/checks/#S1016
- S1016
# Replace manual trimming with 'strings.TrimPrefix'.
# https://staticcheck.dev/docs/checks/#S1017
- S1017
# Use "copy" for sliding elements.
# https://staticcheck.dev/docs/checks/#S1018
- S1018
# Simplify "make" call by omitting redundant arguments.
# https://staticcheck.dev/docs/checks/#S1019
- S1019
# Omit redundant nil check in type assertion.
# https://staticcheck.dev/docs/checks/#S1020
- S1020
# Merge variable declaration and assignment.
# https://staticcheck.dev/docs/checks/#S1021
- S1021
# Omit redundant control flow.
# https://staticcheck.dev/docs/checks/#S1023
- S1023
# Replace 'x.Sub(time.Now())' with 'time.Until(x)'.
# https://staticcheck.dev/docs/checks/#S1024
- S1024
# Don't use 'fmt.Sprintf("%s", x)' unnecessarily.
# https://staticcheck.dev/docs/checks/#S1025
- S1025
# Simplify error construction with 'fmt.Errorf'.
# https://staticcheck.dev/docs/checks/#S1028
- S1028
# Range over the string directly.
# https://staticcheck.dev/docs/checks/#S1029
- S1029
# Use 'bytes.Buffer.String' or 'bytes.Buffer.Bytes'.
# https://staticcheck.dev/docs/checks/#S1030
- S1030
# Omit redundant nil check around loop.
# https://staticcheck.dev/docs/checks/#S1031
- S1031
# Use 'sort.Ints(x)', 'sort.Float64s(x)', and 'sort.Strings(x)'.
# https://staticcheck.dev/docs/checks/#S1032
- S1032
# Unnecessary guard around call to "delete".
# https://staticcheck.dev/docs/checks/#S1033
- S1033
# Use result of type assertion to simplify cases.
# https://staticcheck.dev/docs/checks/#S1034
- S1034
# Redundant call to 'net/http.CanonicalHeaderKey' in method call on 'net/http.Header'.
# https://staticcheck.dev/docs/checks/#S1035
- S1035
# Unnecessary guard around map access.
# https://staticcheck.dev/docs/checks/#S1036
- S1036
# Elaborate way of sleeping.
# https://staticcheck.dev/docs/checks/#S1037
- S1037
# Unnecessarily complex way of printing formatted string.
# https://staticcheck.dev/docs/checks/#S1038
- S1038
# Unnecessary use of 'fmt.Sprint'.
# https://staticcheck.dev/docs/checks/#S1039
- S1039
# Type assertion to current type.
# https://staticcheck.dev/docs/checks/#S1040
- S1040
# Apply De Morgan's law.
# https://staticcheck.dev/docs/checks/#QF1001
- QF1001
# Convert untagged switch to tagged switch.
# https://staticcheck.dev/docs/checks/#QF1002
- QF1002
# Convert if/else-if chain to tagged switch.
# https://staticcheck.dev/docs/checks/#QF1003
- QF1003
# Use 'strings.ReplaceAll' instead of 'strings.Replace' with 'n == -1'.
# https://staticcheck.dev/docs/checks/#QF1004
- QF1004
# Expand call to 'math.Pow'.
# https://staticcheck.dev/docs/checks/#QF1005
- QF1005
# Lift 'if'+'break' into loop condition.
# https://staticcheck.dev/docs/checks/#QF1006
- QF1006
# Merge conditional assignment into variable declaration.
# https://staticcheck.dev/docs/checks/#QF1007
- QF1007
# Omit embedded fields from selector expression.
# https://staticcheck.dev/docs/checks/#QF1008
- QF1008
# Use 'time.Time.Equal' instead of '==' operator.
# https://staticcheck.dev/docs/checks/#QF1009
- QF1009
# Convert slice of bytes to string when printing it.
# https://staticcheck.dev/docs/checks/#QF1010
- QF1010
# Omit redundant type from variable declaration.
# https://staticcheck.dev/docs/checks/#QF1011
- QF1011
# Use 'fmt.Fprintf(x, ...)' instead of 'x.Write(fmt.Sprintf(...))'.
# https://staticcheck.dev/docs/checks/#QF1012
- QF1012
unconvert:
# Remove conversions that force intermediate rounding.
# Default: false
fast-math: true
# Be more conservative (experimental).
# Default: false
safe: true
usestdlibvars:
# Suggest the use of http.MethodXX.
# Default: true
http-method: false
# Suggest the use of http.StatusXX.
# Default: true
http-status-code: false
# Suggest the use of time.Month in time.Date.
# Default: false
time-date-month: true
# Suggest the use of time.Weekday.String().
# Default: true
time-weekday: true
# Suggest the use of time.Month.String().
# Default: false
time-month: true
# Suggest the use of time.Layout.
# Default: false
time-layout: true
# Suggest the use of crypto.Hash.String().
# Default: false
crypto-hash: true
# Suggest the use of rpc.DefaultXXPath.
# Default: false
default-rpc-path: true
# Suggest the use of sql.LevelXX.String().
# Default: false
sql-isolation-level: true
# Suggest the use of tls.SignatureScheme.String().
# Default: false
tls-signature-scheme: true
# Suggest the use of constant.Kind.String().
# Default: false
constant-kind: true
wsl_v5:
allow-first-in-block: true
allow-whole-block: true
branch-max-lines: 2
case-max-lines: 2
default: all
disable:
- if
- err
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- gofmt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
settings:
gofmt:
# Simplify code: gofmt with `-s` option.
# Default: true
simplify: true
# Apply the rewrite rules to the source before reformatting.
# https://pkg.go.dev/cmd/gofmt
# Default: []
rewrite-rules:
- pattern: 'interface{}'
replacement: 'any'
- pattern: 'a[b:len(a)]'
replacement: 'a[b:]'

140
Makefile
View File

@@ -1,23 +1,147 @@
#!/usr/bin/env gmake -f
BUILDOPTS=-ldflags='-s -w' -a -gcflags=all=-l -trimpath
GOOPTS=CGO_ENABLED=0
BUILDOPTS=-ldflags="-s -w" -a -gcflags=all=-l -trimpath -buildvcs=false
MYNAME=aleesa-webapp-go
BINARY=${MYNAME}
UNIX_BINARY=${MYNAME}
WINDOWS_BINARY=${MYNAME}.exe
MYNAME2=flickr_init
BINARY2=${MYNAME2}
UNIX_BINARY2=${MYNAME2}
WINDOWS_BINARY2=${MYNAME2}.exe
MYNAME3=flickr_populate
BINARY3=${MYNAME3}
UNIX_BINARY3=${MYNAME3}
WINDOWS_BINARY3=${MYNAME3}.exe
MYNAME4=flickr_test
BINARY4=${MYNAME4}
UNIX_BINARY4=${MYNAME4}
WINDOWS_BINARY4=${MYNAME4}.exe
RMCMD=rm -rf
# На windows имя бинарника может зависеть не только от платформы, но и от выбранной цели, для linux-а суффикс .exe
# не нужен
ifeq ($(OS),Windows_NT)
ifdef GOOS
ifeq ($(GOOS),windows)
BINARY=${WINDOWS_BINARY}
BINARY2=${WINDOWS_BINARY2}
BINARY3=${WINDOWS_BINARY3}
BINARY4=${WINDOWS_BINARY4}
else # not ifeq ($(GOOS),windows)
BINARY=${MYNAME}
BINARY2=${MYNAME2}
BINARY3=${MYNAME3}
BINARY4=${MYNAME4}
endif # ifeq ($(GOOS),windows)
else # not ifdef GOOS
BINARY=${WINDOWS_BINARY}
BINARY2=${WINDOWS_BINARY}
BINARY3=${WINDOWS_BINARY}
BINARY4=${WINDOWS_BINARY}
endif # ifdef GOOS
ifeq ($(SHELL), sh.exe)
RMCMD=DEL /Q /F
endif
endif
# Явно определяем символ новой строки, чтобы избежать неоднозначности на windows
define IFS
endef
all: clean build
build:
CGO_ENABLED=0 go build ${BUILDOPTS} -o aleesa-webapp-go ./cmd/aleesa-webapp-go
ifeq ($(OS),Windows_NT)
# Looks like on windows gnu make explicitly set SHELL to sh.exe, if it was not set.
ifeq ($(SHELL), sh.exe)
# # Vanilla cmd.exe / powershell.
SET "CGO_ENABLED=0"
go build ${BUILDOPTS} -o ${BINARY} ./cmd/${MYNAME}
else ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
# # git-bash
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY} ./cmd/${MYNAME}
else # not ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
# # Some other shell.
# # TODO: handle it.
$(info "-- Dunno how to handle this shell: ${SHELL}")
endif # ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
else # not ($(OS),Windows_NT)
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY} ./cmd/${MYNAME}
endif # ifeq ($(OS),Windows_NT)
buildutils:
CGO_ENABLED=0 go build ${BUILDOPTS} -o flickr_init ./cmd/flickr_init
CGO_ENABLED=0 go build ${BUILDOPTS} -o flickr_populate ./cmd/flickr_populate
CGO_ENABLED=0 go build ${BUILDOPTS} -o flickr_test ./cmd/flickr_test
ifeq ($(OS),Windows_NT)
# Looks like on windows gnu make explicitly set SHELL to sh.exe, if it was not set.
ifeq ($(SHELL), sh.exe)
# # Vanilla cmd.exe / powershell.
SET "CGO_ENABLED=0"
go build ${BUILDOPTS} -o ${BINARY2} ./cmd/${MYNAME2}
go build ${BUILDOPTS} -o ${BINARY3} ./cmd/${MYNAME3}
go build ${BUILDOPTS} -o ${BINARY4} ./cmd/${MYNAME4}
else ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
# # git-bash
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY2} ./cmd/${MYNAME2}
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY3} ./cmd/${MYNAME3}
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY4} ./cmd/${MYNAME4}
else # not ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
# # Some other shell.
# # TODO: handle it.
$(info "-- Dunno how to handle this shell: ${SHELL}")
endif # ifeq (,$(findstring(Git/usr/bin/sh.exe, $(SHELL))))
else # not ($(OS),Windows_NT)
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY2} ./cmd/${MYNAME2}
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY3} ./cmd/${MYNAME3}
CGO_ENABLED=0 go build ${BUILDOPTS} -o ${BINARY4} ./cmd/${MYNAME4}
endif # ifeq ($(OS),Windows_NT)
clean:
$(RM) -rf aleesa-webapp-go flickr_init flickr_populate flickr_test
ifeq ($(OS),Windows_NT)
ifeq ($(SHELL),sh.exe)
# # Vanilla cmd.exe / powershell.
if exist ${WINDOWS_BINARY} ${RMCMD} ${WINDOWS_BINARY}
if exist ${UNIX_BINARY} ${RMCMD} ${UNIX_BINARY}
if exist ${WINDOWS_BINARY2} ${RMCMD} ${WINDOWS_BINARY2}
if exist ${UNIX_BINARY2} ${RMCMD} ${UNIX_BINARY2}
if exist ${WINDOWS_BINARY3} ${RMCMD} ${WINDOWS_BINARY3}
if exist ${UNIX_BINARY3} ${RMCMD} ${UNIX_BINARY3}
if exist ${WINDOWS_BINARY4} ${RMCMD} ${WINDOWS_BINARY4}
if exist ${UNIX_BINARY4} ${RMCMD} ${UNIX_BINARY4}
else # not ifeq ($(SHELL),sh.exe)
${RMCMD} ./${WINDOWS_BINARY}
${RMCMD} ./${UNIX_BINARY}
${RMCMD} ./${WINDOWS_BINARY2}
${RMCMD} ./${UNIX_BINARY2}
${RMCMD} ./${WINDOWS_BINARY3}
${RMCMD} ./${UNIX_BINARY3}
${RMCMD} ./${WINDOWS_BINARY4}
${RMCMD} ./${UNIX_BINARY4}
endif # ifeq ($(SHELL),sh.exe)
else # not ifeq ($(OS),Windows_NT)
${RMCMD} ./${BINARY}
${RMCMD} ./${BINARY2}
${RMCMD} ./${BINARY3}
${RMCMD} ./${BINARY4}
endif
upgrade:
$(RM) -rf vendor
go get -d -u -t ./...
go get -u ./...
go mod tidy
go mod vendor

12
README.md
View File

@@ -14,6 +14,18 @@
различных UA, по количеству дней в месяце. В общем случае делается произвольная выборка из этого списка, но для
некоторых сервисов сделана привязка ко дню недели.
## Как это собрать?
Понадобится компилятор go версии 1.25.3 или более новый.
Сам процесс сборки простой, достаточно запустить make или gmake, в зависимости от платформы:
```bash
make
```
и через некоторое время мы получаем бинарник **aleesa-webapp-go**. Это и есть наше приложение.
### Как это запустить?
Приложение тестировалось на alpine linux x86_64 3.16, но должно работать на любом (более свежем) linux-е.

71
cmd/aleesa-webapp-go/main.go Normal file
View File

@@ -0,0 +1,71 @@
// Implements aleesa-web-go application.
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
"time"
"aleesa-webapp-go/internal/log"
"aleesa-webapp-go/internal/webapp"
"github.com/go-redis/redis/v8"
)
// Основная функция программы, не добавить и не убавить.
func main() {
var (
logfile *os.File
)
err := webapp.ReadConfig()
if err != nil {
log.Errorf("Unable to parse config: %s", err)
os.Exit(1)
}
// Откроем лог и скормим его логгеру.
if webapp.Config.Log != "" {
logfile, err = os.OpenFile(webapp.Config.Log, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
if err != nil {
log.Errorf("Unable to open log file %s: %s", webapp.Config.Log, err)
os.Exit(1)
}
}
log.Init(webapp.Config.Loglevel, logfile)
// Иницализируем клиента Редиски.
webapp.Config.RedisClient = redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%d", webapp.Config.Server, webapp.Config.Port),
}).WithContext(webapp.Ctx).WithTimeout(time.Duration(webapp.Config.Timeout) * time.Second)
// Обозначим, что хотим после соединения подписаться на события из канала config.Channel.
webapp.Subscriber = webapp.Config.RedisClient.Subscribe(webapp.Ctx, webapp.Config.Channel)
// Самое время поставить трапы на сигналы.
signal.Notify(webapp.SigChan,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
// Запустим обработчик сигналов.
go webapp.SigHandler(webapp.Config)
// Начнём выгребать события из редиски (длина конвеера/буфера канала по-умолчанию - 100 сообщений).
ch := webapp.Subscriber.Channel()
log.Info("Service started.")
for msg := range ch {
if !webapp.Shutdown {
webapp.MsgParser(webapp.Config, webapp.Ctx, msg.Payload)
}
}
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

44
cmd/celebratoday_test/main.go Normal file
View File

@@ -0,0 +1,44 @@
// Package main implements celebratoday api test.
package main
import (
"fmt"
"os"
"aleesa-webapp-go/internal/celebratoday"
"aleesa-webapp-go/internal/log"
"aleesa-webapp-go/internal/webapp"
)
func main() {
var (
logfile *os.File
)
err := webapp.ReadConfig()
if err != nil {
log.Errorf("Unable to parse config: %s", err)
os.Exit(1)
}
// Откроем лог и скормим его логгеру.
if webapp.Config.Log != "" {
logfile, err = os.OpenFile(webapp.Config.Log, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
if err != nil {
log.Errorf("Unable to open log file %s: %s", webapp.Config.Log, err)
os.Exit(1)
}
}
log.Init(webapp.Config.Loglevel, logfile)
str, err := celebratoday.APIClient(webapp.Config)
if err != nil {
fmt.Printf("Error: %s\n", err)
}
fmt.Println(str)
}

23
cmd/flickr_init/main.go Normal file
View File

@@ -0,0 +1,23 @@
// Implements initialisation app for flickr service. Gains Tokens, etc.
package main
import (
"aleesa-webapp-go/internal/flickr"
"aleesa-webapp-go/internal/webapp"
"fmt"
"os"
)
func main() {
if err := webapp.ReadConfig(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err := flickr.APIClientInit(webapp.Config); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

15
cmd/flickr_populate/main.go
View File

@@ -1,21 +1,22 @@
// Package main get flickr oauth keys required for proper client work an stores it locally.
package main
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/flickr"
"aleesa-webapp-go/internal/webapp"
"fmt"
"os"
)
func main() {
if c, err := config.ReadConfig(); err != nil {
if err := webapp.ReadConfig(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err := flickr.Populate(webapp.Config); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
} else {
if err := flickr.Populate(c); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
}

23
cmd/flickr_test/main.go
View File

@@ -1,26 +1,27 @@
// Package main ensures that oauth keys downloaded correctly and client part is working.
package main
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/flickr"
"aleesa-webapp-go/internal/webapp"
"fmt"
"os"
)
func main() {
if c, err := config.ReadConfig(); err != nil {
if err := webapp.ReadConfig(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
} else {
result, err := flickr.SearchByTags(c, []string{"snail", "slug"})
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Println(result)
}
result, err := flickr.SearchByTags(webapp.Config, []string{"snail", "slug"})
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Println(result)
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

50
go.mod
View File

@@ -1,43 +1,45 @@
module aleesa-webapp-go
go 1.22.5
go 1.25.3
require (
github.com/briandowns/openweathermap v0.20.0
github.com/carlmjohnson/requests v0.24.2
github.com/cockroachdb/errors v1.11.3
github.com/cockroachdb/pebble v1.1.2
github.com/briandowns/openweathermap v0.21.1
github.com/carlmjohnson/requests v0.25.1
github.com/cockroachdb/errors v1.12.0
github.com/cockroachdb/pebble v1.1.5
github.com/go-redis/redis/v8 v8.11.5
github.com/hjson/hjson-go v3.3.0+incompatible
github.com/sirupsen/logrus v1.9.3
golang.org/x/net v0.29.0
gopkg.in/masci/flickr.v3 v3.0.0-20230428071620-b971d524ac6f
golang.org/x/net v0.50.0
gopkg.in/masci/flickr.v3 v3.0.0-20251111092038-3b6567f2215e
)
require (
github.com/DataDog/zstd v1.5.6 // indirect
github.com/DataDog/zstd v1.5.7 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
github.com/cockroachdb/redact v1.1.6 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20250429170803-42689b6311bb // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/getsentry/sentry-go v0.29.0 // indirect
github.com/getsentry/sentry-go v0.42.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/klauspost/compress v1.18.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.20.4 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.59.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/exp v0.0.0-20260209203927-2842357ff358 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)

145
go.sum
View File

@@ -1,61 +1,88 @@
github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY=
github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/briandowns/openweathermap v0.20.0 h1:NTfd7RRdmkqtmQVq1gviob+I/fuVYiAT7TXND7t/484=
github.com/briandowns/openweathermap v0.20.0/go.mod h1:0GLnknqicWxXnGi1IqoOaZIw+kIe5hkt+YM5WY3j8+0=
github.com/carlmjohnson/requests v0.24.2 h1:JDakhAmTIKL/qL/1P7Kkc2INGBJIkIFP6xUeUmPzLso=
github.com/carlmjohnson/requests v0.24.2/go.mod h1:duYA/jDnyZ6f3xbcF5PpZ9N8clgopubP2nK5i6MVMhU=
github.com/briandowns/openweathermap v0.21.1 h1:TPbuixuF+aGJP1mpgTNny6eUkdbvj7gqODGXkwhss48=
github.com/briandowns/openweathermap v0.21.1/go.mod h1:0GLnknqicWxXnGi1IqoOaZIw+kIe5hkt+YM5WY3j8+0=
github.com/carlmjohnson/requests v0.25.1 h1:17zNRLecxtAjhtdEIV+F+wrYfe+AGZUjWJtpndcOUYA=
github.com/carlmjohnson/requests v0.25.1/go.mod h1:z3UEf8IE4sZxZ78spW6/tLdqBkfCu1Fn4RaYMnZ8SRM=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
github.com/cockroachdb/errors v1.12.0 h1:d7oCs6vuIMUQRVbi6jWWWEJZahLCfJpnJSVobd1/sUo=
github.com/cockroachdb/errors v1.12.0/go.mod h1:SvzfYNNBshAVbZ8wzNc/UPK3w1vf0dKDUP41ucAIf7g=
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0 h1:pU88SPhIFid6/k0egdR5V6eALQYq2qbSmukrkgIh/0A=
github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILMqgNeV5jiqR4j+sTuvQNHdf2chuKj1M5k=
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw=
github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo=
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/cockroachdb/tokenbucket v0.0.0-20250429170803-42689b6311bb h1:3bCgBvB8PbJVMX1ouCcSIxvsqKPYM7gs72o0zC76n9g=
github.com/cockroachdb/tokenbucket v0.0.0-20250429170803-42689b6311bb/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA=
github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI=
github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
@@ -67,53 +94,68 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0=
github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/exp v0.0.0-20260209203927-2842357ff358 h1:kpfSV7uLwKJbFSEgNhWzGSL47NDSF/5pYYQw1V0ub6c=
golang.org/x/exp v0.0.0-20260209203927-2842357ff358/go.mod h1:R3t0oliuryB5eenPWl3rrQxwnNM3WTwnsRZZiXLAAW8=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -122,15 +164,20 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/masci/flickr.v3 v3.0.0-20230428071620-b971d524ac6f h1:RUQkjvXxx0U73EFbcOE6lz0mjwHyOCpkIkY8Toitzlk=
gopkg.in/masci/flickr.v3 v3.0.0-20230428071620-b971d524ac6f/go.mod h1:2fXoNIK2lULBoY5w6sID6+4rwMujRHgsKIHTMYSu+TU=
gopkg.in/masci/flickr.v3 v3.0.0-20250416134523-515bc5586967 h1:oFthoMSKQ8bHGNGF/KmUwkECdBjx/Odxqnli3zhDD+A=
gopkg.in/masci/flickr.v3 v3.0.0-20250416134523-515bc5586967/go.mod h1:2fXoNIK2lULBoY5w6sID6+4rwMujRHgsKIHTMYSu+TU=
gopkg.in/masci/flickr.v3 v3.0.0-20251111092038-3b6567f2215e h1:Ek9/avygT1lvdpFZuH+UGJlap+GhPvPB/qns7IGlm9s=
gopkg.in/masci/flickr.v3 v3.0.0-20251111092038-3b6567f2215e/go.mod h1:2fXoNIK2lULBoY5w6sID6+4rwMujRHgsKIHTMYSu+TU=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

12
internal/anekdotru/anekdotru.go
View File

@@ -1,3 +1,4 @@
// Package anekdotru implemets client for anekdot.ru service.
package anekdotru
import (
@@ -13,6 +14,7 @@ import (
"github.com/carlmjohnson/requests"
)
// APIClient клиент сервиса anekdot.ru.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
@@ -22,11 +24,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
url = "https://www.anekdot.ru/rss/randomu.html"
)
err := requests.
URL(url).
UserAgent(userAgent).
ToBytesBuffer(&respBody).
Fetch(ctx)
err := requests.URL(url).UserAgent(userAgent).ToBytesBuffer(&respBody).Fetch(ctx)
if err != nil {
return "", fmt.Errorf("unable to GET %s: %w", url, err)
@@ -41,6 +39,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
if (len(aneks) >= 2) && (aneks[1] != "") {
re = regexp.MustCompile(`'\);`)
a := re.Split(aneks[1], 2)
if (len(a) >= 2) && (a[0] != "") {
@@ -50,7 +49,9 @@ func APIClient(cfg *config.MyConfig) (string, error) {
anek = re.ReplaceAll(anek, []byte(`"`))
re = regexp.MustCompile(`\\"`)
anekJSON := re.ReplaceAll(anek, []byte(`"`))
anekJSON = bytes.ToValidUTF8(anekJSON, []byte{0xef, 0xbf, 0xbd})
var anekMap []string
@@ -59,6 +60,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
}
re = regexp.MustCompile(`<br>`)
answer := re.ReplaceAll([]byte(anekMap[0]), []byte("\n"))
return string(answer), nil

9
internal/bunicomic/bunicomic.go
View File

@@ -1,8 +1,10 @@
// Package bunicomic implements client for bunicomic.com service.
package bunicomic
import (
"aleesa-webapp-go/internal/config"
"context"
"errors"
"fmt"
"math/rand/v2"
"regexp"
@@ -12,6 +14,7 @@ import (
"golang.org/x/net/html"
)
// APIClient клиент сервиса bunicomic.com.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
@@ -46,13 +49,13 @@ func APIClient(cfg *config.MyConfig) (string, error) {
if n.Type == html.ElementNode && n.Data == "meta" {
for _, meta := range n.Attr {
if meta.Key == "content" {
// adds a new link entry when the attribute matches
// Adds a new link entry when the attribute matches.
metaTags = append(metaTags, meta.Val)
}
}
}
// traverses the HTML of the webpage from the first child node
// Traverses the HTML of the webpage from the first child node.
for c := n.FirstChild; c != nil; c = c.NextSibling {
meta(c)
}
@@ -76,7 +79,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
}
// Картинки не выпарсилось, это точно ошибка.
err = fmt.Errorf("unable to get link to buni comic strip image")
err = errors.New("unable to get link to buni comic strip image")
return "", err
}

233
internal/celebratoday/celebratoday.go Normal file
View File

@@ -0,0 +1,233 @@
// Package celebratoday implements client for celebratoday.com service.
package celebratoday
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/log"
"aleesa-webapp-go/internal/pcachedb"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"time"
"github.com/carlmjohnson/requests"
"golang.org/x/net/html"
)
type (
// Holy структура, описывающая искомый json-чик.
Holy struct {
AtContext string `json:"@context"`
AtType string `json:"@type"`
MainEntity []struct {
AtType string `json:"@type"`
Name string `json:"name"`
AcceptedAnswer struct {
AtType string `json:"@type"`
Text string `json:"text"`
UpVoteCount int `json:"upvoteCount"`
URL string `json:"url,omitempty"`
} `json:"acceptedAnswer"`
SuggestedAnswer []struct {
AtType string `json:"@type"`
Text string `json:"text"`
UpVoteCount int `json:"upvoteCount"`
URL string `json:"url,omitempty"`
} `json:"suggestedAnswer"`
} `json:"mainEntity"`
}
)
// Client обёртка для клиента сервиса celebratoday.com, работающая с кэшем. Запросы надо делать
// именно через неё.
func Client(cfg *config.MyConfig) (string, error) {
tsNow := time.Now()
// eTime TTL кэша. На текущий момент +1 час к текущему времени.
eTime := tsNow.Unix() + 3600
tsStr := pcachedb.GetValue(cfg, "cache", "celebratoday_timestamp")
// Кэш пустой, запросов пока не было.
if tsStr == "" {
log.Debugf("CT no cache entry for celebratoday.com")
answer, err := APIClient(cfg)
if err != nil {
return "", err
}
if err := UpdateCache(cfg, eTime, answer); err != nil {
log.Errorf("%s", err)
}
return answer, nil
}
tsCache, err := strconv.ParseInt(tsStr, 10, 64)
if err != nil {
log.Debug("KSP cache miss")
log.Warnf("KSP-Cache: unable to parse string as int64 (sha256 collision?): %s", err)
answer, err := APIClient(cfg)
// Бинго! у нас ещё и ошибка при обращении к "api" celebratoday.com, здесь мы ничего сделать не можем.
if err != nil {
return "", err
}
// Если всё хорошо, надо обновить кэш.
if err := UpdateCache(cfg, eTime, answer); err != nil {
log.Errorf("%s", err)
}
return answer, nil
}
// Текущее время в Unix timestamp.
tsNowUnix := tsNow.Unix()
// Кэш просрочен.
if tsCache < tsNowUnix {
log.Debug("KSP cache miss")
answer, err := APIClient(cfg)
// Здесь мы ничего сделать не можем.
if err != nil {
return "", err
}
// Если всё хорошо, надо обновить кэш.
if err := UpdateCache(cfg, eTime, answer); err != nil {
log.Errorf("%s", err)
}
return answer, nil
}
// TODO: как-то научиться определять ситуацию с отсутствующим значением в кэше.
// TODO: Научиться определять ситуацию, когда значением является unix timestamp.
log.Debug("CT cache hit")
return pcachedb.GetValue(cfg, "cache", "celebratoday_value"), nil
}
// APIClient клиент сервиса https://celebratoday.com/.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
url = "https://celebratoday.com/ru/today"
respBody string
)
// Формальная попытка притвориться браузером.
_, _, day := time.Now().Date()
userAgent := cfg.UserAgents[day]
if err := requests.URL(url).UserAgent(userAgent).
Header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8").
Header("Accept-Language", "en-US,en;q=0.5").
Header("User-Agent", "ru-RU").
Header("Accept-Charset", "utf-8").ToString(&respBody).Fetch(ctx); err != nil {
return "", fmt.Errorf("unable to GET %s: %w", url, err)
}
holidays, err := parser(respBody)
if err != nil {
return "", fmt.Errorf("Unable to parse response from celebratoday.com: %w", err)
}
if len(holidays) == 0 {
return "", errors.New("unable to parse response from celebratoday.com: no holidays found")
}
slices.Sort(holidays)
holidays = slices.Compact(holidays)
return "* " + strings.Join(holidays, "\n* "), nil
}
// UpdateCache сохраняет в кэш ответ от сервиса celebratoday и временной штамп.
func UpdateCache(cfg *config.MyConfig, eTime int64, value string) error {
key := "celebratoday_timestamp"
timestamp := strconv.FormatInt(eTime, 10)
if err := pcachedb.SaveKeyWithValue(cfg, "cache", key, timestamp); err != nil {
return fmt.Errorf("CT-Cache: unable to save timestamp to cache: %w", err)
}
key = "celebratoday_value"
if err := pcachedb.SaveKeyWithValue(cfg, "cache", key, value); err != nil {
return fmt.Errorf("CT-Cache: unable to save answer to cache: %w", err)
}
return nil
}
// parser разбирает буфер с html-ем, выискивая праздники.
func parser(data string) ([]string, error) {
var (
holidays = make([]string, 0, 1)
err error
s = make([]Holy, 0, 1)
)
doc, err := html.Parse(strings.NewReader(data))
if err != nil {
return holidays, err
}
traverse(doc, &s)
// А теперь пробежимся по полученной структуре и вытянем из неё все строки с праздниками.
for _, ass := range s {
for _, entity := range ass.MainEntity {
for _, sugestedAnswer := range entity.SuggestedAnswer {
holidays = append(holidays, sugestedAnswer.Text)
}
}
}
return holidays, err
}
// traverse рекурсивная функция, ищущая <script type="application/ld+json"> в структуре документа. Там внутри будет json
// со списком праздников и это и есть цель нашего поиска.
func traverse(n *html.Node, h *[]Holy) {
// Если найденный элемент в поле Data содержит script (то есть это тэг <script>).
if n.Type == html.ElementNode && n.Data == "script" {
// Ищем атрибут type со значением application/ld+json.
for _, attr := range n.Attr {
if attr.Key == "type" && attr.Val == "application/ld+json" {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.TextNode {
// Добавлять рановато, надо вначале попробовать распарсить этот json
// используя целевую структуру.
var holiday Holy
if err := json.Unmarshal([]byte(c.Data), &holiday); err == nil {
*h = append(*h, holiday)
}
}
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
traverse(c, h)
}
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

39
internal/config/types.go Normal file
View File

@@ -0,0 +1,39 @@
// Package config нужен, чтобы разбить кольцевую зависимость.
package config
import (
"github.com/cockroachdb/pebble"
"github.com/go-redis/redis/v8"
)
type (
// MyConfig структурка, описывающая конфиг.
MyConfig struct { //nolint: govet // Идите в пень, мне так удобнее.
Server string `json:"server,omitempty"`
Port int `json:"port,omitempty"`
Timeout int `json:"timeout,omitempty"`
Loglevel string `json:"loglevel,omitempty"`
Log string `json:"log,omitempty"`
Channel string `json:"channel,omitempty"`
DataDir string `json:"datadir,omitempty"`
Csign string `json:"csign,omitempty"`
ForwardsMax int64 `json:"forwards_max,omitempty"`
OpenWeatherMap struct { //nolint: govet // Идите в пень, мне так удобнее.
Enabled bool `json:"enabled,omitempty"`
Country bool `json:"country,omitempty"`
Appid string `json:"appid,omitempty"`
} `json:"openweathermap"`
Flickr struct { //nolint: govet // Идите в пень, мне так удобнее.
Enabled bool `json:"enabled,omitempty"`
Key string `json:"key,omitempty"`
Secret string `json:"secret,omitempty"`
OAuthToken string `json:"oauth_token,omitempty"`
OAuthTokenSecret string `json:"oauth_token_secret,omitempty"`
} `json:"flickr"`
UserAgents []string `json:"user_agents,omitempty"`
PcacheDB map[string]*pebble.DB
RedisClient *redis.Client
}
)
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

82
internal/flickr/flickr.go
View File

@@ -1,8 +1,11 @@
// Package flickr implements flickr.com service client.
package flickr
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/pcachedb" //nolint: gosec // Just go fuck yourself if you don't see difference between hashing and cryptography.
"aleesa-webapp-go/internal/pcachedb"
"errors"
"strconv"
"context"
"fmt"
@@ -15,27 +18,29 @@ import (
"gopkg.in/masci/flickr.v3/test"
)
// SearchResult структура, описывающая ответ api на поисковой запрос.
type SearchResult struct {
Photos struct {
Page int64 `json:"page,omitempty"`
Pages int64 `json:"pages,omitempty"`
Perpage int64 `json:"perpage,omitempty"`
Total int64 `json:"total,omitempty"`
Photo []struct {
Farm int64 `json:"farm,omitempty"`
ID string `json:"id,omitempty"`
Isfamily int64 `json:"isfamily,omitempty"`
Isfriend int64 `json:"isfriend,omitempty"`
Ispublic int64 `json:"ispublic,omitempty"`
Owner string `json:"owner,omitempty"`
Secret string `json:"secret,omitempty"`
Server string `json:"server,omitempty"`
Title string `json:"title,omitempty"`
} `json:"photo,omitempty"`
} `json:"photos,omitempty"`
Stat string `json:"stat,omitempty"`
}
type (
// SearchResult структура, описывающая ответ api на поисковой запрос.
SearchResult struct { //nolint: govet // Идите в пень, мне так удобнее.
Photos struct { //nolint: govet // Идите в пень, мне так удобнее.
Page int64 `json:"page,omitempty"`
Pages int64 `json:"pages,omitempty"`
Perpage int64 `json:"perpage,omitempty"`
Total int64 `json:"total,omitempty"`
Photo []struct { //nolint: govet // Идите в пень, мне так удобнее.
Farm int64 `json:"farm,omitempty"`
ID string `json:"id,omitempty"`
Isfamily int64 `json:"isfamily,omitempty"`
Isfriend int64 `json:"isfriend,omitempty"`
Ispublic int64 `json:"ispublic,omitempty"`
Owner string `json:"owner,omitempty"`
Secret string `json:"secret,omitempty"`
Server string `json:"server,omitempty"`
Title string `json:"title,omitempty"`
} `json:"photo"`
} `json:"photos"`
Stat string `json:"stat,omitempty"`
}
)
// APIClientInit генерит и сохраняет token и token secret с помощью даденных пользователем key и secret.
// В процессе выдаём юзеру в консоль урл и запрос открыть его в браузере, чтобы получить verifier key, который нам
@@ -45,15 +50,15 @@ func APIClientInit(cfg *config.MyConfig) error {
client := flickr.NewFlickrClient(cfg.Flickr.Key, cfg.Flickr.Secret)
// Первым делом достанем request token
// Первым делом достанем request token.
tok, err := flickr.GetRequestToken(client)
if err != nil {
return fmt.Errorf("unable to get Request Token: %w", err)
}
// Соорудим авторизационный URL
url, err := flickr.GetAuthorizeUrl(client, tok)
// Соорудим авторизационный URL.
link, err := flickr.GetAuthorizeUrl(client, tok)
if err != nil {
return fmt.Errorf("unable to get auth URL: %w", err)
@@ -61,11 +66,12 @@ func APIClientInit(cfg *config.MyConfig) error {
// А теперь ручные действия пользователя.
// Выдаём ему url, который надо ввести в адресную строку браузера, в браузере сгенерится verifier code, который нам
// нужен, чтобы вынуть из flicker api oauth token, secret, access token
// нужен, чтобы вынуть из flicker api oauth token, secret, access token.
var oauthVerifier string
fmt.Println("Open your browser at this url:", url)
fmt.Println("Open your browser at this url:", link)
fmt.Print("Then, insert the code: ")
_, _ = fmt.Scanln(&oauthVerifier)
// Наконец, достаём Access Token и Aceess Token Secret и сохраняем их в базку на диске.
@@ -125,7 +131,8 @@ func SearchByTags(cfg *config.MyConfig, tags []string) (string, error) {
)
// Заполняем поля для оаутха.
c.Init() // c.EndpointUrl
c.Init()
c.OAuthToken = oauthToken
c.OAuthTokenSecret = oauthSecret
@@ -150,17 +157,18 @@ func SearchByTags(cfg *config.MyConfig, tags []string) (string, error) {
// разметки, а во-вторых, этот самый flickr.DoGet() валится с segfault-ом, просто потому что. Схема не такая, на
// которую он рассчитан. Но нам повезло, что можно просто вытащить подписанный url и заправить его в свой
// http-клиент.
url := c.GetUrl()
link := c.GetUrl()
var response SearchResult
err := requests.
URL(url).
URL(link).
UserAgent(userAgent).
ToJSON(&response).
Fetch(ctx)
if response.Stat == "fail" {
return "", fmt.Errorf("search request returns no results and fails")
return "", errors.New("search request returns no results and fails")
}
// Ну, вот мы и получили ответ на вопрос - а сколько же у нас результатов поиска всего, по заданному критерию.
@@ -171,15 +179,15 @@ func SearchByTags(cfg *config.MyConfig, tags []string) (string, error) {
}
randomPage := rand.Int64N(response.Photos.Pages)
c.Args.Set("page", fmt.Sprintf("%d", randomPage))
c.Args.Set("page", strconv.FormatInt(randomPage, 10))
// Подписываем запрос oauth-говном.
c.OAuthSign()
url = c.GetUrl()
link = c.GetUrl()
err = requests.
URL(url).
URL(link).
UserAgent(userAgent).
ToJSON(&response).
Fetch(ctx)
@@ -189,7 +197,7 @@ func SearchByTags(cfg *config.MyConfig, tags []string) (string, error) {
}
if response.Stat == "fail" {
return "", fmt.Errorf("search request returns no results and fails")
return "", errors.New("search request returns no results and fails")
}
// Частенько выборка не очень уникальна, много дублей. Чтобы почистить выборку, неплохо бы выбирать только те фотки,
@@ -197,10 +205,10 @@ func SearchByTags(cfg *config.MyConfig, tags []string) (string, error) {
var u SearchResult
for i := 0; i < len(response.Photos.Photo); i++ {
for i := range len(response.Photos.Photo) {
skip := false
for j := 0; j < len(u.Photos.Photo); j++ {
for j := range len(u.Photos.Photo) {
if response.Photos.Photo[i].Title == u.Photos.Photo[j].Title {
skip = true

43
internal/lib/types.go
View File

@@ -1,43 +0,0 @@
package lib
// RMsg структурка, описывающая входящее сообщение из pubsub-канала redis-ки.
type RMsg struct {
From string `json:"from,omitempty"`
Chatid string `json:"chatid,omitempty"`
Userid string `json:"userid,omitempty"`
Threadid string `json:"threadid,omitempty"`
Message string `json:"message,omitempty"`
Plugin string `json:"plugin,omitempty"`
Mode string `json:"mode,omitempty"`
Misc struct {
Answer int64 `json:"answer,omitempty"`
Botnick string `json:"bot_nick,omitempty"`
Csign string `json:"csign,omitempty"`
Fwdcnt int64 `json:"fwd_cnt,omitempty"`
GoodMorning int64 `json:"good_morning,omitempty"`
Msgformat int64 `json:"msg_format,omitempty"`
Username string `json:"username,omitempty"`
} `json:"Misc"`
}
// SMsg структурка, описывающая исходящее сообщение в pubsub-канал redis-ки.
type SMsg struct {
From string `json:"from"`
Chatid string `json:"chatid"`
Userid string `json:"userid"`
Threadid string `json:"threadid"`
Message string `json:"message"`
Plugin string `json:"plugin"`
Mode string `json:"mode"`
Misc struct {
Answer int64 `json:"answer"`
Botnick string `json:"bot_nick"`
Csign string `json:"csign"`
Fwdcnt int64 `json:"fwd_cnt"`
GoodMorning int64 `json:"good_morning"`
Msgformat int64 `json:"msg_format"`
Username string `json:"username"`
} `json:"misc"`
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

106
internal/log/log.go Normal file
View File

@@ -0,0 +1,106 @@
// Package log implements logging facility as golang slog module wrapper.
package log
import (
"fmt"
"log/slog"
"os"
"time"
)
// Init setup logger stuff.
// level can be error, warn, info, debug if something other supplied info level selected.
// fileDescriptor should be opened before supplying it to Init().
func Init(level string, fileDescriptor *os.File) {
var loglevel slog.Level
// no panic, no trace.
switch level {
case "error":
loglevel = slog.LevelError
case "warn":
loglevel = slog.LevelWarn
case "info":
loglevel = slog.LevelInfo
case "debug":
loglevel = slog.LevelDebug
default:
loglevel = slog.LevelInfo
}
opts := &slog.HandlerOptions{
// Use the ReplaceAttr function on the handler options
// to be able to replace any single attribute in the log output.
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { //nolint: revive
// Check that we are handling the time key.
if a.Key != slog.TimeKey {
return a
}
t := a.Value.Time()
// change the value from a time.Time to a String
// where the string has the correct time format.
a.Value = slog.StringValue(t.Format(time.DateTime))
return a
},
Level: loglevel,
}
slog.SetDefault(
slog.New(
slog.NewTextHandler(
fileDescriptor,
opts,
),
),
)
}
// Error passes message directly to [slog.Error()].
func Error(message string) {
slog.Error(message)
}
// Errorf logs message on error log level and allows to supply arguments in printf() style.
func Errorf(format string, a ...any) {
slog.Error(fmt.Sprintf(format, a...))
}
// Warn passes message directly to [slog.Warn()].
func Warn(message string) {
slog.Warn(message)
}
// Warnf logs message on warn log level and allows to supply arguments in printf() style.
func Warnf(format string, a ...any) {
slog.Warn(fmt.Sprintf(format, a...))
}
// Info passes message directly to [slog.Info()].
func Info(message string) {
slog.Info(message)
}
// Infof logs message on info log level and allows to supply arguments in printf() style.
func Infof(format string, a ...any) {
slog.Info(fmt.Sprintf(format, a...))
}
// Debug passes message directly to [slog.Debug()].
func Debug(message string) {
slog.Debug(message)
}
// Debugf logs message on debug log level and allows to supply arguments in printf() style.
func Debugf(format string, a ...any) {
slog.Debug(fmt.Sprintf(format, a...))
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

29
internal/monkeyuser/monkeyuser.go
View File

@@ -1,3 +1,4 @@
// Package monkeyuser implements client for service www.monkeyusers.com.
package monkeyuser
import (
@@ -5,6 +6,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"math/rand/v2"
"strings"
@@ -13,12 +15,15 @@ import (
"golang.org/x/net/html"
)
// MonkeyUsers описывает структуру json-массива ответа www.monkeyusers.com.
type MonkeyUsers []struct {
Title string `json:"title"`
URL string `json:"url"`
}
type (
// MonkeyUsers описывает структуру json-массива ответа www.monkeyusers.com.
MonkeyUsers []struct {
Title string `json:"title"`
URL string `json:"url"`
}
)
// APIClient клиент сервиса monkeyuser.com.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
@@ -27,7 +32,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
body string
userAgent = cfg.UserAgents[rand.IntN(len(cfg.UserAgents))]
baseURL = "https://www.monkeyuser.com"
indexURL = fmt.Sprintf("%s/index.json", baseURL)
indexURL = baseURL + "/index.json"
)
err := requests.
@@ -73,7 +78,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
return "", fmt.Errorf("unable to GET %s: %w", pageURL, err)
}
// Парсим body ответа от сервера
// Парсим body ответа от сервера.
doc, err := html.Parse(strings.NewReader(body))
if err != nil {
@@ -87,9 +92,9 @@ func APIClient(cfg *config.MyConfig) (string, error) {
link = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "img" {
for _, a_href := range n.Attr { //nolint: revive,stylecheck
for _, a_href := range n.Attr { //nolint: revive
// Если у нас class == "lazyload small-image", то после него ключ data-src должен содержать
// искомый relative url
// искомый relative url.
if a_href.Key == "class" && a_href.Val == "logo" {
continue
}
@@ -103,7 +108,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
}
}
// traverses the HTML of the webpage from the first child node
// traverses the HTML of the webpage from the first child node.
for c := n.FirstChild; c != nil; c = c.NextSibling {
link(c)
}
@@ -111,12 +116,12 @@ func APIClient(cfg *config.MyConfig) (string, error) {
link(doc)
// Возвращаем ссылку на картинку buni comic
// Возвращаем ссылку на картинку buni comic.
if hyperlink != "" {
return fmt.Sprintf("%s%s", baseURL, hyperlink), nil
}
err = fmt.Errorf("no links found on monkeyusers.com")
err = errors.New("no links found on monkeyusers.com")
return "", err
}

34
internal/oboobs/oboobs.go
View File

@@ -1,3 +1,4 @@
// Package oboobs implements api.oboobs.ru service client.
package oboobs
import (
@@ -9,15 +10,18 @@ import (
"github.com/carlmjohnson/requests"
)
// Oboobs структура, возвращаемая на запрос в oboobs.ru.
type Oboobs struct {
ID int `json:"id,omitempty"`
Author string `json:"author,omitempty"`
Rank int `json:"rank,omitempty"`
Model string `json:"model,omitempty"`
Preview string `json:"preview"`
}
type (
// Oboobs структура, возвращаемая на запрос в oboobs.ru.
Oboobs struct { //nolint: govet // Идите в пень, мне так удобнее.
ID int `json:"id,omitempty"`
Author string `json:"author,omitempty"`
Rank int `json:"rank,omitempty"`
Model string `json:"model,omitempty"`
Preview string `json:"preview"`
}
)
// APIClient клиент сервиса oboobs.ru.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
@@ -26,23 +30,15 @@ func APIClient(cfg *config.MyConfig) (string, error) {
boobs []Oboobs
)
err := requests.
URL(url).
UserAgent(userAgent).
ToJSON(&boobs).
Fetch(ctx)
if err != nil {
if err := requests.URL(url).UserAgent(userAgent).ToJSON(&boobs).Fetch(ctx); err != nil {
return "", fmt.Errorf("unable to GET %s: %w", url, err)
}
if len(boobs) == 0 {
err = fmt.Errorf("empty json array returned from %s", url)
return "", err
return "", fmt.Errorf("empty json array returned from %s", url)
}
answer := fmt.Sprintf("https://media.oboobs.ru/%s", boobs[0].Preview)
answer := "https://media.oboobs.ru/%s" + boobs[0].Preview
return answer, nil
}

22
internal/obutts/obutts.go
View File

@@ -1,3 +1,4 @@
// Package obutts implements api.obutts.ru service client.
package obutts
import (
@@ -9,15 +10,18 @@ import (
"github.com/carlmjohnson/requests"
)
// Obutts структура, возвращаемая на запрос в obutts.ru.
type Obutts struct {
ID int `json:"id,omitempty"`
Author string `json:"author,omitempty"`
Rank int `json:"rank,omitempty"`
Model string `json:"model,omitempty"`
Preview string `json:"preview"`
}
type (
// Obutts структура, возвращаемая на запрос в obutts.ru.
Obutts struct { //nolint: govet // Идите в пень, мне так удобнее.
ID int `json:"id,omitempty"`
Author string `json:"author,omitempty"`
Rank int `json:"rank,omitempty"`
Model string `json:"model,omitempty"`
Preview string `json:"preview"`
}
)
// APIClient клиент сервиса obutts.ru.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
@@ -42,7 +46,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
return "", err
}
answer := fmt.Sprintf("http://media.obutts.ru/%s", butts[0].Preview)
answer := "http://media.obutts.ru/" + butts[0].Preview
return answer, nil
}

77
internal/openweathermap/openweathermap.go
View File

@@ -1,8 +1,11 @@
// Package openweathermap implements openweatermap.org service client.
package openweathermap
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/log"
"aleesa-webapp-go/internal/pcachedb"
"crypto/sha256"
"fmt"
"net/http"
"strconv"
@@ -10,17 +13,18 @@ import (
"time"
owm "github.com/briandowns/openweathermap"
log "github.com/sirupsen/logrus"
)
// Структурка для передачи в обрабтчик запросов в openweathermap api.
type OwmItem struct {
City string
FakeName string
Where string
Lon float64
Lat float64
}
type (
// OwmItem структурка для передачи в обрабтчик запросов в openweathermap api.
OwmItem struct {
City string
FakeName string
Where string
Lon float64
Lat float64
}
)
// OwmClient обёртка с кэшированием для openweatermap.org.
func OwmClient(cfg *config.MyConfig, city string, days uint32) (string, error) {
@@ -40,23 +44,29 @@ func OwmClient(cfg *config.MyConfig, city string, days uint32) (string, error) {
// "Нормализуем" название города.
city = strings.TrimSpace(city)
firstLetter := strings.SplitN(city, "", 2)[0]
city = fmt.Sprintf("%s%s", strings.ToUpper(firstLetter), city[1:])
city = strings.ToValidUTF8(city, "")
switch city {
case "Msk", "Default", "Dc", "Мск", "Dc-Universe":
item.City = "Москва"
case "Spb", "Спб":
item.City = "Санкт-Петербург"
case "Ект", "Ёбург", "Ебург", "Екат", "Ekt", "Eburg", "Ekat":
item.City = "Екатеринбург"
case "Кудыкина гора":
// Координаты Килиманжаро (-3.066524619045895, 37.35588473583672).
item.Lat = -3.066524619045895
item.Lon = 37.35588473583672
item.FakeName = "Килиманжаро"
item.Where = "на горе"
default:
item.City = city
item.Where = "в городе"
@@ -95,8 +105,8 @@ func OwmAPIClient(cfg *config.MyConfig, item OwmItem) (string, error) {
}
if item.City != "" {
if err = w.CurrentByName(item.City); err != nil {
// TODO: extend error
if err := w.CurrentByName(item.City); err != nil {
// TODO: extend error.
return answer, err
}
@@ -104,8 +114,8 @@ func OwmAPIClient(cfg *config.MyConfig, item OwmItem) (string, error) {
} else {
coord := owm.Coordinates{Longitude: item.Lon, Latitude: item.Lat}
if err = w.CurrentByCoordinates(&coord); err != nil {
// TODO: extend error
if err := w.CurrentByCoordinates(&coord); err != nil {
// TODO: extend error.
return answer, err
}
@@ -227,17 +237,17 @@ func OwmAPIClient(cfg *config.MyConfig, item OwmItem) (string, error) {
// UpdateOwmCache обновляет кэш.
func UpdateOwmCache(cfg *config.MyConfig, tsNowUnix int64, entry string, value string) error {
key := fmt.Sprintf("%s+timestamp", entry)
key := entry + "+timestamp"
// "Кэшируем" на 3 часа.
ts := tsNowUnix + int64((3 * time.Hour).Seconds())
timestamp := fmt.Sprintf("%d", ts)
timestamp := strconv.FormatInt(ts, 10)
if err := pcachedb.SaveKeyWithValue(cfg, "cache", key, timestamp); err != nil {
return fmt.Errorf("OWM-Cache: unable to save timestamp to cache: %w", err)
}
key = fmt.Sprintf("%s+value", entry)
key = entry + "+value"
if err := pcachedb.SaveKeyWithValue(cfg, "cache", key, value); err != nil {
return fmt.Errorf("OWM-Cache: unable to save answer to cache: %w", err)
@@ -290,6 +300,7 @@ func QueryOwmCache(cfg *config.MyConfig, item OwmItem) (string, error) {
// Первое, что приходит в голову - коллизия хэшей sha256.
if err != nil {
log.Warnf("OWM-Cache: unable to parse string as int64 (sha256 collision?): %s", err)
answer, err := OwmAPIClient(cfg, item)
// Бинго! у нас ещё и ошибка при обращении к api openwethermap, здесь мы ничего сделать не можем.
@@ -305,7 +316,7 @@ func QueryOwmCache(cfg *config.MyConfig, item OwmItem) (string, error) {
return answer, err
}
// Кэш просрочен
// Кэш просрочен.
if tsCache < tsNow {
log.Debugf("OWM cache miss for city: %s", key)
@@ -318,13 +329,14 @@ func QueryOwmCache(cfg *config.MyConfig, item OwmItem) (string, error) {
// Если всё хорошо, надо обновить кэш.
if err := UpdateOwmCache(cfg, tsNow, key, answer); err != nil {
log.Error(err)
log.Errorf("%s", err)
}
return answer, err
}
key += "+value"
answer = pcachedb.GetValue(cfg, "cache", key)
// Какие-то проблемы с кэшом - ts есть, но самого кэша нету. Более подробная инфа по идее уже попала в лог.
@@ -333,7 +345,7 @@ func QueryOwmCache(cfg *config.MyConfig, item OwmItem) (string, error) {
if len(answer) < 140 {
log.Debugf("OWM cache miss for city: %s", key)
if len(answer) > 0 {
if answer != "" {
log.Errorf("OWM-Cache: record for city %s in cache is too short, less than 140 chars", key)
}
@@ -357,4 +369,31 @@ func QueryOwmCache(cfg *config.MyConfig, item OwmItem) (string, error) {
return answer, err
}
// UpdateOwmUserCache записывает в кэш, кто каким городом интересовался в последний раз, когда спрашивал о погоде.
func UpdateOwmUserCache(cfg *config.MyConfig, chatid string, entry string, value string) error {
var err error
// Chatid как есть лучше на диск не пробовать класть, мало ли что туда юзер впишет. Возьмём от него sha256.
data := []byte(chatid)
hash := fmt.Sprintf("%x", sha256.Sum256(data))
if err = pcachedb.SaveKeyWithValue(cfg, hash, entry, value); err != nil {
return fmt.Errorf("OWM-Cache: unable to save timestamp to cache: %w", err)
}
return err
}
// QueryOwmUserCache возвращает из кэша город, которым интересовался пользователь из даденного чятика.
func QueryOwmUserCache(cfg *config.MyConfig, chatid string, entry string) string {
// Chatid как есть лучше на диск не пробовать класть, мало ли что туда юзер впишет. Возьмём от него sha256.
var (
data = []byte(chatid)
hash = fmt.Sprintf("%x", sha256.Sum256(data))
value = pcachedb.GetValue(cfg, hash, entry)
)
return value
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

17
internal/pcachedb/pcachedb.go
View File

@@ -1,7 +1,9 @@
// Package pcachedb implements persistent cache using PebbleDB key-value library.
package pcachedb
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/log"
"crypto/sha256"
"errors"
"fmt"
@@ -9,7 +11,6 @@ import (
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble"
log "github.com/sirupsen/logrus"
)
// StoreKV сохраняет в указанной бд ключ и значение.
@@ -51,12 +52,12 @@ func GetValue(cfg *config.MyConfig, cacheID string, setting string) string {
cacheHash := sha256.Sum256([]byte(cacheID))
database := fmt.Sprintf("cache_db/%x", cacheHash)
// Если БД не открыта, откроем её
// Если БД не открыта, откроем её.
if _, ok := cfg.PcacheDB[database]; !ok {
var options pebble.Options
// По дефолту ограничение ставится на мегабайты данных, а не на количество файлов, поэтому с дефолтными
// настройками порождается огромное количество файлов. Умолчальное ограничение на количество файлов - 500 штук,
// что нас не устраивает, поэтому немного снизим эту цифру до более приемлемых значений
// что нас не устраивает, поэтому немного снизим эту цифру до более приемлемых значений.
options.L0CompactionFileThreshold = 8
cfg.PcacheDB[database], err = pebble.Open(cfg.DataDir+"/"+database, &options)
@@ -88,7 +89,7 @@ func GetValue(cfg *config.MyConfig, cacheID string, setting string) string {
return value
}
// saveKeyWithValue сохраним данные в БД.
// SaveKeyWithValue сохраним данные в БД.
func SaveKeyWithValue(cfg *config.MyConfig, cacheID string, setting string, value string) error {
var (
cacheHash = sha256.Sum256([]byte(cacheID))
@@ -96,7 +97,7 @@ func SaveKeyWithValue(cfg *config.MyConfig, cacheID string, setting string, valu
err error
)
// Если БД не открыта, откроем её
// Если БД не открыта, откроем её.
if _, ok := cfg.PcacheDB[database]; !ok {
var options pebble.Options
// По дефолту ограничение ставится на мегабайты данных, а не на количество файлов, поэтому с дефолтными
@@ -115,6 +116,12 @@ func SaveKeyWithValue(cfg *config.MyConfig, cacheID string, setting string, valu
if err := StoreKV(cfg.PcacheDB[database], setting, value); err != nil {
log.Errorf("Unable to save setting %s in database %s: %s", setting, database, err)
return err
}
if _, err = cfg.PcacheDB[database].AsyncFlush(); err != nil {
log.Errorf("Unable to flush data to disk for database %s: %s", database, err)
}
return err

63
internal/prazdnikisegodnyaru/prazdnikisegodnyaru.go
View File

@@ -1,7 +1,10 @@
// Package prazdnikisegodnyaru implements prazdniki-segodnya.ru service client.
// Now defunct on Russian Federation area for some legal reasons.
package prazdnikisegodnyaru
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/log"
"aleesa-webapp-go/internal/pcachedb"
"context"
"errors"
@@ -12,20 +15,21 @@ import (
"time"
"github.com/carlmjohnson/requests"
log "github.com/sirupsen/logrus"
"golang.org/x/net/html"
)
// PsrClient обёртка для клиента сервиса prazdniki-segodnya.ru, работающая с кэшем. Запросы надо делать именно через неё.
func PsrClient(cfg *config.MyConfig) (string, error) {
tsNow := time.Now()
// Вычислим expiration time для MSK+0 в формате unix timestamp
// Вычислим expiration time для MSK+0 в формате unix timestamp.
loc, _ := time.LoadLocation("Europe/Moscow")
year, month, day := tsNow.In(loc).Date()
eTime := time.Date(year, month, day, 0, 0, 0, 0, loc).AddDate(0, 0, +1).Unix()
tsStr := pcachedb.GetValue(cfg, "cache", "prazdnikisegodhyaru_timestamp")
// Кэш пустой, запросов пока не было
// Кэш пустой, запросов пока не было.
if tsStr == "" {
log.Debugf("PSR no cache entry for prazdnikisegodhyaru")
@@ -36,7 +40,7 @@ func PsrClient(cfg *config.MyConfig) (string, error) {
}
if err := UpdatePsrCache(cfg, eTime, answer); err != nil {
log.Error(err)
log.Errorf("%s", err)
}
return answer, nil
@@ -47,76 +51,71 @@ func PsrClient(cfg *config.MyConfig) (string, error) {
if err != nil {
log.Debug("PSR cache miss")
log.Warnf("PSR-Cache: unable to parse string as int64 (sha256 collision?): %s", err)
answer, err := PsrAPIClient(cfg)
// Бинго! у нас ещё и ошибка при обращении к "api" prazdnikisegodhyaru, здесь мы ничего сделать не можем
// Бинго! у нас ещё и ошибка при обращении к "api" prazdnikisegodhyaru, здесь мы ничего сделать не можем.
if err != nil {
return "", err
}
// Если всё хорошо, надо обновить кэш
// Если всё хорошо, надо обновить кэш.
if err := UpdatePsrCache(cfg, eTime, answer); err != nil {
log.Error(err)
log.Errorf("%s", err)
}
return answer, nil
}
// Текущее время в Unix timestamp
// Текущее время в Unix timestamp.
tsNowUnix := tsNow.Unix()
// Кэш просрочен
// Кэш просрочен.
if tsCache < tsNowUnix {
log.Debug("PSR cache miss")
answer, err := PsrAPIClient(cfg)
// Здесь мы ничего сделать не можем
// Здесь мы ничего сделать не можем.
if err != nil {
return "", err
}
// Если всё хорошо, надо обновить кэш
// Если всё хорошо, надо обновить кэш.
if err := UpdatePsrCache(cfg, eTime, answer); err != nil {
log.Error(err)
log.Errorf("%s", err)
}
return answer, nil
}
// TODO: как-то научиться определять ситуацию с отсутствующим значением в кэше
// TODO: Научиться определять ситуацию, когда значением является unix timestamp
// TODO: как-то научиться определять ситуацию с отсутствующим значением в кэше.
// TODO: Научиться определять ситуацию, когда значением является unix timestamp.
log.Debug("PSR cache hit")
return pcachedb.GetValue(cfg, "cache", "prazdnikisegodhyaru_value"), nil
}
// PsrAPIClient клиент сервиса prazdniki-segodnya.ru.
func PsrAPIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
url = "https://prazdniki-segodnya.ru/"
link = "https://prazdniki-segodnya.ru/"
respBody string
)
// Формальная попытка притвориться браузером
// Формальная попытка притвориться браузером.
_, _, day := time.Now().Date()
userAgent := cfg.UserAgents[day]
err := requests.
URL(url).
UserAgent(userAgent).
Header("User-Agent", "ru-RU").
Header("Accept-Charset", "utf-8").
ToString(&respBody).
Fetch(ctx)
if err != nil {
return "", fmt.Errorf("unable to GET %s: %w", url, err)
if err := requests.URL(link).UserAgent(userAgent).Header("User-Agent", "ru-RU").
Header("Accept-Charset", "utf-8").ToString(&respBody).Fetch(ctx); err != nil {
return "", fmt.Errorf("unable to GET %s: %w", link, err)
}
var holidays []string
// Парсим body ответа от сервера
// Парсим body ответа от сервера.
tokenizer := html.NewTokenizer(strings.NewReader(respBody))
for {
@@ -126,7 +125,7 @@ func PsrAPIClient(cfg *config.MyConfig) (string, error) {
err := tokenizer.Err()
if errors.Is(err, io.EOF) {
// end of the file, break out of the loop
// end of the file, break out of the loop.
break
}
@@ -143,6 +142,7 @@ func PsrAPIClient(cfg *config.MyConfig) (string, error) {
if tokenType == html.TextToken {
str := tokenizer.Token().Data
holidays = append(holidays, strings.TrimSpace(str))
}
}
@@ -152,17 +152,16 @@ func PsrAPIClient(cfg *config.MyConfig) (string, error) {
}
if len(holidays) == 0 {
err = errors.New("unable to parse response from prazdniki-segodnya.ru: no holidays found")
return "", err
return "", errors.New("unable to parse response from prazdniki-segodnya.ru: no holidays found")
}
return "* " + strings.Join(holidays, "\n* "), nil
}
// UpdatePsrCache сохраняет в кэш ответ от сервиса prazdniki-segodnya.ru и временной штамп.
func UpdatePsrCache(cfg *config.MyConfig, eTime int64, value string) error {
key := "prazdnikisegodhyaru_timestamp"
timestamp := fmt.Sprintf("%d", eTime)
timestamp := strconv.FormatInt(eTime, 10)
if err := pcachedb.SaveKeyWithValue(cfg, "cache", key, timestamp); err != nil {
return fmt.Errorf("PSR-Cache: unable to save timestamp to cache: %w", err)

14
internal/randomfox/randomfox.go
View File

@@ -1,3 +1,4 @@
// Package randomfox implements randomfox.ca service client.
package randomfox
import (
@@ -10,12 +11,15 @@ import (
"github.com/carlmjohnson/requests"
)
// RandomFox структура, возвращаемая на запрос в randomfox.ca API.
type RandomFox struct {
Image string `json:"image,omitempty"`
Link string `json:"link"`
}
type (
// RandomFox структура, возвращаемая на запрос в randomfox.ca API.
RandomFox struct {
Image string `json:"image,omitempty"`
Link string `json:"link"`
}
)
// APIClient клиент сервиса randomfox.ca.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()

28
internal/thecatapi/thecatapi.go
View File

@@ -1,3 +1,4 @@
// Package thecatapi implements api.thecatapi.com service client.
package thecatapi
import (
@@ -9,17 +10,20 @@ import (
"github.com/carlmjohnson/requests"
)
// TheCatAPIinnerStruct структурка, описывающая элемент массива, возвращемый Тhe cat api.
type TheCatAPIinnerStruct struct {
ID string `json:"id,omitempty"`
URL string `json:"url"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
}
type (
// TheCatAPIinnerStruct структурка, описывающая элемент массива, возвращемый Тhe cat api.
TheCatAPIinnerStruct struct {
ID string `json:"id,omitempty"`
URL string `json:"url"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
}
// TheCatAPI структурка, описывающая массив, возвращаемый на запрос урла картинки из The Cat API.
type TheCatAPI []TheCatAPIinnerStruct
// TheCatAPI структурка, описывающая массив, возвращаемый на запрос урла картинки из The Cat API.
TheCatAPI []TheCatAPIinnerStruct
)
// APIClient клиент сервиса thecatapi.com.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
ctx = context.Background()
@@ -28,11 +32,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
cat TheCatAPI
)
err := requests.
URL(url).
UserAgent(userAgent).
ToJSON(&cat).
Fetch(ctx)
err := requests.URL(url).UserAgent(userAgent).ToJSON(&cat).Fetch(ctx)
if err != nil {
return "", fmt.Errorf("unable to GET %s: %w", url, err)

203
internal/lib/aleesa-webapp-lib.go → internal/webapp/MsgParser.go
View File

@@ -1,15 +1,17 @@
package lib
// Package webapp is main package implementing application logic.
package webapp
import (
"aleesa-webapp-go/internal/anekdotru"
"aleesa-webapp-go/internal/bunicomic"
"aleesa-webapp-go/internal/celebratoday"
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/flickr"
"aleesa-webapp-go/internal/log"
"aleesa-webapp-go/internal/monkeyuser"
"aleesa-webapp-go/internal/oboobs"
"aleesa-webapp-go/internal/obutts"
"aleesa-webapp-go/internal/openweathermap"
"aleesa-webapp-go/internal/prazdnikisegodnyaru"
"aleesa-webapp-go/internal/randomfox"
"aleesa-webapp-go/internal/thecatapi"
"aleesa-webapp-go/internal/xkcdru"
@@ -19,12 +21,10 @@ import (
"math/rand/v2"
"regexp"
"time"
log "github.com/sirupsen/logrus"
)
// MsgParser горутинка, которая парсит json-чики прилетевшие из REDIS-ки.
func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) { //nolint: revive
var (
sendTo string
answer string
@@ -45,12 +45,12 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
j, err = ValidateRmsg(cfg, j, msg)
if err != nil {
log.Warn(err)
log.Warnf("%s", err)
return
}
// Если у нас циклическая пересылка сообщения, попробуем её тут разорвать, отбросив сообщение
// Если у нас циклическая пересылка сообщения, попробуем её тут разорвать, отбросив сообщение.
if j.Misc.Fwdcnt > cfg.ForwardsMax {
log.Warnf("Discarding msg with fwd_cnt exceeding max value: %s", msg)
@@ -61,15 +61,15 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
sendTo = j.Plugin
// Классифицируем входящие сообщения. Первым делом, попробуем определить команды
// Классифицируем входящие сообщения. Первым делом, попробуем определить команды.
if j.Message[0:len(j.Misc.Csign)] == j.Misc.Csign {
var cmd = j.Message[len(j.Misc.Csign):]
switch {
case cmd == "cat" || cmd == "кис":
// Пытаемся вычитать ответ thecatapi 3 раза
for i := 0; i < 3; i++ {
answer, err = thecatapi.APIClient(cfg)
// Пытаемся вычитать ответ thecatapi 3 раза.
for i := range 3 {
answer, err = thecatapi.APIClient(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query api.thecatapi.com: %v", i+1, err)
@@ -84,18 +84,20 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
if answer == "" {
j.Message = "Нету кошечек, все разбежались."
} else {
// for telegram return answer in markdown format
// for telegram return answer in markdown format.
if j.Plugin == "telegram" {
cats := []string{"龴ↀ◡ↀ龴", "=^..^=", "≧◔◡◔≦", "^ↀᴥↀ^"}
cats := []string{"龴ↀ◡ↀ龴", "=^..^=", "≧◔◡◔≦", "^ↀᴥↀ^"} //nolint: gosmopolitan
j.Message = fmt.Sprintf("[%s](%s)", cats[rand.IntN(len(cats))], answer)
} else {
j.Message = answer
}
}
case cmd == "fox" || cmd == "лис":
// Пытаемся вычитать ответ randonfox.ca 3 раза
for i := 0; i < 3; i++ {
answer, err = randomfox.APIClient(cfg)
// Пытаемся вычитать ответ randonfox.ca 3 раза.
for i := range 3 {
answer, err = randomfox.APIClient(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query randomfox.ca: %v", i+1, err)
@@ -116,9 +118,10 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
j.Message = answer
}
}
case cmd == "xkcd":
// Пытаемся вычитать ответ xkcd.ru 3 раза
for i := 0; i < 3; i++ {
// Пытаемся вычитать ответ xkcd.ru 3 раза.
for i := range 3 {
answer, err = xkcdru.APIClient(cfg)
if err != nil {
@@ -140,10 +143,11 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
j.Message = answer
}
}
case cmd == "buni":
// Пытаемся вычитать ответ www.bunicomic.com 3 раза
for i := 0; i < 3; i++ {
answer, err = bunicomic.APIClient(cfg)
// Пытаемся вычитать ответ www.bunicomic.com 3 раза.
for i := range 3 {
answer, err = bunicomic.APIClient(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query www.bunicomic.com: %v", i+1, err)
@@ -164,10 +168,11 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
j.Message = answer
}
}
case cmd == "anek" || cmd == "анек" || cmd == "анекдот":
// Пытаемся вычитать ответ www.anekdot.ru 3 раза
for i := 0; i < 3; i++ {
answer, err = anekdotru.APIClient(cfg)
// Пытаемся вычитать ответ www.anekdot.ru 3 раза.
for i := range 3 {
answer, err = anekdotru.APIClient(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query www.anekdot.ru: %v", i+1, err)
@@ -188,10 +193,11 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
j.Message = answer
}
}
case cmd == "monkeyuser":
// Пытаемся вычитать ответ www.bunicomic.com 3 раза
for i := 0; i < 3; i++ {
answer, err = monkeyuser.APIClient(cfg)
// Пытаемся вычитать ответ monkeyuser.com 3 раза.
for i := range 3 {
answer, err = monkeyuser.APIClient(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query monkeyuser.com: %v", i+1, err)
@@ -212,10 +218,11 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
j.Message = answer
}
}
case cmd == "butt" || cmd == "booty" || cmd == "ass" || cmd == "попа" || cmd == "попка":
// Пытаемся вычитать ответ api.obutts.ru 3 раза
for i := 0; i < 3; i++ {
answer, err = obutts.APIClient(cfg)
// Пытаемся вычитать ответ api.obutts.ru 3 раза.
for i := range 3 {
answer, err = obutts.APIClient(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query api.obutts.ru: %v", i+1, err)
@@ -232,15 +239,17 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
} else {
if j.Plugin == "telegram" {
arts := []string{"(__(__)", "(_!_)", "(__.__)"}
j.Message = fmt.Sprintf("[%s](%s)", arts[rand.IntN(len(arts))], answer)
} else {
j.Message = answer
}
}
case cmd == "titts" || cmd == "boobs" || cmd == "tities" || cmd == "boobies" || cmd == "сиси" || cmd == "сисечки":
// Пытаемся вычитать ответ api.oboobs.ru 3 раза
for i := 0; i < 3; i++ {
answer, err = oboobs.APIClient(cfg)
// Пытаемся вычитать ответ api.oboobs.ru 3 раза.
for i := range 3 {
answer, err = oboobs.APIClient(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query api.oboobs.ru: %v", i+1, err)
@@ -257,18 +266,20 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
} else {
if j.Plugin == "telegram" {
arts := []string{"(. )( .)", "( . Y . )", "(o)(o)", "( @ )( @ )", "(.)(.)"}
j.Message = fmt.Sprintf("[%s](%s)", arts[rand.IntN(len(arts))], answer)
} else {
j.Message = answer
}
}
case cmd == "drink" || cmd == "праздник":
// Пытаемся вычитать ответ prazdniki-segodnya.ru 3 раза
for i := 0; i < 3; i++ {
answer, err = prazdnikisegodnyaru.PsrClient(cfg)
// Пытаемся вычитать ответ celebratoday.com 3 раза.
for i := range 3 {
answer, err = celebratoday.Client(cfg) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query prazdniki-segodnya.ru: %v", i+1, err)
log.Errorf("Try %d/3 unable to query celebratoday.com: %v", i+1, err)
time.Sleep(1 * time.Second)
continue
@@ -284,8 +295,8 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
}
case cmd == "frog" || cmd == "лягушка":
for i := 0; i < 3; i++ {
answer, err = flickr.SearchByTags(cfg, []string{"frog", "toad", "amphibian"})
for i := range 3 {
answer, err = flickr.SearchByTags(cfg, []string{"frog", "toad", "amphibian"}) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query flickr api about frogs: %s", i+1, err)
@@ -303,6 +314,7 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
// Костыль для красивого ответа в telegram.
if j.Plugin == "telegram" {
art := []string{"frog", "toad", "лягушка"}
answer = fmt.Sprintf("[%s](%s)", art[rand.IntN(len(art))], answer)
}
@@ -310,8 +322,8 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
}
case cmd == "owl" || cmd == "сова" || cmd == "сыч":
for i := 0; i < 3; i++ {
answer, err = flickr.SearchByTags(cfg, []string{"owlet", "owl", "raptor", "bird of prey", "nocturnal bird", "barn owl", "tawny owl", "brown owl"})
for i := range 3 {
answer, err = flickr.SearchByTags(cfg, []string{"owlet", "owl", "raptor", "bird of prey", "nocturnal bird", "barn owl", "tawny owl", "brown owl"}) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query flickr api about owls: %s", i+1, err)
@@ -335,8 +347,8 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
}
case cmd == "horse" || cmd == "лошадь" || cmd == "лошадка":
for i := 0; i < 3; i++ {
answer, err = flickr.SearchByTags(cfg, []string{"horse", "equine", "stallion", "mare", "steed", "thoroughbred"})
for i := range 3 {
answer, err = flickr.SearchByTags(cfg, []string{"horse", "equine", "stallion", "mare", "steed", "thoroughbred"}) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query flickr api about horses: %s", i+1, err)
@@ -354,6 +366,7 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
// Костыль для красивого ответа в telegram.
if j.Plugin == "telegram" {
art := []string{"horse", "лошадь", "лошадка"}
answer = fmt.Sprintf("[%s](%s)", art[rand.IntN(len(art))], answer)
}
@@ -361,8 +374,8 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
}
case cmd == "rabbit" || cmd == "bunny" || cmd == "кролик":
for i := 0; i < 3; i++ {
answer, err = flickr.SearchByTags(cfg, []string{"bunny", "rabbit", "buck"})
for i := range 3 {
answer, err = flickr.SearchByTags(cfg, []string{"bunny", "rabbit", "buck"}) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query flickr api about rabbits: %s", i+1, err)
@@ -386,8 +399,8 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
}
case cmd == "snail" || cmd == "улитка":
for i := 0; i < 3; i++ {
answer, err = flickr.SearchByTags(cfg, []string{"snail", "slug"})
for i := range 3 {
answer, err = flickr.SearchByTags(cfg, []string{"snail", "slug"}) //nolint: contextcheck // Fuck off.
if err != nil {
log.Errorf("Try %d/3 unable to query flickr api about snails: %s", i+1, err)
@@ -405,12 +418,29 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
// Костыль для красивого ответа в telegram.
if j.Plugin == "telegram" {
art := []string{"'-'_@_", "@╜", "@_'-'"}
answer = fmt.Sprintf("[%s](%s)", art[rand.IntN(len(art))], answer)
}
j.Message = answer
}
case cmd == "w" || cmd == "п" || cmd == "погода" || cmd == "weather" || cmd == "погодка" || cmd == "погадка":
// TODO: научиться различать 5-дневные и моментальные прогнозы. Пока умеем только моментальные.
city := openweathermap.QueryOwmUserCache(cfg, j.Chatid, j.Userid)
if city == "" {
j.Message = "Не припоминаю, какой город вас интересовал в прошлый раз."
} else {
if answer, err := openweathermap.OwmClient(cfg, city, 0); err != nil {
log.Errorf("Unable to handle city %s in openweartermap api: %s", city, err)
j.Message = "Я не знаю, какая погода в " + city
} else {
j.Message = answer
}
}
case regexpWeather.MatchString(cmd):
re := regexp.MustCompile("[[:space:]]+")
cmdStr := re.Split(cmd, 2)[0]
@@ -419,19 +449,31 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
if regexp.MustCompile(`5$`).MatchString(cmdStr) {
if answer, err := openweathermap.OwmClient(cfg, city, 5); err != nil {
log.Errorf("Unable to handle city %s in openweartermap api: %s", city, err)
j.Message = fmt.Sprintf("Я не знаю, какая погода в %s", city)
j.Message = "Я не знаю, какая погода в " + city
} else {
j.Message = answer
}
} else {
if answer, err := openweathermap.OwmClient(cfg, city, 0); err != nil {
log.Errorf("Unable to handle city %s in openweartermap api: %s", city, err)
j.Message = fmt.Sprintf("Я не знаю, какая погода в %s", city)
j.Message = "Я не знаю, какая погода в " + city
} else {
j.Message = answer
}
}
if err = openweathermap.UpdateOwmUserCache(cfg, j.Chatid, j.Userid, city); err != nil {
log.Errorf(
"Unable to save value to cache, for chatid %s, userid %s and city %s: %s",
j.Chatid,
j.Userid,
city,
err,
)
}
default:
log.Errorf("Unknown command %s, unable to handle, skipping", j.Message)
@@ -443,8 +485,9 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
return
}
// Настало время формировать json и засылать его в дальше
// Настало время формировать json и засылать его в дальше.
var message SMsg
message.From = j.From
message.Userid = j.Userid
message.Chatid = j.Chatid
@@ -468,7 +511,7 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
return
}
// Заталкиваем наш json в редиску
// Заталкиваем наш json в редиску.
if err := cfg.RedisClient.Publish(ctx, sendTo, data).Err(); err != nil {
log.Warnf("Unable to send data to redis channel %s: %s", sendTo, err)
} else {
@@ -476,62 +519,4 @@ func MsgParser(cfg *config.MyConfig, ctx context.Context, msg string) {
}
}
// ValidateRmsg валидирует входящее сообщение.
func ValidateRmsg(cfg *config.MyConfig, j RMsg, msg string) (RMsg, error) {
if exist := j.From; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no from field: %s", msg)
return j, emsg
}
if exist := j.Chatid; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no chatid field: %s", msg)
return j, emsg
}
if exist := j.Userid; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no userid field: %s", msg)
return j, emsg
}
// j.Threadid может быть пустым, значит либо нам его не дали, либо дали пустым. Это нормально.
if exist := j.Message; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no message field: %s", msg)
return j, emsg
}
if exist := j.Plugin; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no plugin field: %s", msg)
return j, emsg
}
if exist := j.Mode; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no mode field: %s", msg)
return j, emsg
}
// j.Misc.Answer может и не быть, тогда ответа на такое сообщение не будет
// j.Misc.Botnick тоже можно не передавать, тогда будет записана пустая строка
// j.Misc.Csign если нам его не передали, возьмём значение из конфига
if exist := j.Misc.Csign; exist == "" {
j.Misc.Csign = cfg.Csign
}
// j.Misc.Fwdcnt если нам его не передали, то будет 0
if exist := j.Misc.Fwdcnt; exist == 0 {
j.Misc.Fwdcnt = 1
}
// j.Misc.GoodMorning может быть быть 1 или 0, по-умолчанию 0
// j.Misc.Msgformat может быть быть 1 или 0, по-умолчанию 0
// j.Misc.Username можно не передавать, тогда будет пустая строка
return j, nil
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

114
internal/config/config.go → internal/webapp/ReadConfig.go
View File

@@ -1,66 +1,37 @@
package config
package webapp
import (
"aleesa-webapp-go/internal/defaults"
"bufio"
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/log"
"encoding/json"
"errors"
"fmt"
"os"
"unsafe"
"github.com/cockroachdb/pebble"
"github.com/go-redis/redis/v8"
"github.com/hjson/hjson-go"
log "github.com/sirupsen/logrus"
)
// MyConfig структурка, описывающая конфиг.
type MyConfig struct {
Server string `json:"server,omitempty"`
Port int `json:"port,omitempty"`
Timeout int `json:"timeout,omitempty"`
Loglevel string `json:"loglevel,omitempty"`
Log string `json:"log,omitempty"`
Channel string `json:"channel,omitempty"`
DataDir string `json:"datadir,omitempty"`
Csign string `json:"csign,omitempty"`
ForwardsMax int64 `json:"forwards_max,omitempty"`
OpenWeatherMap struct {
Enabled bool `json:"enabled,omitempty"`
Country bool `json:"country,omitempty"`
Appid string `json:"appid,omitempty"`
} `json:"openweathermap,omitempty"`
Flickr struct {
Enabled bool `json:"enabled,omitempty"`
Key string `json:"key,omitempty"`
Secret string `json:"secret,omitempty"`
OAuthToken string `json:"oauth_token,omitempty"`
OAuthTokenSecret string `json:"oauth_token_secret,omitempty"`
} `json:"flickr,omitempty"`
UserAgents []string `json:"user_agents,omitempty"`
PcacheDB map[string]*pebble.DB
RedisClient *redis.Client
}
// ReadConfig читает и валидирует конфиг, а также выставляет некоторые default-ы, если значений для параметров в конфиге
// нет.
func ReadConfig() (*MyConfig, error) {
locations, err := defaults.DefaultConfigFileLocations()
func ReadConfig() error {
locations, err := DefaultConfigFileLocations()
if err != nil {
return &MyConfig{}, err
return err
}
for _, location := range locations {
fileInfo, err := os.Stat(location)
// Предполагаем, что файла либо нет, либо мы не можем его прочитать, второе надо бы логгировать, но пока забьём
// Предполагаем, что файла либо нет, либо мы не можем его прочитать, второе надо бы логгировать, но пока забьём.
if err != nil {
continue
}
// Конфиг-файл длинноват для конфига, попробуем следующего кандидата
if fileInfo.Size() > defaults.ConfigFileSize {
// Конфиг-файл длинноват для конфига, попробуем следующего кандидата.
if fileInfo.Size() > ConfigFileSize {
log.Warnf("Config file %s is too long for config, skipping", location)
continue
@@ -68,7 +39,7 @@ func ReadConfig() (*MyConfig, error) {
buf, err := os.ReadFile(location)
// Не удалось прочитать, попробуем следующего кандидата
// Не удалось прочитать, попробуем следующего кандидата.
if err != nil {
log.Warnf("Skip reading config file %s: %s", location, err)
@@ -79,13 +50,13 @@ func ReadConfig() (*MyConfig, error) {
// Интереснее на выходе получить структурку: то есть мы вначале конфиг преобразуем в map-ку, затем эту map-ку
// сериализуем в json, а потом json преврщааем в стркутурку. Не очень эффективно, но он и не часто требуется.
var (
sampleConfig *MyConfig
tmp map[string]interface{}
sampleConfig *config.MyConfig
tmp map[string]any
)
err = hjson.Unmarshal(buf, &tmp)
// Не удалось распарсить - попробуем следующего кандидата
// Не удалось распарсить - попробуем следующего кандидата.
if err != nil {
log.Warnf("Skip parsing config file %s: %s", location, err)
@@ -94,7 +65,7 @@ func ReadConfig() (*MyConfig, error) {
tmpjson, err := json.Marshal(tmp)
// Не удалось преобразовать map-ку в json
// Не удалось преобразовать map-ку в json.
if err != nil {
log.Warnf("Skip parsing config file %s: %s", location, err)
@@ -107,24 +78,24 @@ func ReadConfig() (*MyConfig, error) {
continue
}
// Валидируем значения из конфига
// Валидируем значения из конфига.
if sampleConfig.Server == "" {
sampleConfig.Server = defaults.Host
sampleConfig.Server = Host
}
if sampleConfig.Port == 0 {
sampleConfig.Port = defaults.RedisPort
sampleConfig.Port = RedisPort
}
if sampleConfig.Timeout == 0 {
sampleConfig.Timeout = defaults.NetworkTimeout
sampleConfig.Timeout = NetworkTimeout
}
if sampleConfig.Loglevel == "" {
sampleConfig.Loglevel = defaults.Loglevel
sampleConfig.Loglevel = Loglevel
}
// sampleConfig.Log = "" if not set
// sampleConfig.Log = "" if not set.
if sampleConfig.Channel == "" {
log.Errorf("Channel field in config file %s must be set", location)
@@ -133,7 +104,7 @@ func ReadConfig() (*MyConfig, error) {
}
if sampleConfig.DataDir == "" {
sampleConfig.DataDir = defaults.DataDir
sampleConfig.DataDir = DataDir
}
if sampleConfig.Csign == "" {
@@ -143,7 +114,7 @@ func ReadConfig() (*MyConfig, error) {
}
if sampleConfig.ForwardsMax == 0 {
sampleConfig.ForwardsMax = defaults.ForwardMax
sampleConfig.ForwardsMax = ForwardMax
}
if unsafe.Sizeof(sampleConfig.OpenWeatherMap) == 0 {
@@ -158,14 +129,14 @@ func ReadConfig() (*MyConfig, error) {
continue
}
// sampleConfig.OpenWeatherMap.Country is false by default
// sampleConfig.OpenWeatherMap.Country is false by default.
// Теперь надо бы попробовать загрузить список User Agent-ов, с которыми ходить в разные апишки.
uaFile := fmt.Sprintf("%s/useragents.txt", sampleConfig.DataDir)
uaFile := sampleConfig.DataDir + "/useragents.txt"
uaList, err := ReadLines(uaFile)
if err != nil {
return &MyConfig{}, fmt.Errorf("unable to read %s: %w", uaFile, err)
return fmt.Errorf("unable to read %s: %w", uaFile, err)
}
sampleConfig.UserAgents = uaList
@@ -174,37 +145,12 @@ func ReadConfig() (*MyConfig, error) {
log.Infof("Using %s as config file", location)
return sampleConfig, nil
Config = sampleConfig
return nil
}
return &MyConfig{}, fmt.Errorf("config was not loaded! Refusing to start")
}
// ReadLines читает даденный файл построчно в массив строк.
func ReadLines(path string) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
log.Errorf("Unable to close file %s:%s", path, err)
}
}(file)
var (
lines []string
scanner = bufio.NewScanner(file)
)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
return errors.New("config was not loaded! Refusing to start")
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

36
internal/webapp/ReadLines.go Normal file
View File

@@ -0,0 +1,36 @@
package webapp
import (
"aleesa-webapp-go/internal/log"
"bufio"
"os"
)
// ReadLines читает даденный файл построчно в массив строк.
func ReadLines(path string) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
log.Errorf("Unable to close file %s:%s", path, err)
}
}(file)
var (
lines []string
scanner = bufio.NewScanner(file)
)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

58
internal/webapp/SigHandler.go Normal file
View File

@@ -0,0 +1,58 @@
package webapp
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/log"
"os"
"syscall"
)
// SigHandler хэндлер сигналов закрывает все бд и сваливает из приложения.
func SigHandler(cfg *config.MyConfig) {
var err error
for {
var s = <-SigChan
switch s {
case syscall.SIGINT:
log.Info("Got SIGINT, quitting")
case syscall.SIGTERM:
log.Info("Got SIGTERM, quitting")
case syscall.SIGQUIT:
log.Info("Got SIGQUIT, quitting")
// Заходим на новую итерацию, если у нас "неинтересный" сигнал.
default:
continue
}
Shutdown = true
// Отпишемся от всех каналов и закроем коннект к редиске.
if err = Subscriber.Unsubscribe(Ctx); err != nil {
log.Errorf("Unable to unsubscribe from redis channels cleanly: %s", err)
}
if err = Subscriber.Close(); err != nil {
log.Errorf("Unable to close redis connection cleanly: %s", err)
}
if len(cfg.PcacheDB) > 0 {
log.Debug("Closing persistent cache db")
for dbName, db := range cfg.PcacheDB {
if err := db.Flush(); err != nil {
log.Errorf("Unable to flush %s db to disk: %s", dbName, err)
}
if err := db.Close(); err != nil {
log.Errorf("Unable to close %s db to disk: %s", dbName, err)
}
}
}
os.Exit(0)
}
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

66
internal/webapp/ValidateRmsg.go Normal file
View File

@@ -0,0 +1,66 @@
package webapp
import (
"aleesa-webapp-go/internal/config"
"fmt"
)
// ValidateRmsg валидирует входящее сообщение.
func ValidateRmsg(cfg *config.MyConfig, j RMsg, msg string) (RMsg, error) {
if exist := j.From; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no from field: %s", msg)
return j, emsg
}
if exist := j.Chatid; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no chatid field: %s", msg)
return j, emsg
}
if exist := j.Userid; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no userid field: %s", msg)
return j, emsg
}
// j.Threadid может быть пустым, значит либо нам его не дали, либо дали пустым. Это нормально.
if exist := j.Message; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no message field: %s", msg)
return j, emsg
}
if exist := j.Plugin; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no plugin field: %s", msg)
return j, emsg
}
if exist := j.Mode; exist == "" {
emsg := fmt.Errorf("incorrect msg from redis, no mode field: %s", msg)
return j, emsg
}
// j.Misc.Answer может и не быть, тогда ответа на такое сообщение не будет.
// j.Misc.Botnick тоже можно не передавать, тогда будет записана пустая строка.
// j.Misc.Csign если нам его не передали, возьмём значение из конфига.
if exist := j.Misc.Csign; exist == "" {
j.Misc.Csign = cfg.Csign
}
// j.Misc.Fwdcnt если нам его не передали, то будет 0.
if exist := j.Misc.Fwdcnt; exist == 0 {
j.Misc.Fwdcnt = 1
}
// j.Misc.GoodMorning может быть быть 1 или 0, по-умолчанию 0.
// j.Misc.Msgformat может быть быть 1 или 0, по-умолчанию 0.
// j.Misc.Username можно не передавать, тогда будет пустая строка.
return j, nil
}
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

13
internal/defaults/defaults.go → internal/webapp/defaults.go
View File

@@ -1,4 +1,4 @@
package defaults
package webapp
import (
"fmt"
@@ -6,14 +6,7 @@ import (
"path/filepath"
)
var ForwardMax int64 = 5
var ConfigFileSize int64 = 65535
var DataDir = "data"
var Host = "localhost"
var Loglevel = "info"
var RedisPort = 6379
var NetworkTimeout = 10
// DefaultConfigFileLocations выдаёт варианты путей, по которым может находиться конфиг-файл.
func DefaultConfigFileLocations() ([]string, error) {
var locations []string
@@ -23,7 +16,7 @@ func DefaultConfigFileLocations() ([]string, error) {
return locations, fmt.Errorf("unable to get my executable path: %w", err)
}
configJSONPath := fmt.Sprintf("%s/data/config.json", filepath.Dir(executablePath))
configJSONPath := filepath.Dir(executablePath) + "/data/config.json"
locations = []string{
"~/.aleesa-webapp-go.json",

49
internal/webapp/globals.go Normal file
View File

@@ -0,0 +1,49 @@
package webapp
import (
"aleesa-webapp-go/internal/config"
"context"
"os"
"github.com/go-redis/redis/v8"
)
var (
// ForwardMax максимальное количество перенаправлений между сервисами бота, по-умолчанию, 5.
ForwardMax int64 = 5
// ConfigFileSize максимальная длина конфиг-файла, предполагается что 64кб хватит всем.
ConfigFileSize int64 = 65535
// DataDir каталог с данными, в котором размещаются например, конфиг и кшт веб-клиентов, а также ключи flickr-а.
DataDir = "data"
// Host хост, на котором работает сервер redis.
Host = "localhost"
// Loglevel уровень логгирования, по-умолчанию, info.
Loglevel = "info"
// RedisPort порт, на которм слушает сервер redis, по-умолчанию 6379.
RedisPort = 6379
// NetworkTimeout таймаут сетевого соединения.
NetworkTimeout = 10
// Config структурка, содержащая данные конфига.
Config *config.MyConfig
// Ctx контекст редиски.
Ctx = context.Background()
// Subscriber объектик сабскрайбера редиски.
Subscriber *redis.PubSub
// Shutdown ставится в true, если мы получили сигнал на выключение.
Shutdown = false
// SigChan канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов.
SigChan = make(chan os.Signal, 1)
)
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

45
internal/webapp/types.go Normal file
View File

@@ -0,0 +1,45 @@
package webapp
type (
// RMsg структурка, описывающая входящее сообщение из pubsub-канала redis-ки.
RMsg struct {
From string `json:"from,omitempty"`
Chatid string `json:"chatid,omitempty"`
Userid string `json:"userid,omitempty"`
Threadid string `json:"threadid,omitempty"`
Message string `json:"message,omitempty"`
Plugin string `json:"plugin,omitempty"`
Mode string `json:"mode,omitempty"`
Misc struct { //nolint: govet // Мне так удобнее.
Answer int64 `json:"answer,omitempty"`
Botnick string `json:"bot_nick,omitempty"`
Csign string `json:"csign,omitempty"`
Fwdcnt int64 `json:"fwd_cnt,omitempty"`
GoodMorning int64 `json:"good_morning,omitempty"`
Msgformat int64 `json:"msg_format,omitempty"`
Username string `json:"username,omitempty"`
} `json:"Misc"`
}
// SMsg структурка, описывающая исходящее сообщение в pubsub-канал redis-ки.
SMsg struct {
From string `json:"from"`
Chatid string `json:"chatid"`
Userid string `json:"userid"`
Threadid string `json:"threadid"`
Message string `json:"message"`
Plugin string `json:"plugin"`
Mode string `json:"mode"`
Misc struct { //nolint: govet // Мне так удобнее.
Answer int64 `json:"answer"`
Botnick string `json:"bot_nick"`
Csign string `json:"csign"`
Fwdcnt int64 `json:"fwd_cnt"`
GoodMorning int64 `json:"good_morning"`
Msgformat int64 `json:"msg_format"`
Username string `json:"username"`
} `json:"misc"`
}
)
/* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

17
internal/xkcdru/xkcdru.go
View File

@@ -1,16 +1,18 @@
// Package xkcdru provides api to xkcd.ru.
package xkcdru
import (
"aleesa-webapp-go/internal/config"
"aleesa-webapp-go/internal/log"
"errors"
"fmt"
"io"
"math/rand/v2"
"net/http"
"time"
log "github.com/sirupsen/logrus"
)
// APIClient клиент сервиса xkcd.ru.
func APIClient(cfg *config.MyConfig) (string, error) {
var (
err error
@@ -24,7 +26,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
}
)
req, err := http.NewRequest(http.MethodGet, url, nil) //nolint: noctx
req, err := http.NewRequest(http.MethodGet, url, http.NoBody) //nolint: noctx
if err != nil {
return "", err
@@ -33,6 +35,7 @@ func APIClient(cfg *config.MyConfig) (string, error) {
req.Header.Set("User-Agent", cfg.UserAgents[rand.IntN(len(cfg.UserAgents))])
var resp *http.Response
resp, err = c.Do(req) //nolint: bodyclose // Посмотри ниже, блять, тело запроса закрывается.
if err != nil {
@@ -41,19 +44,17 @@ func APIClient(cfg *config.MyConfig) (string, error) {
// Типа, надо закрывать Body в любом случае, как рекомендуют в документации https://pkg.go.dev/net/http .
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
if err := Body.Close(); err != nil {
log.Errorf("Unable to close response body for request to %s: %s", url, err)
}
}(resp.Body)
// Response body нам не интересен, нам интересен статус и нам интересен заголовок Location
// Response body нам не интересен, нам интересен статус и нам интересен заголовок Location.
if resp.StatusCode == 302 {
location := resp.Header.Get("Location")
if len(location) <= 3 {
err = fmt.Errorf("location header too short")
err = errors.New("location header too short")
return "", err
}

6
vendor/github.com/DataDog/zstd/README.md generated vendored
View File

@@ -6,8 +6,8 @@
[C Zstd Homepage](https://github.com/facebook/zstd)
The current headers and C files are from *v1.5.6* (Commit
[794ea1b](https://github.com/facebook/zstd/releases/tag/v1.5.6)).
The current headers and C files are from *v1.5.7* (Commit
[f8745da](https://github.com/facebook/zstd/releases/tag/v1.5.7)).
## Usage
@@ -41,7 +41,7 @@ go build -tags external_libzstd
// Compress compresses the byte array given in src and writes it to dst.
// If you already have a buffer allocated, you can pass it to prevent allocation
// If not, you can pass nil as dst.
// If the buffer is too small, it will be reallocated, resized, and returned bu the function
// If the buffer is too small, it will be reallocated, resized, and returned by the function
// If dst is nil, this will allocate the worst case size (CompressBound(src))
Compress(dst, src []byte) ([]byte, error)
```

179
vendor/github.com/DataDog/zstd/bits.h generated vendored
View File

@@ -29,27 +29,29 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
{
assert(val != 0);
# if defined(_MSC_VER)
# if STATIC_BMI2 == 1
return (unsigned)_tzcnt_u32(val);
# else
if (val != 0) {
unsigned long r;
_BitScanForward(&r, val);
return (unsigned)r;
} else {
/* Should not reach this code path */
__assume(0);
}
# endif
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)__builtin_ctz(val);
# else
return ZSTD_countTrailingZeros32_fallback(val);
# endif
#if defined(_MSC_VER)
# if STATIC_BMI2
return (unsigned)_tzcnt_u32(val);
# else
if (val != 0) {
unsigned long r;
_BitScanForward(&r, val);
return (unsigned)r;
} else {
__assume(0); /* Should not reach this code path */
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)__builtin_ctz(val);
#elif defined(__ICCARM__)
return (unsigned)__builtin_ctz(val);
#else
return ZSTD_countTrailingZeros32_fallback(val);
#endif
}
MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
{
assert(val != 0);
{
static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
@@ -68,86 +70,89 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
{
assert(val != 0);
# if defined(_MSC_VER)
# if STATIC_BMI2 == 1
return (unsigned)_lzcnt_u32(val);
# else
if (val != 0) {
unsigned long r;
_BitScanReverse(&r, val);
return (unsigned)(31 - r);
} else {
/* Should not reach this code path */
__assume(0);
}
# endif
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)__builtin_clz(val);
# else
return ZSTD_countLeadingZeros32_fallback(val);
# endif
#if defined(_MSC_VER)
# if STATIC_BMI2
return (unsigned)_lzcnt_u32(val);
# else
if (val != 0) {
unsigned long r;
_BitScanReverse(&r, val);
return (unsigned)(31 - r);
} else {
__assume(0); /* Should not reach this code path */
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)__builtin_clz(val);
#elif defined(__ICCARM__)
return (unsigned)__builtin_clz(val);
#else
return ZSTD_countLeadingZeros32_fallback(val);
#endif
}
MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
{
assert(val != 0);
# if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2 == 1
return (unsigned)_tzcnt_u64(val);
# else
if (val != 0) {
unsigned long r;
_BitScanForward64(&r, val);
return (unsigned)r;
} else {
/* Should not reach this code path */
__assume(0);
}
# endif
# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
return (unsigned)__builtin_ctzll(val);
# else
{
U32 mostSignificantWord = (U32)(val >> 32);
U32 leastSignificantWord = (U32)val;
if (leastSignificantWord == 0) {
return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
} else {
return ZSTD_countTrailingZeros32(leastSignificantWord);
}
#if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2
return (unsigned)_tzcnt_u64(val);
# else
if (val != 0) {
unsigned long r;
_BitScanForward64(&r, val);
return (unsigned)r;
} else {
__assume(0); /* Should not reach this code path */
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
return (unsigned)__builtin_ctzll(val);
#elif defined(__ICCARM__)
return (unsigned)__builtin_ctzll(val);
#else
{
U32 mostSignificantWord = (U32)(val >> 32);
U32 leastSignificantWord = (U32)val;
if (leastSignificantWord == 0) {
return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
} else {
return ZSTD_countTrailingZeros32(leastSignificantWord);
}
# endif
}
#endif
}
MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
{
assert(val != 0);
# if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2 == 1
return (unsigned)_lzcnt_u64(val);
# else
if (val != 0) {
unsigned long r;
_BitScanReverse64(&r, val);
return (unsigned)(63 - r);
} else {
/* Should not reach this code path */
__assume(0);
}
# endif
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)(__builtin_clzll(val));
# else
{
U32 mostSignificantWord = (U32)(val >> 32);
U32 leastSignificantWord = (U32)val;
if (mostSignificantWord == 0) {
return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
} else {
return ZSTD_countLeadingZeros32(mostSignificantWord);
}
#if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2
return (unsigned)_lzcnt_u64(val);
# else
if (val != 0) {
unsigned long r;
_BitScanReverse64(&r, val);
return (unsigned)(63 - r);
} else {
__assume(0); /* Should not reach this code path */
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)(__builtin_clzll(val));
#elif defined(__ICCARM__)
return (unsigned)(__builtin_clzll(val));
#else
{
U32 mostSignificantWord = (U32)(val >> 32);
U32 leastSignificantWord = (U32)val;
if (mostSignificantWord == 0) {
return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
} else {
return ZSTD_countLeadingZeros32(mostSignificantWord);
}
# endif
}
#endif
}
MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)

55
vendor/github.com/DataDog/zstd/bitstream.h generated vendored
View File

@@ -15,9 +15,6 @@
#ifndef BITSTREAM_H_MODULE
#define BITSTREAM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*
* This API consists of small unitary functions, which must be inlined for best performance.
* Since link-time-optimization is not available for all compilers,
@@ -33,7 +30,6 @@ extern "C" {
#include "error_private.h" /* error codes and messages */
#include "bits.h" /* ZSTD_highbit32 */
/*=========================================
* Target specific
=========================================*/
@@ -53,12 +49,13 @@ extern "C" {
/*-******************************************
* bitStream encoding API (write forward)
********************************************/
typedef size_t BitContainerType;
/* bitStream can mix input from multiple sources.
* A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/
typedef struct {
size_t bitContainer;
BitContainerType bitContainer;
unsigned bitPos;
char* startPtr;
char* ptr;
@@ -66,7 +63,7 @@ typedef struct {
} BIT_CStream_t;
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
@@ -75,7 +72,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
*
* bits are first added to a local register.
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
* Writing data into memory is an explicit operation, performed by the flushBits function.
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
* After a flushBits, a maximum of 7 bits might still be stored into local register.
@@ -91,7 +88,6 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
/*-********************************************
* bitStream decoding API (read backward)
**********************************************/
typedef size_t BitContainerType;
typedef struct {
BitContainerType bitContainer;
unsigned bitsConsumed;
@@ -107,7 +103,7 @@ typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */
} BIT_DStream_status; /* result of BIT_reloadDStream() */
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
@@ -126,7 +122,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
/*-****************************************
* unsafe API
******************************************/
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
@@ -164,10 +160,15 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
return 0;
}
FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits)
{
#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
return _bzhi_u64(bitContainer, nbBits);
#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS)
# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__)
return _bzhi_u64(bitContainer, nbBits);
# else
DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32));
return _bzhi_u32(bitContainer, nbBits);
# endif
#else
assert(nbBits < BIT_MASK_SIZE);
return bitContainer & BIT_mask[nbBits];
@@ -178,7 +179,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbB
* can add up to 31 bits into `bitC`.
* Note : does not check for register overflow ! */
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
size_t value, unsigned nbBits)
BitContainerType value, unsigned nbBits)
{
DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
assert(nbBits < BIT_MASK_SIZE);
@@ -191,7 +192,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
* works only if `value` is _clean_,
* meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
size_t value, unsigned nbBits)
BitContainerType value, unsigned nbBits)
{
assert((value>>nbBits) == 0);
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
@@ -238,7 +239,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
BIT_addBitsFast(bitC, 1, 1); /* endMark */
BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
}
@@ -299,12 +300,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
return srcSize;
}
FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
{
return bitContainer >> start;
}
FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
{
U32 const regMask = sizeof(bitContainer)*8 - 1;
/* if start > regMask, bitstream is corrupted, and result is undefined */
@@ -314,7 +315,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U3
* such cpus old (pre-Haswell, 2013) and their performance is not of that
* importance.
*/
#if defined(__x86_64__) || defined(_M_X86)
#if defined(__x86_64__) || defined(_M_X64)
return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
#else
return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
@@ -327,7 +328,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U3
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted */
FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
{
/* arbitrate between double-shift and shift+mask */
#if 1
@@ -343,7 +344,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits
/*! BIT_lookBitsFast() :
* unsafe version; only works if nbBits >= 1 */
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
{
U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
assert(nbBits >= 1);
@@ -359,18 +360,18 @@ FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
FORCE_INLINE_TEMPLATE size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
{
size_t const value = BIT_lookBits(bitD, nbBits);
BitContainerType const value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works if nbBits >= 1 */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
{
size_t const value = BIT_lookBitsFast(bitD, nbBits);
BitContainerType const value = BIT_lookBitsFast(bitD, nbBits);
assert(nbBits >= 1);
BIT_skipBits(bitD, nbBits);
return value;
@@ -451,10 +452,6 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
}
#if defined (__cplusplus)
}
#endif
#endif /* BITSTREAM_H_MODULE */
#endif /* USE_EXTERNAL_ZSTD */

58
vendor/github.com/DataDog/zstd/compiler.h generated vendored
View File

@@ -28,7 +28,7 @@
# define INLINE_KEYWORD
#endif
#if defined(__GNUC__) || defined(__ICCARM__)
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
# define FORCE_INLINE_ATTR __attribute__((always_inline))
#elif defined(_MSC_VER)
# define FORCE_INLINE_ATTR __forceinline
@@ -55,7 +55,7 @@
#endif
/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
#if defined(__GNUC__)
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
# define UNUSED_ATTR __attribute__((unused))
#else
# define UNUSED_ATTR
@@ -96,6 +96,8 @@
#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */
#if defined(__GNUC__)
# define MEM_STATIC static __inline UNUSED_ATTR
#elif defined(__IAR_SYSTEMS_ICC__)
# define MEM_STATIC static inline UNUSED_ATTR
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define MEM_STATIC static inline
#elif defined(_MSC_VER)
@@ -109,7 +111,7 @@
#ifdef _MSC_VER
# define FORCE_NOINLINE static __declspec(noinline)
#else
# if defined(__GNUC__) || defined(__ICCARM__)
# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
# define FORCE_NOINLINE static __attribute__((__noinline__))
# else
# define FORCE_NOINLINE static
@@ -118,7 +120,7 @@
/* target attribute */
#if defined(__GNUC__) || defined(__ICCARM__)
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
#else
# define TARGET_ATTRIBUTE(target)
@@ -206,30 +208,21 @@
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
#endif
/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
#ifndef STATIC_BMI2
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
# define STATIC_BMI2 1
# endif
# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__)
# define STATIC_BMI2 1
# endif
#endif
#ifndef STATIC_BMI2
#define STATIC_BMI2 0
#endif
/* compile time determination of SIMD support */
#if !defined(ZSTD_NO_INTRINSICS)
# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
# if defined(__AVX2__)
# define ZSTD_ARCH_X86_AVX2
# endif
# if defined(__SSE2__) || defined(_M_X64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
# define ZSTD_ARCH_X86_SSE2
# endif
# if defined(__ARM_NEON) || defined(_M_ARM64)
# define ZSTD_ARCH_ARM_NEON
# endif
#
# if defined(ZSTD_ARCH_X86_AVX2)
# include <immintrin.h>
# endif
# if defined(ZSTD_ARCH_X86_SSE2)
# include <emmintrin.h>
# elif defined(ZSTD_ARCH_ARM_NEON)
@@ -274,9 +267,15 @@
#endif
/*-**************************************************************
* Alignment check
* Alignment
*****************************************************************/
/* @return 1 if @u is a 2^n value, 0 otherwise
* useful to check a value is valid for alignment restrictions */
MEM_STATIC int ZSTD_isPower2(size_t u) {
return (u & (u-1)) == 0;
}
/* this test was initially positioned in mem.h,
* but this file is removed (or replaced) for linux kernel
* so it's now hosted in compiler.h,
@@ -302,6 +301,21 @@
# endif
#endif /* ZSTD_ALIGNOF */
#ifndef ZSTD_ALIGNED
/* C90-compatible alignment macro (GCC/Clang). Adjust for other compilers if needed. */
# if defined(__GNUC__) || defined(__clang__)
# define ZSTD_ALIGNED(a) __attribute__((aligned(a)))
# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
# define ZSTD_ALIGNED(a) _Alignas(a)
#elif defined(_MSC_VER)
# define ZSTD_ALIGNED(n) __declspec(align(n))
# else
/* this compiler will require its own alignment instruction */
# define ZSTD_ALIGNED(...)
# endif
#endif /* ZSTD_ALIGNED */
/*-**************************************************************
* Sanitizer
*****************************************************************/
@@ -325,7 +339,7 @@
#endif
/**
* Helper function to perform a wrapped pointer difference without trigging
* Helper function to perform a wrapped pointer difference without triggering
* UBSAN.
*
* @returns lhs - rhs with wrapping

79
vendor/github.com/DataDog/zstd/cover.c generated vendored
View File

@@ -22,8 +22,17 @@
/*-*************************************
* Dependencies
***************************************/
/* qsort_r is an extension. */
#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
defined(__CYGWIN__) || defined(__MSYS__)
#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
#define _GNU_SOURCE
#endif
#endif
#include <stdio.h> /* fprintf */
#include <stdlib.h> /* malloc, free, qsort */
#include <stdlib.h> /* malloc, free, qsort_r */
#include <string.h> /* memset */
#include <time.h> /* clock */
@@ -233,8 +242,10 @@ typedef struct {
unsigned d;
} COVER_ctx_t;
/* We need a global context for qsort... */
#if !defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER)
/* C90 only offers qsort() that needs a global context. */
static COVER_ctx_t *g_coverCtx = NULL;
#endif
/*-*************************************
* Helper functions
@@ -277,11 +288,15 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
/**
* Same as COVER_cmp() except ties are broken by pointer value
* NOTE: g_coverCtx must be set to call this function. A global is required because
* qsort doesn't take an opaque pointer.
*/
static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
int result = COVER_cmp(g_coverCtx, lp, rp);
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) {
#elif defined(_GNU_SOURCE)
static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) {
#else /* C90 fallback.*/
static int COVER_strict_cmp(const void *lp, const void *rp) {
#endif
int result = COVER_cmp((COVER_ctx_t*)g_coverCtx, lp, rp);
if (result == 0) {
result = lp < rp ? -1 : 1;
}
@@ -290,14 +305,50 @@ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
/**
* Faster version for d <= 8.
*/
static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
int result = COVER_cmp8(g_coverCtx, lp, rp);
#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__)
static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) {
#elif defined(_GNU_SOURCE)
static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) {
#else /* C90 fallback.*/
static int COVER_strict_cmp8(const void *lp, const void *rp) {
#endif
int result = COVER_cmp8((COVER_ctx_t*)g_coverCtx, lp, rp);
if (result == 0) {
result = lp < rp ? -1 : 1;
}
return result;
}
/**
* Abstract away divergence of qsort_r() parameters.
* Hopefully when C11 become the norm, we will be able
* to clean it up.
*/
static void stableSort(COVER_ctx_t *ctx) {
#if defined(__APPLE__)
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
ctx,
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#elif defined(_GNU_SOURCE)
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#elif defined(_WIN32) && defined(_MSC_VER)
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#elif defined(__OpenBSD__)
g_coverCtx = ctx;
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#else /* C90 fallback.*/
g_coverCtx = ctx;
/* TODO(cavalcanti): implement a reentrant qsort() when is not available. */
qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#endif
}
/**
* Returns the first pointer in [first, last) whose element does not compare
* less than value. If no such element exists it returns last.
@@ -621,17 +672,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
for (i = 0; i < ctx->suffixSize; ++i) {
ctx->suffix[i] = i;
}
/* qsort doesn't take an opaque pointer, so pass as a global.
* On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
*/
g_coverCtx = ctx;
#if defined(__OpenBSD__)
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#else
qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#endif
stableSort(ctx);
}
DISPLAYLEVEL(2, "Computing frequencies\n");
/* For each dmer group (group of positions with the same first d bytes):

2
vendor/github.com/DataDog/zstd/cpu.h generated vendored
View File

@@ -36,7 +36,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
U32 f7b = 0;
U32 f7c = 0;
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
#if !defined(__clang__)
#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16
int reg[4];
__cpuid((int*)reg, 0);
{

9
vendor/github.com/DataDog/zstd/debug.h generated vendored
View File

@@ -33,10 +33,6 @@
#ifndef DEBUG_H_12987983217
#define DEBUG_H_12987983217
#if defined (__cplusplus)
extern "C" {
#endif
/* static assert is triggered at compile time, leaving no runtime artefact.
* static assert only works with compile-time constants.
@@ -109,11 +105,6 @@ extern int g_debuglevel; /* the variable is only declared,
# define DEBUGLOG(l, ...) do { } while (0) /* disabled */
#endif
#if defined (__cplusplus)
}
#endif
#endif /* DEBUG_H_12987983217 */
#endif /* USE_EXTERNAL_ZSTD */

10
vendor/github.com/DataDog/zstd/divsufsort.h generated vendored
View File

@@ -28,11 +28,6 @@
#ifndef _DIVSUFSORT_H
#define _DIVSUFSORT_H 1
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
/*- Prototypes -*/
/**
@@ -60,11 +55,6 @@ divsufsort(const unsigned char *T, int *SA, int n, int openMP);
int
divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
#endif /* _DIVSUFSORT_H */
#endif /* USE_EXTERNAL_ZSTD */

1
vendor/github.com/DataDog/zstd/error_private.c generated vendored
View File

@@ -41,6 +41,7 @@ const char* ERR_getErrorString(ERR_enum code)
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block";
case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";

10
vendor/github.com/DataDog/zstd/error_private.h generated vendored
View File

@@ -14,11 +14,6 @@
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Dependencies
******************************************/
@@ -27,7 +22,6 @@ extern "C" {
#include "debug.h"
#include "zstd_deps.h" /* size_t */
/* ****************************************
* Compiler-specific
******************************************/
@@ -162,10 +156,6 @@ void _force_has_format_string(const char *format, ...) {
} \
} while(0)
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */
#endif /* USE_EXTERNAL_ZSTD */

19
vendor/github.com/DataDog/zstd/fse.h generated vendored
View File

@@ -12,11 +12,6 @@
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
****************************************************************** */
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef FSE_H
#define FSE_H
@@ -26,7 +21,6 @@ extern "C" {
******************************************/
#include "zstd_deps.h" /* size_t, ptrdiff_t */
/*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility
******************************************/
@@ -233,11 +227,8 @@ If there is an error, the function will return an error code, which can be teste
#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
#define FSE_H_FSE_STATIC_LINKING_ONLY
/* *** Dependency *** */
#include "bitstream.h"
/* *****************************************
* Static allocation
*******************************************/
@@ -466,13 +457,13 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);
U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, (size_t)statePtr->value, nbBitsOut);
BIT_addBits(bitC, (BitContainerType)statePtr->value, nbBitsOut);
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
{
BIT_addBits(bitC, (size_t)statePtr->value, statePtr->stateLog);
BIT_addBits(bitC, (BitContainerType)statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
@@ -632,12 +623,6 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
#endif /* FSE_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* USE_EXTERNAL_ZSTD */

2
vendor/github.com/DataDog/zstd/fse_decompress.c generated vendored
View File

@@ -191,6 +191,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
FSE_initDState(&state1, &bitD, dt);
FSE_initDState(&state2, &bitD, dt);
RETURN_ERROR_IF(BIT_reloadDStream(&bitD)==BIT_DStream_overflow, corruption_detected, "");
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
/* 4 symbols per loop */

10
vendor/github.com/DataDog/zstd/hist.c generated vendored
View File

@@ -27,6 +27,16 @@ unsigned HIST_isError(size_t code) { return ERR_isError(code); }
/*-**************************************************************
* Histogram functions
****************************************************************/
void HIST_add(unsigned* count, const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const end = ip + srcSize;
while (ip<end) {
count[*ip++]++;
}
}
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize)
{

7
vendor/github.com/DataDog/zstd/hist.h generated vendored
View File

@@ -75,4 +75,11 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize);
/*! HIST_add() :
* Lowest level: just add nb of occurrences of characters from @src into @count.
* @count is not reset. @count array is presumed large enough (i.e. 1 KB).
@ This function does not need any additional stack memory.
*/
void HIST_add(unsigned* count, const void* src, size_t srcSize);
#endif /* USE_EXTERNAL_ZSTD */

9
vendor/github.com/DataDog/zstd/huf.h generated vendored
View File

@@ -13,10 +13,6 @@
* You may select, at your option, one of the above-listed licenses.
****************************************************************** */
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef HUF_H_298734234
#define HUF_H_298734234
@@ -26,7 +22,6 @@ extern "C" {
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* *** Tool functions *** */
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
@@ -282,8 +277,4 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize
#endif /* HUF_H_298734234 */
#if defined (__cplusplus)
}
#endif
#endif /* USE_EXTERNAL_ZSTD */

13
vendor/github.com/DataDog/zstd/huf_decompress_amd64.S generated vendored
View File

@@ -42,13 +42,11 @@
/* Calling convention:
*
* %rdi contains the first argument: HUF_DecompressAsmArgs*.
* %rdi (or %rcx on Windows) contains the first argument: HUF_DecompressAsmArgs*.
* %rbp isn't maintained (no frame pointer).
* %rsp contains the stack pointer that grows down.
* No red-zone is assumed, only addresses >= %rsp are used.
* All register contents are preserved.
*
* TODO: Support Windows calling convention.
*/
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
@@ -137,7 +135,11 @@ HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
push %r15
/* Read HUF_DecompressAsmArgs* args from %rax */
#if defined(_WIN32)
movq %rcx, %rax
#else
movq %rdi, %rax
#endif
movq 0(%rax), %ip0
movq 8(%rax), %ip1
movq 16(%rax), %ip2
@@ -391,7 +393,12 @@ HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
push %r14
push %r15
/* Read HUF_DecompressAsmArgs* args from %rax */
#if defined(_WIN32)
movq %rcx, %rax
#else
movq %rdi, %rax
#endif
movq 0(%rax), %ip0
movq 8(%rax), %ip1
movq 16(%rax), %ip2

18
vendor/github.com/DataDog/zstd/mem.h generated vendored
View File

@@ -12,10 +12,6 @@
#ifndef MEM_H_MODULE
#define MEM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
@@ -31,6 +27,8 @@ extern "C" {
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#elif defined(__ICCARM__)
# include <intrinsics.h>
#endif
/*-**************************************************************
@@ -75,7 +73,6 @@ extern "C" {
typedef signed long long S64;
#endif
/*-**************************************************************
* Memory I/O API
*****************************************************************/
@@ -151,10 +148,12 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
return 1;
#elif defined(__clang__) && __BIG_ENDIAN__
return 0;
#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
#elif defined(_MSC_VER) && (_M_X64 || _M_IX86)
return 1;
#elif defined(__DMC__) && defined(_M_IX86)
return 1;
#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__
return 1;
#else
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
@@ -247,6 +246,8 @@ MEM_STATIC U32 MEM_swap32(U32 in)
#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
|| (defined(__clang__) && __has_builtin(__builtin_bswap32))
return __builtin_bswap32(in);
#elif defined(__ICCARM__)
return __REV(in);
#else
return MEM_swap32_fallback(in);
#endif
@@ -419,11 +420,6 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
/* code only tested on 32 and 64 bits systems */
MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
#if defined (__cplusplus)
}
#endif
#endif /* MEM_H_MODULE */
#endif /* USE_EXTERNAL_ZSTD */

9
vendor/github.com/DataDog/zstd/pool.h generated vendored
View File

@@ -12,10 +12,6 @@
#ifndef POOL_H
#define POOL_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_deps.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
@@ -83,11 +79,6 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
*/
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
#if defined (__cplusplus)
}
#endif
#endif
#endif /* USE_EXTERNAL_ZSTD */

31
vendor/github.com/DataDog/zstd/portability_macros.h generated vendored
View File

@@ -75,26 +75,39 @@
# define ZSTD_HIDE_ASM_FUNCTION(func)
#endif
/* Compile time determination of BMI2 support */
#ifndef STATIC_BMI2
# if defined(__BMI2__)
# define STATIC_BMI2 1
# elif defined(_MSC_VER) && defined(__AVX2__)
# define STATIC_BMI2 1 /* MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 */
# endif
#endif
#ifndef STATIC_BMI2
# define STATIC_BMI2 0
#endif
/* Enable runtime BMI2 dispatch based on the CPU.
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
*/
#ifndef DYNAMIC_BMI2
#if ((defined(__clang__) && __has_attribute(__target__)) \
# if ((defined(__clang__) && __has_attribute(__target__)) \
|| (defined(__GNUC__) \
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
&& (defined(__x86_64__) || defined(_M_X64)) \
&& (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)) \
&& !defined(__BMI2__)
# define DYNAMIC_BMI2 1
#else
# define DYNAMIC_BMI2 0
#endif
# define DYNAMIC_BMI2 1
# else
# define DYNAMIC_BMI2 0
# endif
#endif
/**
* Only enable assembly for GNUC compatible compilers,
* Only enable assembly for GNU C compatible compilers,
* because other platforms may not support GAS assembly syntax.
*
* Only enable assembly for Linux / MacOS, other platforms may
* Only enable assembly for Linux / MacOS / Win32, other platforms may
* work, but they haven't been tested. This could likely be
* extended to BSD systems.
*
@@ -102,7 +115,7 @@
* 100% of code to be instrumented to work.
*/
#if defined(__GNUC__)
# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
# if defined(__linux__) || defined(__linux) || defined(__APPLE__) || defined(_WIN32)
# if ZSTD_MEMORY_SANITIZER
# define ZSTD_ASM_SUPPORTED 0
# elif ZSTD_DATAFLOW_SANITIZER

8
vendor/github.com/DataDog/zstd/threading.h generated vendored
View File

@@ -17,10 +17,6 @@
#include "debug.h"
#if defined (__cplusplus)
extern "C" {
#endif
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
/**
@@ -73,7 +69,6 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread);
* add here more wrappers as required
*/
#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
/* === POSIX Systems === */
# include <pthread.h>
@@ -144,9 +139,6 @@ typedef int ZSTD_pthread_cond_t;
#endif /* ZSTD_MULTITHREAD */
#if defined (__cplusplus)
}
#endif
#endif /* THREADING_H_938743 */

112
vendor/github.com/DataDog/zstd/xxhash.h generated vendored
View File

@@ -228,10 +228,6 @@
* xxHash prototypes and implementation
*/
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************
* INLINE mode
******************************/
@@ -538,6 +534,9 @@ extern "C" {
/*! @brief Version number, encoded as two digits each */
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @brief Obtains the xxHash version.
*
@@ -548,6 +547,9 @@ extern "C" {
*/
XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);
#if defined (__cplusplus)
}
#endif
/* ****************************
* Common basic types
@@ -594,6 +596,10 @@ typedef uint32_t XXH32_hash_t;
# endif
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @}
*
@@ -822,6 +828,9 @@ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canoni
#endif
/*! @endcond */
#if defined (__cplusplus)
} /* end of extern "C" */
#endif
/*!
* @}
@@ -860,6 +869,9 @@ typedef uint64_t XXH64_hash_t;
# endif
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @}
*
@@ -1563,6 +1575,11 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE con
#endif /* !XXH_NO_XXH3 */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_NO_LONG_LONG */
/*!
@@ -1749,6 +1766,10 @@ struct XXH3_state_s {
} while(0)
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @brief Calculates the 128-bit hash of @p data using XXH3.
*
@@ -1964,8 +1985,13 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
XXH64_hash_t seed64);
#endif /* !XXH_NO_STREAM */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* !XXH_NO_XXH3 */
#endif /* XXH_NO_LONG_LONG */
#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
# define XXH_IMPLEMENTATION
#endif
@@ -2264,10 +2290,12 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
* @{
*/
/* *************************************
* Includes & Memory related functions
***************************************/
#include <string.h> /* memcmp, memcpy */
#include <limits.h> /* ULLONG_MAX */
#if defined(XXH_NO_STREAM)
/* nothing */
#elif defined(XXH_NO_STDLIB)
@@ -2281,9 +2309,17 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
* without access to dynamic allocation.
*/
#if defined (__cplusplus)
extern "C" {
#endif
static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
static void XXH_free(void* p) { (void)p; }
#if defined (__cplusplus)
} /* extern "C" */
#endif
#else
/*
@@ -2292,6 +2328,9 @@ static void XXH_free(void* p) { (void)p; }
*/
#include <stdlib.h>
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @internal
* @brief Modify this function to use a different routine than malloc().
@@ -2304,10 +2343,15 @@ static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }
*/
static void XXH_free(void* p) { free(p); }
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_NO_STDLIB */
#include <string.h>
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @internal
* @brief Modify this function to use a different routine than memcpy().
@@ -2317,8 +2361,9 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size)
return memcpy(dest,src,size);
}
#include <limits.h> /* ULLONG_MAX */
#if defined (__cplusplus)
} /* extern "C" */
#endif
/* *************************************
* Compiler Specific Options
@@ -2453,6 +2498,10 @@ typedef XXH32_hash_t xxh_u32;
# define U32 xxh_u32
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/* *** Memory access *** */
/*!
@@ -3609,6 +3658,10 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can
return XXH_readBE64(src);
}
#if defined (__cplusplus)
}
#endif
#ifndef XXH_NO_XXH3
/* *********************************************************************
@@ -3840,7 +3893,7 @@ enum XXH_VECTOR_TYPE /* fake enum */ {
# define XXH_VECTOR XXH_AVX512
# elif defined(__AVX2__)
# define XXH_VECTOR XXH_AVX2
# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
# elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
# define XXH_VECTOR XXH_SSE2
# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
|| (defined(__s390x__) && defined(__VEC__)) \
@@ -3929,6 +3982,10 @@ enum XXH_VECTOR_TYPE /* fake enum */ {
# pragma GCC optimize("-O2")
#endif
#if defined (__cplusplus)
extern "C" {
#endif
#if XXH_VECTOR == XXH_NEON
/*
@@ -4051,6 +4108,10 @@ XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
# endif
#endif /* XXH_VECTOR == XXH_NEON */
#if defined (__cplusplus)
} /* extern "C" */
#endif
/*
* VSX and Z Vector helpers.
*
@@ -4112,6 +4173,9 @@ typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;
# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
# define XXH_vec_revb vec_revb
# else
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* A polyfill for POWER9's vec_revb().
*/
@@ -4121,9 +4185,15 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
return vec_perm(val, val, vByteSwap);
}
#if defined (__cplusplus)
} /* extern "C" */
#endif
# endif
# endif /* XXH_VSX_BE */
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* Performs an unaligned vector load and byte swaps it on big endian.
*/
@@ -4168,6 +4238,11 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
return result;
}
# endif /* XXH_vec_mulo, XXH_vec_mule */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_VECTOR == XXH_VSX */
#if XXH_VECTOR == XXH_SVE
@@ -4201,7 +4276,9 @@ do { \
# endif
#endif /* XXH_NO_PREFETCH */
#if defined (__cplusplus)
extern "C" {
#endif
/* ==========================================
* XXH3 default settings
* ========================================== */
@@ -6878,8 +6955,6 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_
#endif /* !XXH_NO_STREAM */
/* 128-bit utility functions */
#include <string.h> /* memcmp, memcpy */
/* return : 1 is equal, 0 if different */
/*! @ingroup XXH3_family */
XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
@@ -7006,8 +7081,12 @@ XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
# pragma GCC pop_options
#endif
#endif /* XXH_NO_LONG_LONG */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_NO_LONG_LONG */
#endif /* XXH_NO_XXH3 */
/*!
@@ -7015,9 +7094,4 @@ XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
*/
#endif /* XXH_IMPLEMENTATION */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* USE_EXTERNAL_ZSTD */

4
vendor/github.com/DataDog/zstd/zdict.c generated vendored
View File

@@ -581,7 +581,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
const SeqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
/* literals stats */
{ const BYTE* bytePtr;
@@ -609,7 +609,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
}
if (nbSeq >= 2) { /* rep offsets */
const seqDef* const seq = seqStorePtr->sequencesStart;
const SeqDef* const seq = seqStorePtr->sequencesStart;
U32 offset1 = seq[0].offBase - ZSTD_REP_NUM;
U32 offset2 = seq[1].offBase - ZSTD_REP_NUM;
if (offset1 >= MAXREPOFFSET) offset1 = 0;

23
vendor/github.com/DataDog/zstd/zdict.h generated vendored
View File

@@ -9,16 +9,16 @@
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef ZSTD_ZDICT_H
#define ZSTD_ZDICT_H
/*====== Dependencies ======*/
#include <stddef.h> /* size_t */
#if defined (__cplusplus)
extern "C" {
#endif
/* ===== ZDICTLIB_API : control library symbols visibility ===== */
#ifndef ZDICTLIB_VISIBLE
@@ -249,7 +249,7 @@ typedef struct {
* is presumed that the most profitable content is at the end of the dictionary,
* since that is the cheapest to reference.
*
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
* `maxDictSize` must be >= max(dictContentSize, ZDICT_DICTSIZE_MIN).
*
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
* or an error code, which can be tested by ZDICT_isError().
@@ -272,11 +272,19 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_ZDICT_H */
#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
#define ZSTD_ZDICT_H_STATIC
#if defined (__cplusplus)
extern "C" {
#endif
/* This can be overridden externally to hide static symbols. */
#ifndef ZDICTLIB_STATIC_API
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
@@ -467,11 +475,10 @@ ZDICTLIB_STATIC_API
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
#endif /* ZSTD_ZDICT_H_STATIC */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_ZDICT_H_STATIC */
#endif /* USE_EXTERNAL_ZSTD */

373
vendor/github.com/DataDog/zstd/zstd.h generated vendored
View File

@@ -8,17 +8,22 @@
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef ZSTD_H_235446
#define ZSTD_H_235446
/* ====== Dependencies ======*/
#include <limits.h> /* INT_MAX */
#include <stddef.h> /* size_t */
#include "zstd_errors.h" /* list of errors */
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#include <limits.h> /* INT_MAX */
#endif /* ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
extern "C" {
#endif
/* ===== ZSTDLIB_API : control library symbols visibility ===== */
#ifndef ZSTDLIB_VISIBLE
@@ -58,7 +63,7 @@ extern "C" {
#else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZSTD_DEPRECATED(message) [[deprecated(message)]]
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__)
# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZSTD_DEPRECATED(message) __attribute__((deprecated))
@@ -107,7 +112,7 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 5
#define ZSTD_VERSION_RELEASE 6
#define ZSTD_VERSION_RELEASE 7
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
/*! ZSTD_versionNumber() :
@@ -145,7 +150,7 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
/***************************************
* Simple API
* Simple Core API
***************************************/
/*! ZSTD_compress() :
* Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
@@ -158,68 +163,80 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
int compressionLevel);
/*! ZSTD_decompress() :
* `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
* `dstCapacity` is an upper bound of originalSize to regenerate.
* If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
* @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
* or an errorCode if it fails (which can be tested using ZSTD_isError()). */
* `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
* Multiple compressed frames can be decompressed at once with this method.
* The result will be the concatenation of all decompressed frames, back to back.
* `dstCapacity` is an upper bound of originalSize to regenerate.
* First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
* If maximum upper bound isn't known, prefer using streaming mode to decompress data.
* @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
* or an errorCode if it fails (which can be tested using ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/*====== Decompression helper functions ======*/
/*! ZSTD_getFrameContentSize() : requires v1.3.0+
* `src` should point to the start of a ZSTD encoded frame.
* `srcSize` must be at least as large as the frame header.
* hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
* @return : - decompressed size of `src` frame content, if known
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
* note 1 : a 0 return value means the frame is valid but "empty".
* note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can rely on some implicit limit,
* as ZSTD_decompress() only needs an upper bound of decompressed size.
* (For example, data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size is always present when compression is completed using single-pass functions,
* such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
* note 4 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure return value fits within application's authorized limits.
* Each application can set its own limits.
* note 6 : This function replaces ZSTD_getDecompressedSize() */
* `src` should point to the start of a ZSTD encoded frame.
* `srcSize` must be at least as large as the frame header.
* hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
* @return : - decompressed size of `src` frame content, if known
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
* note 1 : a 0 return value means the frame is valid but "empty".
* When invoking this method on a skippable frame, it will return 0.
* note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode).
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can rely on some implicit limit,
* as ZSTD_decompress() only needs an upper bound of decompressed size.
* (For example, data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size is always present when compression is completed using single-pass functions,
* such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
* note 4 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure return value fits within application's authorized limits.
* Each application can set its own limits.
* note 6 : This function replaces ZSTD_getDecompressedSize() */
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
/*! ZSTD_getDecompressedSize() :
* NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
/*! ZSTD_getDecompressedSize() (obsolete):
* This function is now obsolete, in favor of ZSTD_getFrameContentSize().
* Both functions work the same way, but ZSTD_getDecompressedSize() blends
* "empty", "unknown" and "error" results to the same return value (0),
* while ZSTD_getFrameContentSize() gives them separate return values.
* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
ZSTDLIB_API
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
* `src` should point to the start of a ZSTD frame or skippable frame.
* `srcSize` must be >= first frame size
* @return : the compressed size of the first frame starting at `src`,
* suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
* or an error code if input is invalid */
* or an error code if input is invalid
* Note 1: this method is called _find*() because it's not enough to read the header,
* it may have to scan through the frame's content, to reach its end.
* Note 2: this method also works with Skippable Frames. In which case,
* it returns the size of the complete skippable frame,
* which is always equal to its content size + 8 bytes for headers. */
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
/*====== Helper functions ======*/
/* ZSTD_compressBound() :
/*====== Compression helper functions ======*/
/*! ZSTD_compressBound() :
* maximum compressed size in worst case single-pass scenario.
* When invoking `ZSTD_compress()` or any other one-pass compression function,
* When invoking `ZSTD_compress()`, or any other one-pass compression function,
* it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
* as it eliminates one potential failure scenario,
* aka not enough room in dst buffer to write the compressed frame.
* Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
* Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE .
* In which case, ZSTD_compressBound() will return an error code
* which can be tested using ZSTD_isError().
*
@@ -227,21 +244,25 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
* same as ZSTD_compressBound(), but as a macro.
* It can be used to produce constants, which can be useful for static allocation,
* for example to size a static array on stack.
* Will produce constant value 0 if srcSize too large.
* Will produce constant value 0 if srcSize is too large.
*/
#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
/*====== Error helper functions ======*/
/* ZSTD_isError() :
* Most ZSTD_* functions returning a size_t value can be tested for error,
* using ZSTD_isError().
* @return 1 if error, 0 otherwise
*/
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
/***************************************
@@ -249,17 +270,17 @@ ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compres
***************************************/
/*= Compression context
* When compressing many times,
* it is recommended to allocate a context just once,
* it is recommended to allocate a compression context just once,
* and reuse it for each successive compression operation.
* This will make workload friendlier for system's memory.
* This will make the workload easier for system's memory.
* Note : re-using context is just a speed / resource optimization.
* It doesn't change the compression ratio, which remains identical.
* Note 2 : In multi-threaded environments,
* use one different context per thread for parallel execution.
* Note 2: For parallel execution in multi-threaded environments,
* use one different context per thread .
*/
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */
/*! ZSTD_compressCCtx() :
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
@@ -267,7 +288,7 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer *
* this function compresses at the requested compression level,
* __ignoring any other advanced parameter__ .
* If any advanced parameter was set using the advanced API,
* they will all be reset. Only `compressionLevel` remains.
* they will all be reset. Only @compressionLevel remains.
*/
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
@@ -393,7 +414,7 @@ typedef enum {
* Special: value 0 means "use default strategy". */
ZSTD_c_targetCBlockSize=130, /* v1.5.6+
* Attempts to fit compressed block size into approximatively targetCBlockSize.
* Attempts to fit compressed block size into approximately targetCBlockSize.
* Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
* Note that it's not a guarantee, just a convergence target (default:0).
* No target when targetCBlockSize == 0.
@@ -489,7 +510,8 @@ typedef enum {
* ZSTD_c_stableOutBuffer
* ZSTD_c_blockDelimiters
* ZSTD_c_validateSequences
* ZSTD_c_useBlockSplitter
* ZSTD_c_blockSplitterLevel
* ZSTD_c_splitAfterSequences
* ZSTD_c_useRowMatchFinder
* ZSTD_c_prefetchCDictTables
* ZSTD_c_enableSeqProducerFallback
@@ -516,7 +538,8 @@ typedef enum {
ZSTD_c_experimentalParam16=1013,
ZSTD_c_experimentalParam17=1014,
ZSTD_c_experimentalParam18=1015,
ZSTD_c_experimentalParam19=1016
ZSTD_c_experimentalParam19=1016,
ZSTD_c_experimentalParam20=1017
} ZSTD_cParameter;
typedef struct {
@@ -856,7 +879,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
*
* A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
* ZSTD_DStream objects can be reused multiple times.
* ZSTD_DStream objects can be re-employed multiple times.
*
* Use ZSTD_initDStream() to start a new decompression operation.
* @return : recommended first input size
@@ -866,16 +889,21 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
* The function will update both `pos` fields.
* If `input.pos < input.size`, some input has not been consumed.
* It's up to the caller to present again remaining data.
*
* The function tries to flush all data decoded immediately, respecting output buffer size.
* If `output.pos < output.size`, decoder has flushed everything it could.
* But if `output.pos == output.size`, there might be some data left within internal buffers.,
*
* However, when `output.pos == output.size`, it's more difficult to know.
* If @return > 0, the frame is not complete, meaning
* either there is still some data left to flush within internal buffers,
* or there is more input to read to complete the frame (or both).
* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
* @return : 0 when a frame is completely decoded and fully flushed,
* or an error code, which can be tested using ZSTD_isError(),
* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
* the return value is a suggested next input size (just a hint for better latency)
* that will never request more than the remaining frame size.
* that will never request more than the remaining content of the compressed frame.
* *******************************************************************************/
typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
@@ -902,9 +930,10 @@ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
* Function will update both input and output `pos` fields exposing current state via these fields:
* - `input.pos < input.size`, some input remaining and caller should provide remaining input
* on the next call.
* - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
* - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
* call ZSTD_decompressStream() again to flush remaining data to output.
* - `output.pos < output.size`, decoder flushed internal output buffer.
* - `output.pos == output.size`, unflushed data potentially present in the internal buffers,
* check ZSTD_decompressStream() @return value,
* if > 0, invoke it again to flush remaining data to output.
* Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
*
* @return : 0 when a frame is completely decoded and fully flushed,
@@ -1182,6 +1211,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_H_235446 */
@@ -1197,6 +1230,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
#if defined (__cplusplus)
extern "C" {
#endif
/* This can be overridden externally to hide static symbols. */
#ifndef ZSTDLIB_STATIC_API
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
@@ -1308,7 +1345,7 @@ typedef struct {
*
* Note: This field is optional. ZSTD_generateSequences() will calculate the value of
* 'rep', but repeat offsets do not necessarily need to be calculated from an external
* sequence provider's perspective. For example, ZSTD_compressSequences() does not
* sequence provider perspective. For example, ZSTD_compressSequences() does not
* use this 'rep' field at all (as of now).
*/
} ZSTD_Sequence;
@@ -1413,14 +1450,15 @@ typedef enum {
} ZSTD_literalCompressionMode_e;
typedef enum {
/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
* decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
* or ZSTD_ps_disable allow for a force enable/disable the feature.
/* Note: This enum controls features which are conditionally beneficial.
* Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto),
* but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature.
*/
ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
ZSTD_ps_enable = 1, /* Force-enable the feature */
ZSTD_ps_disable = 2 /* Do not use the feature */
} ZSTD_paramSwitch_e;
} ZSTD_ParamSwitch_e;
#define ZSTD_paramSwitch_e ZSTD_ParamSwitch_e /* old name */
/***************************************
* Frame header and size functions
@@ -1465,34 +1503,36 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src,
ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
/*! ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
* srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX.
* @return : size of the Frame Header,
* or an error code (if srcSize is too small) */
ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e;
#define ZSTD_frameType_e ZSTD_FrameType_e /* old name */
typedef struct {
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
unsigned blockSizeMax;
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
ZSTD_FrameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
unsigned headerSize;
unsigned dictID;
unsigned dictID; /* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */
unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_frameHeader;
} ZSTD_FrameHeader;
#define ZSTD_frameHeader ZSTD_FrameHeader /* old name */
/*! ZSTD_getFrameHeader() :
* decode Frame Header, or requires larger `srcSize`.
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
* decode Frame Header into `zfhPtr`, or requires larger `srcSize`.
* @return : 0 => header is complete, `zfhPtr` is correctly filled,
* >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled,
* or an error code, which can be tested using ZSTD_isError() */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize);
/*! ZSTD_getFrameHeader_advanced() :
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
/*! ZSTD_decompressionMargin() :
* Zstd supports in-place decompression, where the input and output buffers overlap.
@@ -1540,9 +1580,10 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSi
))
typedef enum {
ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
} ZSTD_sequenceFormat_e;
ZSTD_sf_noBlockDelimiters = 0, /* ZSTD_Sequence[] has no block delimiters, just sequences */
ZSTD_sf_explicitBlockDelimiters = 1 /* ZSTD_Sequence[] contains explicit block delimiters */
} ZSTD_SequenceFormat_e;
#define ZSTD_sequenceFormat_e ZSTD_SequenceFormat_e /* old name */
/*! ZSTD_sequenceBound() :
* `srcSize` : size of the input buffer
@@ -1566,7 +1607,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
* @param zc The compression context to be used for ZSTD_compress2(). Set any
* compression parameters you need on this context.
* @param outSeqs The output sequences buffer of size @p outSeqsSize
* @param outSeqsSize The size of the output sequences buffer.
* @param outSeqsCapacity The size of the output sequences buffer.
* ZSTD_sequenceBound(srcSize) is an upper bound on the number
* of sequences that can be generated.
* @param src The source buffer to generate sequences from of size @p srcSize.
@@ -1584,7 +1625,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
ZSTDLIB_STATIC_API size_t
ZSTD_generateSequences(ZSTD_CCtx* zc,
ZSTD_Sequence* outSeqs, size_t outSeqsSize,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize);
/*! ZSTD_mergeBlockDelimiters() :
@@ -1604,7 +1645,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
* Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
* @src contains the entire input (not just the literals).
* If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
* If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.).
* The entire source is compressed into a single frame.
*
* The compression behavior changes based on cctx params. In particular:
@@ -1613,11 +1654,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
* the block size derived from the cctx, and sequences may be split. This is the default setting.
*
* If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
* block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
* valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
*
* If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
* behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
* specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
* When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes
* using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit
* can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation.
* By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10).
* ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction.
*
* If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined
* behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for
* specifics regarding offset/matchlength requirements) and then bail out and return an error.
*
* In addition to the two adjustable experimental params, there are other important cctx params.
* - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
@@ -1625,15 +1672,42 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
* - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
* is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
*
* Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
* Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
* and cannot emit an RLE block that disagrees with the repcode history
* Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused.
* Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly,
* and cannot emit an RLE block that disagrees with the repcode history.
* @return : final compressed size, or a ZSTD error code.
*/
ZSTDLIB_STATIC_API size_t
ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize);
ZSTD_compressSequences(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize);
/*! ZSTD_compressSequencesAndLiterals() :
* This is a variant of ZSTD_compressSequences() which,
* instead of receiving (src,srcSize) as input parameter, receives (literals,litSize),
* aka all the literals, already extracted and laid out into a single continuous buffer.
* This can be useful if the process generating the sequences also happens to generate the buffer of literals,
* thus skipping an extraction + caching stage.
* It's a speed optimization, useful when the right conditions are met,
* but it also features the following limitations:
* - Only supports explicit delimiter mode
* - Currently does not support Sequences validation (so input Sequences are trusted)
* - Not compatible with frame checksum, which must be disabled
* - If any block is incompressible, will fail and return an error
* - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error.
* - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals.
* @litBufCapacity must be at least 8 bytes larger than @litSize.
* - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error.
* @return : final compressed size, or a ZSTD error code.
*/
ZSTDLIB_STATIC_API size_t
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t nbSequences,
const void* literals, size_t litSize, size_t litBufCapacity,
size_t decompressedSize);
/*! ZSTD_writeSkippableFrame() :
@@ -1641,8 +1715,8 @@ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
*
* Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
* ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
* the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used,
* so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
*
* Returns an error if destination buffer is not large enough, if the source size is not representable
* with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
@@ -1650,26 +1724,28 @@ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
* @return : number of bytes written or a ZSTD error.
*/
ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
const void* src, size_t srcSize, unsigned magicVariant);
const void* src, size_t srcSize,
unsigned magicVariant);
/*! ZSTD_readSkippableFrame() :
* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
* Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer.
*
* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
* in the magicVariant.
* The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written,
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.
* This can be NULL if the caller is not interested in the magicVariant.
*
* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
*
* @return : number of bytes written or a ZSTD error.
*/
ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
const void* src, size_t srcSize);
ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
unsigned* magicVariant,
const void* src, size_t srcSize);
/*! ZSTD_isSkippableFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
*/
ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
@@ -1797,7 +1873,15 @@ static
#ifdef __GNUC__
__attribute__((__unused__))
#endif
#if defined(__clang__) && __clang_major__ >= 5
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#endif
ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
#if defined(__clang__) && __clang_major__ >= 5
#pragma clang diagnostic pop
#endif
ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
@@ -1977,7 +2061,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* See the comments on that enum for an explanation of the feature. */
#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
/* Controlled with ZSTD_paramSwitch_e enum.
/* Controlled with ZSTD_ParamSwitch_e enum.
* Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never compress literals.
* Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
@@ -2118,22 +2202,46 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
/* ZSTD_c_validateSequences
* Default is 0 == disabled. Set to 1 to enable sequence validation.
*
* For use with sequence compression API: ZSTD_compressSequences().
* Designates whether or not we validate sequences provided to ZSTD_compressSequences()
* For use with sequence compression API: ZSTD_compressSequences*().
* Designates whether or not provided sequences are validated within ZSTD_compressSequences*()
* during function execution.
*
* Without validation, providing a sequence that does not conform to the zstd spec will cause
* undefined behavior, and may produce a corrupted block.
* When Sequence validation is disabled (default), Sequences are compressed as-is,
* so they must correct, otherwise it would result in a corruption error.
*
* With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
* Sequence validation adds some protection, by ensuring that all values respect boundary conditions.
* If a Sequence is detected invalid (see doc/zstd_compression_format.md for
* specifics regarding offset/matchlength requirements) then the function will bail out and
* return an error.
*
*/
#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
/* ZSTD_c_useBlockSplitter
* Controlled with ZSTD_paramSwitch_e enum.
/* ZSTD_c_blockSplitterLevel
* note: this parameter only influences the first splitter stage,
* which is active before producing the sequences.
* ZSTD_c_splitAfterSequences controls the next splitter stage,
* which is active after sequence production.
* Note that both can be combined.
* Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included.
* 0 means "auto", which will select a value depending on current ZSTD_c_strategy.
* 1 means no splitting.
* Then, values from 2 to 6 are sorted in increasing cpu load order.
*
* Note that currently the first block is never split,
* to ensure expansion guarantees in presence of incompressible data.
*/
#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6
#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20
/* ZSTD_c_splitAfterSequences
* This is a stronger splitter algorithm,
* based on actual sequences previously produced by the selected parser.
* It's also slower, and as a consequence, mostly used for high compression levels.
* While the post-splitter does overlap with the pre-splitter,
* both can nonetheless be combined,
* notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX,
* resulting in higher compression ratio than just one of them.
*
* Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never use block splitter.
* Set to ZSTD_ps_enable to always use block splitter.
@@ -2141,10 +2249,10 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
* block splitting based on the compression parameters.
*/
#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13
/* ZSTD_c_useRowMatchFinder
* Controlled with ZSTD_paramSwitch_e enum.
* Controlled with ZSTD_ParamSwitch_e enum.
* Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never use row-based matchfinder.
* Set to ZSTD_ps_enable to force usage of row-based matchfinder.
@@ -2176,7 +2284,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
/* ZSTD_c_prefetchCDictTables
* Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
* Controlled with ZSTD_ParamSwitch_e enum. Default is ZSTD_ps_auto.
*
* In some situations, zstd uses CDict tables in-place rather than copying them
* into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
@@ -2220,19 +2328,21 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
* bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
* compressBound() inaccurate). Only currently meant to be used for testing.
*
*/
#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
/* ZSTD_c_searchForExternalRepcodes
* This parameter affects how zstd parses external sequences, such as sequences
* provided through the compressSequences() API or from an external block-level
* sequence producer.
/* ZSTD_c_repcodeResolution
* This parameter only has an effect if ZSTD_c_blockDelimiters is
* set to ZSTD_sf_explicitBlockDelimiters (may change in the future).
*
* If set to ZSTD_ps_enable, the library will check for repeated offsets in
* This parameter affects how zstd parses external sequences,
* provided via the ZSTD_compressSequences*() API
* or from an external block-level sequence producer.
*
* If set to ZSTD_ps_enable, the library will check for repeated offsets within
* external sequences, even if those repcodes are not explicitly indicated in
* the "rep" field. Note that this is the only way to exploit repcode matches
* while using compressSequences() or an external sequence producer, since zstd
* while using compressSequences*() or an external sequence producer, since zstd
* currently ignores the "rep" field of external sequences.
*
* If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
@@ -2241,12 +2351,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* compression ratio.
*
* The default value is ZSTD_ps_auto, for which the library will enable/disable
* based on compression level.
*
* Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
* set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
* based on compression level (currently: level<10 disables, level>=10 enables).
*/
#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
#define ZSTD_c_repcodeResolution ZSTD_c_experimentalParam19
#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 /* older name */
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
@@ -2953,7 +3062,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame,
such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
As a consequence, check that values remain within valid application range.
@@ -3083,12 +3192,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
#else /* USE_EXTERNAL_ZSTD */
#include_next <zstd.h>
#endif /* USE_EXTERNAL_ZSTD */

1424
vendor/github.com/DataDog/zstd/zstd_compress.c generated vendored
View File

File diff suppressed because it is too large Load Diff

352
vendor/github.com/DataDog/zstd/zstd_compress_internal.h generated vendored
View File

@@ -25,10 +25,7 @@
# include "zstdmt_compress.h"
#endif
#include "bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */
/*-*************************************
* Constants
@@ -83,6 +80,70 @@ typedef struct {
ZSTD_fseCTables_t fse;
} ZSTD_entropyCTables_t;
/***********************************************
* Sequences *
***********************************************/
typedef struct SeqDef_s {
U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
U16 litLength;
U16 mlBase; /* mlBase == matchLength - MINMATCH */
} SeqDef;
/* Controls whether seqStore has a single "long" litLength or matchLength. See SeqStore_t. */
typedef enum {
ZSTD_llt_none = 0, /* no longLengthType */
ZSTD_llt_literalLength = 1, /* represents a long literal */
ZSTD_llt_matchLength = 2 /* represents a long match */
} ZSTD_longLengthType_e;
typedef struct {
SeqDef* sequencesStart;
SeqDef* sequences; /* ptr to end of sequences */
BYTE* litStart;
BYTE* lit; /* ptr to end of literals */
BYTE* llCode;
BYTE* mlCode;
BYTE* ofCode;
size_t maxNbSeq;
size_t maxNbLit;
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
* the existing value of the litLength or matchLength by 0x10000.
*/
ZSTD_longLengthType_e longLengthType;
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
} SeqStore_t;
typedef struct {
U32 litLength;
U32 matchLength;
} ZSTD_SequenceLength;
/**
* Returns the ZSTD_SequenceLength for the given sequences. It handles the decoding of long sequences
* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
*/
MEM_STATIC ZSTD_SequenceLength ZSTD_getSequenceLength(SeqStore_t const* seqStore, SeqDef const* seq)
{
ZSTD_SequenceLength seqLen;
seqLen.litLength = seq->litLength;
seqLen.matchLength = seq->mlBase + MINMATCH;
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
seqLen.litLength += 0x10000;
}
if (seqStore->longLengthType == ZSTD_llt_matchLength) {
seqLen.matchLength += 0x10000;
}
}
return seqLen;
}
const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
/***********************************************
* Entropy buffer statistics structs and funcs *
***********************************************/
@@ -92,7 +153,7 @@ typedef struct {
* hufDesSize refers to the size of huffman tree description in bytes.
* This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
typedef struct {
symbolEncodingType_e hType;
SymbolEncodingType_e hType;
BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
size_t hufDesSize;
} ZSTD_hufCTablesMetadata_t;
@@ -103,9 +164,9 @@ typedef struct {
* fseTablesSize refers to the size of fse tables in bytes.
* This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
typedef struct {
symbolEncodingType_e llType;
symbolEncodingType_e ofType;
symbolEncodingType_e mlType;
SymbolEncodingType_e llType;
SymbolEncodingType_e ofType;
SymbolEncodingType_e mlType;
BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
size_t fseTablesSize;
size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
@@ -120,7 +181,7 @@ typedef struct {
* Builds entropy for the block.
* @return : 0 on success or error code */
size_t ZSTD_buildBlockEntropyStats(
const seqStore_t* seqStorePtr,
const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
@@ -149,15 +210,9 @@ typedef struct {
stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */
size_t size; /* The number of sequences. <= capacity. */
size_t capacity; /* The capacity starting from `seq` pointer */
} rawSeqStore_t;
} RawSeqStore_t;
typedef struct {
U32 idx; /* Index in array of ZSTD_Sequence */
U32 posInSequence; /* Position within sequence at idx */
size_t posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_sequencePosition;
UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
UNUSED_ATTR static const RawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
typedef struct {
int price; /* price from beginning of segment to this position */
@@ -189,7 +244,7 @@ typedef struct {
U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
ZSTD_paramSwitch_e literalCompressionMode;
ZSTD_ParamSwitch_e literalCompressionMode;
} optState_t;
typedef struct {
@@ -211,11 +266,11 @@ typedef struct {
#define ZSTD_WINDOW_START_INDEX 2
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
typedef struct ZSTD_MatchState_t ZSTD_MatchState_t;
#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
struct ZSTD_matchState_t {
struct ZSTD_MatchState_t {
ZSTD_window_t window; /* State for window round buffer management */
U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
* When loadedDictEnd != 0, a dictionary is in use, and still valid.
@@ -237,15 +292,15 @@ struct ZSTD_matchState_t {
U32* hashTable3;
U32* chainTable;
U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
int forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
int dedicatedDictSearch; /* Indicates whether this matchState is using the
* dedicated dictionary search structure.
*/
optState_t opt; /* optimal parser state */
const ZSTD_matchState_t* dictMatchState;
const ZSTD_MatchState_t* dictMatchState;
ZSTD_compressionParameters cParams;
const rawSeqStore_t* ldmSeqStore;
const RawSeqStore_t* ldmSeqStore;
/* Controls prefetching in some dictMatchState matchfinders.
* This behavior is controlled from the cctx ms.
@@ -263,7 +318,7 @@ struct ZSTD_matchState_t {
typedef struct {
ZSTD_compressedBlockState_t* prevCBlock;
ZSTD_compressedBlockState_t* nextCBlock;
ZSTD_matchState_t matchState;
ZSTD_MatchState_t matchState;
} ZSTD_blockState_t;
typedef struct {
@@ -290,7 +345,7 @@ typedef struct {
} ldmState_t;
typedef struct {
ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
ZSTD_ParamSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
U32 hashLog; /* Log size of hashTable */
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */
@@ -321,7 +376,7 @@ struct ZSTD_CCtx_params_s {
* There is no guarantee that hint is close to actual source size */
ZSTD_dictAttachPref_e attachDictPref;
ZSTD_paramSwitch_e literalCompressionMode;
ZSTD_ParamSwitch_e literalCompressionMode;
/* Multithreading: used to pass parameters to mtctx */
int nbWorkers;
@@ -340,14 +395,27 @@ struct ZSTD_CCtx_params_s {
ZSTD_bufferMode_e outBufferMode;
/* Sequence compression API */
ZSTD_sequenceFormat_e blockDelimiters;
ZSTD_SequenceFormat_e blockDelimiters;
int validateSequences;
/* Block splitting */
ZSTD_paramSwitch_e useBlockSplitter;
/* Block splitting
* @postBlockSplitter executes split analysis after sequences are produced,
* it's more accurate but consumes more resources.
* @preBlockSplitter_level splits before knowing sequences,
* it's more approximative but also cheaper.
* Valid @preBlockSplitter_level values range from 0 to 6 (included).
* 0 means auto, 1 means do not split,
* then levels are sorted in increasing cpu budget, from 2 (fastest) to 6 (slowest).
* Highest @preBlockSplitter_level combines well with @postBlockSplitter.
*/
ZSTD_ParamSwitch_e postBlockSplitter;
int preBlockSplitter_level;
/* Adjust the max block size*/
size_t maxBlockSize;
/* Param for deciding whether to use row-based matchfinder */
ZSTD_paramSwitch_e useRowMatchFinder;
ZSTD_ParamSwitch_e useRowMatchFinder;
/* Always load a dictionary in ext-dict mode (not prefix mode)? */
int deterministicRefPrefix;
@@ -356,7 +424,7 @@ struct ZSTD_CCtx_params_s {
ZSTD_customMem customMem;
/* Controls prefetching in some dictMatchState matchfinders */
ZSTD_paramSwitch_e prefetchCDictTables;
ZSTD_ParamSwitch_e prefetchCDictTables;
/* Controls whether zstd will fall back to an internal matchfinder
* if the external matchfinder returns an error code. */
@@ -368,15 +436,13 @@ struct ZSTD_CCtx_params_s {
void* extSeqProdState;
ZSTD_sequenceProducer_F extSeqProdFunc;
/* Adjust the max block size*/
size_t maxBlockSize;
/* Controls repcode search in external sequence parsing */
ZSTD_paramSwitch_e searchForExternalRepcodes;
ZSTD_ParamSwitch_e searchForExternalRepcodes;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
#define TMP_WORKSPACE_SIZE (MAX(ENTROPY_WORKSPACE_SIZE, ZSTD_SLIPBLOCK_WORKSPACESIZE))
/**
* Indicates whether this compression proceeds directly from user-provided
@@ -394,11 +460,11 @@ typedef enum {
*/
#define ZSTD_MAX_NB_BLOCK_SPLITS 196
typedef struct {
seqStore_t fullSeqStoreChunk;
seqStore_t firstHalfSeqStore;
seqStore_t secondHalfSeqStore;
seqStore_t currSeqStore;
seqStore_t nextSeqStore;
SeqStore_t fullSeqStoreChunk;
SeqStore_t firstHalfSeqStore;
SeqStore_t secondHalfSeqStore;
SeqStore_t currSeqStore;
SeqStore_t nextSeqStore;
U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
ZSTD_entropyCTablesMetadata_t entropyMetadata;
@@ -415,7 +481,7 @@ struct ZSTD_CCtx_s {
size_t dictContentSize;
ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
size_t blockSize;
size_t blockSizeMax;
unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
unsigned long long consumedSrcSize;
unsigned long long producedCSize;
@@ -427,13 +493,14 @@ struct ZSTD_CCtx_s {
int isFirstBlock;
int initialized;
seqStore_t seqStore; /* sequences storage ptrs */
SeqStore_t seqStore; /* sequences storage ptrs */
ldmState_t ldmState; /* long distance matching state */
rawSeq* ldmSequences; /* Storage for the ldm output sequences */
size_t maxNbLdmSequences;
rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
RawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
ZSTD_blockState_t blockState;
U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
void* tmpWorkspace; /* used as substitute of stack space - must be aligned for S64 type */
size_t tmpWkspSize;
/* Whether we are streaming or not */
ZSTD_buffered_policy_e bufferedPolicy;
@@ -507,12 +574,12 @@ typedef enum {
* behavior of taking both the source size and the dict size into account
* when selecting and adjusting parameters.
*/
} ZSTD_cParamMode_e;
} ZSTD_CParamMode_e;
typedef size_t (*ZSTD_blockCompressor) (
ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
typedef size_t (*ZSTD_BlockCompressor_f) (
ZSTD_MatchState_t* bs, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@@ -558,6 +625,25 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
return 1;
}
/* ZSTD_selectAddr:
* @return index >= lowLimit ? candidate : backup,
* tries to force branchless codegen. */
MEM_STATIC const BYTE*
ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* backup)
{
#if defined(__GNUC__) && defined(__x86_64__)
__asm__ (
"cmp %1, %2\n"
"cmova %3, %0\n"
: "+r"(candidate)
: "r"(index), "r"(lowLimit), "r"(backup)
);
return candidate;
#else
return index >= lowLimit ? candidate : backup;
#endif
}
/* ZSTD_noCompressBlock() :
* Writes uncompressed block to dst buffer from given src.
* Returns the size of the block */
@@ -640,14 +726,55 @@ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE con
#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM)
#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */
/*! ZSTD_storeSeqOnly() :
* Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t.
* Literals themselves are not copied, but @litPtr is updated.
* @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
* @matchLength : must be >= MINMATCH
*/
HINT_INLINE UNUSED_ATTR void
ZSTD_storeSeqOnly(SeqStore_t* seqStorePtr,
size_t litLength,
U32 offBase,
size_t matchLength)
{
assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
/* literal Length */
assert(litLength <= ZSTD_BLOCKSIZE_MAX);
if (UNLIKELY(litLength>0xFFFF)) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */
seqStorePtr->sequences[0].offBase = offBase;
/* match Length */
assert(matchLength <= ZSTD_BLOCKSIZE_MAX);
assert(matchLength >= MINMATCH);
{ size_t const mlBase = matchLength - MINMATCH;
if (UNLIKELY(mlBase>0xFFFF)) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].mlBase = (U16)mlBase;
}
seqStorePtr->sequences++;
}
/*! ZSTD_storeSeq() :
* Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t.
* Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t.
* @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
* @matchLength : must be >= MINMATCH
* Allowed to over-read literals up to litLimit.
*/
HINT_INLINE UNUSED_ATTR void
ZSTD_storeSeq(seqStore_t* seqStorePtr,
ZSTD_storeSeq(SeqStore_t* seqStorePtr,
size_t litLength, const BYTE* literals, const BYTE* litLimit,
U32 offBase,
size_t matchLength)
@@ -681,29 +808,7 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
}
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */
seqStorePtr->sequences[0].offBase = offBase;
/* match Length */
assert(matchLength >= MINMATCH);
{ size_t const mlBase = matchLength - MINMATCH;
if (mlBase>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].mlBase = (U16)mlBase;
}
seqStorePtr->sequences++;
ZSTD_storeSeqOnly(seqStorePtr, litLength, offBase, matchLength);
}
/* ZSTD_updateRep() :
@@ -732,12 +837,12 @@ ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
typedef struct repcodes_s {
U32 rep[3];
} repcodes_t;
} Repcodes_t;
MEM_STATIC repcodes_t
MEM_STATIC Repcodes_t
ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
{
repcodes_t newReps;
Repcodes_t newReps;
ZSTD_memcpy(&newReps, rep, sizeof(newReps));
ZSTD_updateRep(newReps.rep, offBase, ll0);
return newReps;
@@ -780,8 +885,8 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
size_t const matchLength = ZSTD_count(ip, match, vEnd);
if (match + matchLength != mEnd) return matchLength;
DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength);
DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match);
DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip);
DEBUGLOG(7, "distance from match beginning to end dictionary = %i", (int)(mEnd - match));
DEBUGLOG(7, "distance from current pos to end buffer = %i", (int)(iEnd - ip));
DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart);
DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd));
return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
@@ -919,11 +1024,12 @@ MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64
/*-*************************************
* Round buffer management
***************************************/
#if (ZSTD_WINDOWLOG_MAX_64 > 31)
# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX"
#endif
/* Max current allowed */
#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX))
/* Max @current value allowed:
* In 32-bit mode: we want to avoid crossing the 2 GB limit,
* reducing risks of side effects in case of signed operations on indexes.
* In 64-bit mode: we want to ensure that adding the maximum job size (512 MB)
* doesn't overflow U32 index capacity (4 GB) */
#define ZSTD_CURRENT_MAX (MEM_64bits() ? 3500U MB : 2000U MB)
/* Maximum chunk size before overflow correction needs to be called again */
#define ZSTD_CHUNKSIZE_MAX \
( ((U32)-1) /* Maximum ending current index */ \
@@ -963,7 +1069,7 @@ MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
* Inspects the provided matchState and figures out what dictMode should be
* passed to the compressor.
*/
MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms)
{
return ZSTD_window_hasExtDict(ms->window) ?
ZSTD_extDict :
@@ -1152,7 +1258,7 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
const void* blockEnd,
U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_matchState_t** dictMatchStatePtr)
const ZSTD_MatchState_t** dictMatchStatePtr)
{
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
@@ -1197,7 +1303,7 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
const void* blockEnd,
U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_matchState_t** dictMatchStatePtr)
const ZSTD_MatchState_t** dictMatchStatePtr)
{
assert(loadedDictEndPtr != NULL);
assert(dictMatchStatePtr != NULL);
@@ -1247,8 +1353,8 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
MEM_STATIC
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_window_update(ZSTD_window_t* window,
void const* src, size_t srcSize,
int forceNonContiguous)
const void* src, size_t srcSize,
int forceNonContiguous)
{
BYTE const* const ip = (BYTE const*)src;
U32 contiguous = 1;
@@ -1275,8 +1381,9 @@ U32 ZSTD_window_update(ZSTD_window_t* window,
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
if ( (ip+srcSize > window->dictBase + window->lowLimit)
& (ip < window->dictBase + window->dictLimit)) {
ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase;
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
size_t const highInputIdx = (size_t)((ip + srcSize) - window->dictBase);
U32 const lowLimitMax = (highInputIdx > (size_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
assert(highInputIdx < UINT_MAX);
window->lowLimit = lowLimitMax;
DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit);
}
@@ -1286,7 +1393,7 @@ U32 ZSTD_window_update(ZSTD_window_t* window,
/**
* Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
*/
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog)
{
U32 const maxDistance = 1U << windowLog;
U32 const lowestValid = ms->window.lowLimit;
@@ -1303,7 +1410,7 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, u
/**
* Returns the lowest allowed match index in the prefix.
*/
MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog)
{
U32 const maxDistance = 1U << windowLog;
U32 const lowestValid = ms->window.dictLimit;
@@ -1316,6 +1423,13 @@ MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr,
return matchLowest;
}
/* index_safety_check:
* intentional underflow : ensure repIndex isn't overlapping dict + prefix
* @return 1 if values are not overlapping,
* 0 otherwise */
MEM_STATIC int ZSTD_index_overlap_check(const U32 prefixLowestIndex, const U32 repIndex) {
return ((U32)((prefixLowestIndex-1) - repIndex) >= 3);
}
/* debug functions */
@@ -1386,10 +1500,6 @@ MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) {
return tag1 == tag2;
}
#if defined (__cplusplus)
}
#endif
/* ===============================================================
* Shared internal declarations
* These prototypes may be called from sources not in lib/compress
@@ -1405,6 +1515,25 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
typedef struct {
U32 idx; /* Index in array of ZSTD_Sequence */
U32 posInSequence; /* Position within sequence at idx */
size_t posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_SequencePosition;
/* for benchmark */
size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
int const repcodeResolution);
typedef struct {
size_t nbSequences;
size_t blockSize;
size_t litSize;
} BlockSummary;
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs);
/* ==============================================================
* Private declarations
* These prototypes shall only be called from within lib/compress
@@ -1416,7 +1545,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
* Note: srcSizeHint == 0 means 0!
*/
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode);
/*! ZSTD_initCStream_internal() :
* Private use only. Init streaming operation.
@@ -1428,7 +1557,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
const ZSTD_CDict* cdict,
const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize);
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
void ZSTD_resetSeqStore(SeqStore_t* ssPtr);
/*! ZSTD_getCParamsFromCDict() :
* as the name implies */
@@ -1481,33 +1610,6 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
*/
void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
* ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
* Note that the block delimiter must include the last literals of the block.
*/
size_t
ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
/* Returns the number of bytes to move the current read position back by.
* Only non-zero if we ended up splitting a sequence.
* Otherwise, it may return a ZSTD error if something went wrong.
*
* This function will attempt to scan through blockSize bytes
* represented by the sequences in @inSeqs,
* storing any (partial) sequences.
*
* Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
* avoid splitting a match, or to avoid splitting a match such that it would produce a match
* smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
*/
size_t
ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
/* Returns 1 if an external sequence producer is registered, otherwise returns 0. */
MEM_STATIC int ZSTD_hasExtSeqProd(const ZSTD_CCtx_params* params) {
return params->extSeqProdFunc != NULL;

2
vendor/github.com/DataDog/zstd/zstd_compress_literals.c generated vendored
View File

@@ -141,7 +141,7 @@ size_t ZSTD_compressLiterals (
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
BYTE* const ostart = (BYTE*)dst;
U32 singleStream = srcSize < 256;
symbolEncodingType_e hType = set_compressed;
SymbolEncodingType_e hType = set_compressed;
size_t cLitSize;
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",

14
vendor/github.com/DataDog/zstd/zstd_compress_sequences.c generated vendored
View File

@@ -154,13 +154,13 @@ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
return cost >> 8;
}
symbolEncodingType_e
SymbolEncodingType_e
ZSTD_selectEncodingType(
FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
FSE_CTable const* prevCTable,
short const* defaultNorm, U32 defaultNormLog,
ZSTD_defaultPolicy_e const isDefaultAllowed,
ZSTD_DefaultPolicy_e const isDefaultAllowed,
ZSTD_strategy const strategy)
{
ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0);
@@ -242,7 +242,7 @@ typedef struct {
size_t
ZSTD_buildCTable(void* dst, size_t dstCapacity,
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type,
unsigned* count, U32 max,
const BYTE* codeTable, size_t nbSeq,
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
@@ -294,7 +294,7 @@ ZSTD_encodeSequences_body(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
seqDef const* sequences, size_t nbSeq, int longOffsets)
SeqDef const* sequences, size_t nbSeq, int longOffsets)
{
BIT_CStream_t blockStream;
FSE_CState_t stateMatchLength;
@@ -388,7 +388,7 @@ ZSTD_encodeSequences_default(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
seqDef const* sequences, size_t nbSeq, int longOffsets)
SeqDef const* sequences, size_t nbSeq, int longOffsets)
{
return ZSTD_encodeSequences_body(dst, dstCapacity,
CTable_MatchLength, mlCodeTable,
@@ -406,7 +406,7 @@ ZSTD_encodeSequences_bmi2(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
seqDef const* sequences, size_t nbSeq, int longOffsets)
SeqDef const* sequences, size_t nbSeq, int longOffsets)
{
return ZSTD_encodeSequences_body(dst, dstCapacity,
CTable_MatchLength, mlCodeTable,
@@ -422,7 +422,7 @@ size_t ZSTD_encodeSequences(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
{
DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity);
#if DYNAMIC_BMI2

13
vendor/github.com/DataDog/zstd/zstd_compress_sequences.h generated vendored
View File

@@ -12,26 +12,27 @@
#ifndef ZSTD_COMPRESS_SEQUENCES_H
#define ZSTD_COMPRESS_SEQUENCES_H
#include "zstd_compress_internal.h" /* SeqDef */
#include "fse.h" /* FSE_repeat, FSE_CTable */
#include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
#include "zstd_internal.h" /* SymbolEncodingType_e, ZSTD_strategy */
typedef enum {
ZSTD_defaultDisallowed = 0,
ZSTD_defaultAllowed = 1
} ZSTD_defaultPolicy_e;
} ZSTD_DefaultPolicy_e;
symbolEncodingType_e
SymbolEncodingType_e
ZSTD_selectEncodingType(
FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
FSE_CTable const* prevCTable,
short const* defaultNorm, U32 defaultNormLog,
ZSTD_defaultPolicy_e const isDefaultAllowed,
ZSTD_DefaultPolicy_e const isDefaultAllowed,
ZSTD_strategy const strategy);
size_t
ZSTD_buildCTable(void* dst, size_t dstCapacity,
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type,
unsigned* count, U32 max,
const BYTE* codeTable, size_t nbSeq,
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
@@ -43,7 +44,7 @@ size_t ZSTD_encodeSequences(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
size_t ZSTD_fseBitCost(
FSE_CTable const* ctable,

34
vendor/github.com/DataDog/zstd/zstd_compress_superblock.c generated vendored
View File

@@ -52,7 +52,7 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart + lhSize;
U32 const singleStream = lhSize == 3;
symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
SymbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
size_t cLitSize = 0;
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
@@ -127,15 +127,15 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
}
static size_t
ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
const seqDef* sequences, size_t nbSeqs,
ZSTD_seqDecompressedSize(SeqStore_t const* seqStore,
const SeqDef* sequences, size_t nbSeqs,
size_t litSize, int lastSubBlock)
{
size_t matchLengthSum = 0;
size_t litLengthSum = 0;
size_t n;
for (n=0; n<nbSeqs; n++) {
const ZSTD_sequenceLength seqLen = ZSTD_getSequenceLength(seqStore, sequences+n);
const ZSTD_SequenceLength seqLen = ZSTD_getSequenceLength(seqStore, sequences+n);
litLengthSum += seqLen.litLength;
matchLengthSum += seqLen.matchLength;
}
@@ -163,7 +163,7 @@ ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
static size_t
ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
const ZSTD_fseCTablesMetadata_t* fseMetadata,
const seqDef* sequences, size_t nbSeq,
const SeqDef* sequences, size_t nbSeq,
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity,
@@ -263,7 +263,7 @@ ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
* Or 0 if it failed to compress. */
static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
const seqDef* sequences, size_t nbSeq,
const SeqDef* sequences, size_t nbSeq,
const BYTE* literals, size_t litSize,
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
const ZSTD_CCtx_params* cctxParams,
@@ -328,7 +328,7 @@ static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t lit
return 0;
}
static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
static size_t ZSTD_estimateSubBlockSize_symbolType(SymbolEncodingType_e type,
const BYTE* codeTable, unsigned maxCode,
size_t nbSeq, const FSE_CTable* fseCTable,
const U8* additionalBits,
@@ -427,7 +427,7 @@ static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMe
return 0;
}
static size_t countLiterals(seqStore_t const* seqStore, const seqDef* sp, size_t seqCount)
static size_t countLiterals(SeqStore_t const* seqStore, const SeqDef* sp, size_t seqCount)
{
size_t n, total = 0;
assert(sp != NULL);
@@ -440,7 +440,7 @@ static size_t countLiterals(seqStore_t const* seqStore, const seqDef* sp, size_t
#define BYTESCALE 256
static size_t sizeBlockSequences(const seqDef* sp, size_t nbSeqs,
static size_t sizeBlockSequences(const SeqDef* sp, size_t nbSeqs,
size_t targetBudget, size_t avgLitCost, size_t avgSeqCost,
int firstSubBlock)
{
@@ -477,7 +477,7 @@ static size_t sizeBlockSequences(const seqDef* sp, size_t nbSeqs,
* Sub-blocks are all compressed, except the last one when beneficial.
* @return : compressed size of the super block (which features multiple ZSTD blocks)
* or 0 if it failed to compress. */
static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr,
const ZSTD_compressedBlockState_t* prevCBlock,
ZSTD_compressedBlockState_t* nextCBlock,
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
@@ -487,9 +487,9 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
const int bmi2, U32 lastBlock,
void* workspace, size_t wkspSize)
{
const seqDef* const sstart = seqStorePtr->sequencesStart;
const seqDef* const send = seqStorePtr->sequences;
const seqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */
const SeqDef* const sstart = seqStorePtr->sequencesStart;
const SeqDef* const send = seqStorePtr->sequences;
const SeqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */
size_t const nbSeqs = (size_t)(send - sstart);
const BYTE* const lstart = seqStorePtr->litStart;
const BYTE* const lend = seqStorePtr->lit;
@@ -648,8 +648,8 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
op += cSize;
/* We have to regenerate the repcodes because we've skipped some sequences */
if (sp < send) {
const seqDef* seq;
repcodes_t rep;
const SeqDef* seq;
Repcodes_t rep;
ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
for (seq = sstart; seq < sp; ++seq) {
ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
@@ -675,7 +675,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
&zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
&entropyMetadata,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), "");
return ZSTD_compressSubBlock_multi(&zc->seqStore,
zc->blockState.prevCBlock,
@@ -685,7 +685,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
dst, dstCapacity,
src, srcSize,
zc->bmi2, lastBlock,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */);
}
#endif /* USE_EXTERNAL_ZSTD */

29
vendor/github.com/DataDog/zstd/zstd_ctx.go generated vendored
View File

@@ -24,6 +24,15 @@ type Ctx interface {
// prevent allocation. If it is too small, or if nil is passed, a new buffer
// will be allocated and returned.
Decompress(dst, src []byte) ([]byte, error)
// DecompressInto decompresses src into dst. Unlike Decompress, DecompressInto
// requires that dst be sufficiently large to hold the decompressed payload.
// DecompressInto may be used when the caller knows the size of the decompressed
// payload before attempting decompression.
//
// It returns the number of bytes copied and an error if any is encountered. If
// dst is too small, DecompressInto errors.
DecompressInto(dst, src []byte) (int, error)
}
type ctx struct {
@@ -104,14 +113,7 @@ func (c *ctx) Decompress(dst, src []byte) ([]byte, error) {
dst = make([]byte, bound)
}
written := int(C.ZSTD_decompressDCtx(
c.dctx,
unsafe.Pointer(&dst[0]),
C.size_t(len(dst)),
unsafe.Pointer(&src[0]),
C.size_t(len(src))))
err := getError(written)
written, err := c.DecompressInto(dst, src)
if err == nil {
return dst[:written], nil
}
@@ -125,6 +127,17 @@ func (c *ctx) Decompress(dst, src []byte) ([]byte, error) {
return ioutil.ReadAll(r)
}
func (c *ctx) DecompressInto(dst, src []byte) (int, error) {
written := int(C.ZSTD_decompressDCtx(
c.dctx,
unsafe.Pointer(&dst[0]),
C.size_t(len(dst)),
unsafe.Pointer(&src[0]),
C.size_t(len(src))))
err := getError(written)
return written, err
}
func finalizeCtx(c *ctx) {
C.ZSTD_freeCCtx(c.cctx)
C.ZSTD_freeDCtx(c.dctx)

65
vendor/github.com/DataDog/zstd/zstd_cwksp.h generated vendored
View File

@@ -18,10 +18,7 @@
#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */
#include "zstd_internal.h"
#include "portability_macros.h"
#if defined (__cplusplus)
extern "C" {
#endif
#include "compiler.h" /* ZS2_isPower2 */
/*-*************************************
* Constants
@@ -207,9 +204,9 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
/**
* Align must be a power of 2.
*/
MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t align) {
size_t const mask = align - 1;
assert((align & mask) == 0);
assert(ZSTD_isPower2(align));
return (size + mask) & ~mask;
}
@@ -223,7 +220,7 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
* to figure out how much space you need for the matchState tables. Everything
* else is though.
*
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size().
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned64_alloc_size().
*/
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
if (size == 0)
@@ -235,12 +232,16 @@ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
#endif
}
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size, size_t alignment) {
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, alignment));
}
/**
* Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes.
* Used to determine the number of bytes required for a given "aligned".
*/
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES));
MEM_STATIC size_t ZSTD_cwksp_aligned64_alloc_size(size_t size) {
return ZSTD_cwksp_aligned_alloc_size(size, ZSTD_CWKSP_ALIGNMENT_BYTES);
}
/**
@@ -263,7 +264,7 @@ MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) {
size_t const alignBytesMask = alignBytes - 1;
size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
assert((alignBytes & alignBytesMask) == 0);
assert(ZSTD_isPower2(alignBytes));
assert(bytes < alignBytes);
return bytes;
}
@@ -272,8 +273,12 @@ MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignByt
* Returns the initial value for allocStart which is used to determine the position from
* which we can allocate from the end of the workspace.
*/
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) {
return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1));
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws)
{
char* endPtr = (char*)ws->workspaceEnd;
assert(ZSTD_isPower2(ZSTD_CWKSP_ALIGNMENT_BYTES));
endPtr = endPtr - ((size_t)endPtr % ZSTD_CWKSP_ALIGNMENT_BYTES);
return (void*)endPtr;
}
/**
@@ -288,7 +293,7 @@ ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes)
{
void* const alloc = (BYTE*)ws->allocStart - bytes;
void* const bottom = ws->tableEnd;
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
DEBUGLOG(5, "cwksp: reserving [0x%p]:%zd bytes; %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
ZSTD_cwksp_assert_internal_consistency(ws);
assert(alloc >= bottom);
@@ -405,7 +410,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t byt
{
size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES);
void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
if(ptr && ptr < ws->initOnceStart) {
/* We assume the memory following the current allocation is either:
* 1. Not usable as initOnce memory (end of workspace)
@@ -425,11 +430,12 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t byt
/**
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
*/
MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
MEM_STATIC void* ZSTD_cwksp_reserve_aligned64(ZSTD_cwksp* ws, size_t bytes)
{
void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
ZSTD_cwksp_alloc_aligned);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
void* const ptr = ZSTD_cwksp_reserve_internal(ws,
ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
ZSTD_cwksp_alloc_aligned);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
return ptr;
}
@@ -475,7 +481,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
#endif
assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
return alloc;
}
@@ -521,6 +527,20 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes)
return alloc;
}
/**
* with alignment control
* Note : should happen only once, at workspace first initialization
*/
MEM_STATIC void* ZSTD_cwksp_reserve_object_aligned(ZSTD_cwksp* ws, size_t byteSize, size_t alignment)
{
size_t const mask = alignment - 1;
size_t const surplus = (alignment > sizeof(void*)) ? alignment - sizeof(void*) : 0;
void* const start = ZSTD_cwksp_reserve_object(ws, byteSize + surplus);
if (start == NULL) return NULL;
if (surplus == 0) return start;
assert(ZSTD_isPower2(alignment));
return (void*)(((size_t)start + surplus) & ~mask);
}
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws)
{
@@ -578,7 +598,8 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
* Invalidates table allocations.
* All other allocations remain valid.
*/
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws)
{
DEBUGLOG(4, "cwksp: clearing tables!");
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
@@ -742,10 +763,6 @@ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
}
}
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_CWKSP_H */
#endif /* USE_EXTERNAL_ZSTD */

25
vendor/github.com/DataDog/zstd/zstd_decompress.c generated vendored
View File

@@ -445,7 +445,7 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
** or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
{
const BYTE* ip = (const BYTE*)src;
size_t const minInputSize = ZSTD_startingInputLength(format);
@@ -485,8 +485,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
zfhPtr->frameType = ZSTD_skippableFrame;
zfhPtr->dictID = MEM_readLE32(src) - ZSTD_MAGIC_SKIPPABLE_START;
zfhPtr->headerSize = ZSTD_SKIPPABLEHEADERSIZE;
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
return 0;
}
RETURN_ERROR(prefix_unknown, "");
@@ -555,7 +557,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize)
{
return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
}
@@ -573,7 +575,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
}
#endif
{ ZSTD_frameHeader zfh;
{ ZSTD_FrameHeader zfh;
if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
return ZSTD_CONTENTSIZE_ERROR;
if (zfh.frameType == ZSTD_skippableFrame) {
@@ -751,7 +753,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
const BYTE* const ipstart = ip;
size_t remainingSize = srcSize;
size_t nbBlocks = 0;
ZSTD_frameHeader zfh;
ZSTD_FrameHeader zfh;
/* Extract Frame Header */
{ size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format);
@@ -812,7 +814,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
/** ZSTD_decompressBound() :
* compatible with legacy mode
* `src` must point to the start of a ZSTD frame or a skippeable frame
* `src` must point to the start of a ZSTD frame or a skippable frame
* `srcSize` must be at least as large as the frame contained
* @return : the maximum decompressed size of the compressed source
*/
@@ -844,7 +846,7 @@ size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1);
size_t const compressedSize = frameSizeInfo.compressedSize;
unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
ZSTD_frameHeader zfh;
ZSTD_FrameHeader zfh;
FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
@@ -918,7 +920,7 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
return regenSize;
}
static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, int streaming)
{
#if ZSTD_TRACE
if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
@@ -1058,7 +1060,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
}
ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
/* Allow caller to get size read */
DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr);
DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr));
*srcPtr = ip;
*srcSizePtr = remainingSrcSize;
return (size_t)(op-ostart);
@@ -1642,7 +1644,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
* ZSTD_getFrameHeader(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
{
ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
ZSTD_FrameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
if (ZSTD_isError(hError)) return 0;
return zfp.dictID;
@@ -2000,7 +2002,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
{
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
ZSTD_frameHeader zfh;
ZSTD_FrameHeader zfh;
size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
if (ZSTD_isError(err)) return err;
RETURN_ERROR_IF(err>0, srcSize_wrong, "");
@@ -2095,6 +2097,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
U32 someMoreWork = 1;
DEBUGLOG(5, "ZSTD_decompressStream");
assert(zds != NULL);
RETURN_ERROR_IF(
input->pos > input->size,
srcSize_wrong,

18
vendor/github.com/DataDog/zstd/zstd_decompress_block.c generated vendored
View File

@@ -140,7 +140,7 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
{ const BYTE* const istart = (const BYTE*) src;
symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3);
size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
switch(litEncType)
@@ -359,7 +359,7 @@ size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
* - start from default distributions, present in /lib/common/zstd_internal.h
* - generate tables normally, using ZSTD_buildFSETable()
* - printout the content of tables
* - pretify output, report below, test with fuzzer to ensure it's correct */
* - prettify output, report below, test with fuzzer to ensure it's correct */
/* Default FSE distribution table for Literal Lengths */
static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
@@ -646,7 +646,7 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
* @return : nb bytes read from src,
* or an error code if it fails */
static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
symbolEncodingType_e type, unsigned max, U32 maxLog,
SymbolEncodingType_e type, unsigned max, U32 maxLog,
const void* src, size_t srcSize,
const U32* baseValue, const U8* nbAdditionalBits,
const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
@@ -729,9 +729,9 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
/* FSE table descriptors */
RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */
{ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
{ SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6);
SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3);
SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3);
ip++;
/* Build DTables */
@@ -1936,12 +1936,6 @@ ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
#endif /* DYNAMIC_BMI2 */
typedef size_t (*ZSTD_decompressSequences_t)(
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset);
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
static size_t
ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,

10
vendor/github.com/DataDog/zstd/zstd_decompress_internal.h generated vendored
View File

@@ -137,7 +137,7 @@ struct ZSTD_DCtx_s
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
ZSTD_frameHeader fParams;
ZSTD_FrameHeader fParams;
U64 processedCSize;
U64 decodedSize;
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
@@ -155,7 +155,7 @@ struct ZSTD_DCtx_s
size_t rleSize;
size_t staticSize;
int isFrameDecompression;
#if DYNAMIC_BMI2 != 0
#if DYNAMIC_BMI2
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
#endif
@@ -212,11 +212,11 @@ struct ZSTD_DCtx_s
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
#if DYNAMIC_BMI2 != 0
return dctx->bmi2;
#if DYNAMIC_BMI2
return dctx->bmi2;
#else
(void)dctx;
return 0;
return 0;
#endif
}

12
vendor/github.com/DataDog/zstd/zstd_deps.h generated vendored
View File

@@ -25,6 +25,18 @@
#ifndef ZSTD_DEPS_COMMON
#define ZSTD_DEPS_COMMON
/* Even though we use qsort_r only for the dictionary builder, the macro
* _GNU_SOURCE has to be declared *before* the inclusion of any standard
* header and the script 'combine.sh' combines the whole zstd source code
* in a single file.
*/
#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
defined(__CYGWIN__) || defined(__MSYS__)
#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
#define _GNU_SOURCE
#endif
#endif
#include <limits.h>
#include <stddef.h>
#include <string.h>

108
vendor/github.com/DataDog/zstd/zstd_double_fast.c generated vendored
View File

@@ -16,7 +16,7 @@
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms,
void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -54,7 +54,7 @@ void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms,
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms,
void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -88,7 +88,7 @@ void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms,
} }
}
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp)
@@ -104,7 +104,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_noDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls /* template */)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
@@ -143,9 +143,14 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
const BYTE* matchl0; /* the long match for ip */
const BYTE* matchs0; /* the short match for ip */
const BYTE* matchl1; /* the long match for ip1 */
const BYTE* matchs0_safe; /* matchs0 or safe address */
const BYTE* ip = istart; /* the current position */
const BYTE* ip1; /* the next position */
/* Array of ~random data, should have low probability of matching data
* we load from here instead of from tables, if matchl0/matchl1 are
* invalid indices. Used to avoid unpredictable branches. */
const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4};
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
@@ -192,24 +197,29 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
if (idxl0 > prefixLowestIndex) {
/* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch.
* However expression below complies into conditional move. Since
* match is unlikely and we only *branch* on idxl0 > prefixLowestIndex
* if there is a match, all branches become predictable. */
{ const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]);
/* check prefix long match */
if (MEM_read64(matchl0) == MEM_read64(ip)) {
if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) {
mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
offset = (U32)(ip-matchl0);
while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
goto _match_found;
}
}
} }
idxl1 = hashLong[hl1];
matchl1 = base + idxl1;
if (idxs0 > prefixLowestIndex) {
/* check prefix short match */
if (MEM_read32(matchs0) == MEM_read32(ip)) {
goto _search_next_long;
}
/* Same optimization as matchl0 above */
matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]);
/* check prefix short match */
if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) {
goto _search_next_long;
}
if (ip1 >= nextStep) {
@@ -243,21 +253,23 @@ _cleanup:
_search_next_long:
/* check prefix long +1 match */
if (idxl1 > prefixLowestIndex) {
if (MEM_read64(matchl1) == MEM_read64(ip1)) {
/* short match found: let's check for a longer one */
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
offset = (U32)(ip - matchs0);
/* check long match at +1 position */
if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) {
size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8;
if (l1len > mLength) {
/* use the long match instead */
ip = ip1;
mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8;
mLength = l1len;
offset = (U32)(ip-matchl1);
while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */
goto _match_found;
matchs0 = matchl1;
}
}
/* if no long +1 match, explore the short match we found */
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
offset = (U32)(ip - matchs0);
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */
/* fall-through */
@@ -315,7 +327,7 @@ _match_stored:
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls /* template */)
{
@@ -336,7 +348,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=rep[0], offset_2=rep[1];
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
const U32* const dictHashLong = dms->hashTable;
const U32* const dictHashSmall = dms->chainTable;
@@ -393,7 +405,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
/* check repcode */
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -402,14 +414,12 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
goto _match_stored;
}
if (matchIndexL > prefixLowestIndex) {
if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
/* check prefix long match */
if (MEM_read64(matchLong) == MEM_read64(ip)) {
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
offset = (U32)(ip-matchLong);
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
goto _match_found;
}
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
offset = (U32)(ip-matchLong);
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
goto _match_found;
} else if (dictTagsMatchL) {
/* check dictMatchState long match */
U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
@@ -424,7 +434,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
} }
if (matchIndexS > prefixLowestIndex) {
/* check prefix short match */
/* short match candidate */
if (MEM_read32(match) == MEM_read32(ip)) {
goto _search_next_long;
}
@@ -454,14 +464,12 @@ _search_next_long:
hashLong[hl3] = curr + 1;
/* check prefix long +1 match */
if (matchIndexL3 > prefixLowestIndex) {
if (MEM_read64(matchL3) == MEM_read64(ip+1)) {
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
ip++;
offset = (U32)(ip-matchL3);
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
goto _match_found;
}
if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) {
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
ip++;
offset = (U32)(ip-matchL3);
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
goto _match_found;
} else if (dictTagsMatchL3) {
/* check dict long +1 match */
U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
@@ -514,7 +522,7 @@ _match_stored:
const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
dictBase + repIndex2 - dictIndexDelta :
base + repIndex2;
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2))
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
@@ -541,7 +549,7 @@ _match_stored:
#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
void const* src, size_t srcSize) \
{ \
return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
@@ -559,7 +567,7 @@ ZSTD_GEN_DFAST_FN(dictMatchState, 7)
size_t ZSTD_compressBlock_doubleFast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.minMatch;
@@ -579,7 +587,7 @@ size_t ZSTD_compressBlock_doubleFast(
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.minMatch;
@@ -601,7 +609,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_extDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls /* template */)
{
@@ -652,7 +660,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
size_t mLength;
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
& (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
@@ -720,7 +728,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
U32 const current2 = (U32)(ip-base);
U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
& (offset_2 <= current2 - dictStartIndex))
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
@@ -750,7 +758,7 @@ ZSTD_GEN_DFAST_FN(extDict, 6)
ZSTD_GEN_DFAST_FN(extDict, 7)
size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;

16
vendor/github.com/DataDog/zstd/zstd_double_fast.h generated vendored
View File

@@ -12,27 +12,23 @@
#ifndef ZSTD_DOUBLE_FAST_H
#define ZSTD_DOUBLE_FAST_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "mem.h" /* U32 */
#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp);
size_t ZSTD_compressBlock_doubleFast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast
@@ -44,10 +40,6 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_DOUBLE_FAST_H */
#endif /* USE_EXTERNAL_ZSTD */

9
vendor/github.com/DataDog/zstd/zstd_errors.h generated vendored
View File

@@ -16,10 +16,6 @@
extern "C" {
#endif
/*===== dependency =====*/
#include <stddef.h> /* size_t */
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
#ifndef ZSTDERRORLIB_VISIBLE
/* Backwards compatibility with old macro name */
@@ -81,6 +77,7 @@ typedef enum {
ZSTD_error_tableLog_tooLarge = 44,
ZSTD_error_maxSymbolValue_tooLarge = 46,
ZSTD_error_maxSymbolValue_tooSmall = 48,
ZSTD_error_cannotProduce_uncompressedBlock = 49,
ZSTD_error_stabilityCondition_notRespected = 50,
ZSTD_error_stage_wrong = 60,
ZSTD_error_init_missing = 62,
@@ -101,10 +98,6 @@ typedef enum {
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
} ZSTD_ErrorCode;
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare with enum list published above */
ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */

165
vendor/github.com/DataDog/zstd/zstd_fast.c generated vendored
View File

@@ -14,7 +14,7 @@
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms,
void ZSTD_fillHashTableForCDict(ZSTD_MatchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm)
{
@@ -46,12 +46,12 @@ void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms,
size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls);
if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */
ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p);
} } } }
} } } }
}
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms,
void ZSTD_fillHashTableForCCtx(ZSTD_MatchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm)
{
@@ -85,7 +85,7 @@ void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms,
} } } }
}
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
void ZSTD_fillHashTable(ZSTD_MatchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp)
@@ -98,6 +98,50 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
}
typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit);
static int
ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit)
{
/* Array of ~random data, should have low probability of matching data.
* Load from here if the index is invalid.
* Used to avoid unpredictable branches. */
static const BYTE dummy[] = {0x12,0x34,0x56,0x78};
/* currentIdx >= lowLimit is a (somewhat) unpredictable branch.
* However expression below compiles into conditional move.
*/
const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy);
/* Note: this used to be written as : return test1 && test2;
* Unfortunately, once inlined, these tests become branches,
* in which case it becomes critical that they are executed in the right order (test1 then test2).
* So we have to write these tests in a specific manner to ensure their ordering.
*/
if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0;
/* force ordering of these tests, which matters once the function is inlined, as they become branches */
#if defined(__GNUC__)
__asm__("");
#endif
return matchIdx >= idxLowLimit;
}
static int
ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit)
{
/* using a branch instead of a cmov,
* because it's faster in scenarios where matchIdx >= idxLowLimit is generally true,
* aka almost all candidates are within range */
U32 mval;
if (matchIdx >= idxLowLimit) {
mval = MEM_read32(matchAddress);
} else {
mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */
}
return (MEM_read32(currentPtr) == mval);
}
/**
* If you squint hard enough (and ignore repcodes), the search operation at any
* given position is broken into 4 stages:
@@ -147,15 +191,14 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_noDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls, U32 const hasStep)
U32 const mls, int useCmov)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hlog = cParams->hashLog;
/* support stepSize of 0 */
size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2;
size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; /* min 2 */
const BYTE* const base = ms->window.base;
const BYTE* const istart = (const BYTE*)src;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
@@ -177,8 +220,7 @@ size_t ZSTD_compressBlock_fast_noDict_generic(
size_t hash0; /* hash for ip0 */
size_t hash1; /* hash for ip1 */
U32 idx; /* match idx for ip0 */
U32 mval; /* src value at match idx */
U32 matchIdx; /* match idx for ip0 */
U32 offcode;
const BYTE* match0;
@@ -191,6 +233,7 @@ size_t ZSTD_compressBlock_fast_noDict_generic(
size_t step;
const BYTE* nextStep;
const size_t kStepIncr = (1 << (kSearchStrength - 1));
const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch;
DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
ip0 += (ip0 == prefixStart);
@@ -219,7 +262,7 @@ _start: /* Requires: ip0 */
hash0 = ZSTD_hashPtr(ip0, hlog, mls);
hash1 = ZSTD_hashPtr(ip1, hlog, mls);
idx = hashTable[hash0];
matchIdx = hashTable[hash0];
do {
/* load repcode match for ip[2]*/
@@ -239,35 +282,25 @@ _start: /* Requires: ip0 */
offcode = REPCODE1_TO_OFFBASE;
mLength += 4;
/* First write next hash table entry; we've already calculated it.
* This write is known to be safe because the ip1 is before the
/* Write next hash table entry: it's already calculated.
* This write is known to be safe because ip1 is before the
* repcode (ip2). */
hashTable[hash1] = (U32)(ip1 - base);
goto _match;
}
/* load match for ip[0] */
if (idx >= prefixStartIndex) {
mval = MEM_read32(base + idx);
} else {
mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
}
/* check match at ip[0] */
if (MEM_read32(ip0) == mval) {
/* found a match! */
/* First write next hash table entry; we've already calculated it.
* This write is known to be safe because the ip1 == ip0 + 1, so
* we know we will resume searching after ip1 */
if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) {
/* Write next hash table entry (it's already calculated).
* This write is known to be safe because the ip1 == ip0 + 1,
* so searching will resume after ip1 */
hashTable[hash1] = (U32)(ip1 - base);
goto _offset;
}
/* lookup ip[1] */
idx = hashTable[hash1];
matchIdx = hashTable[hash1];
/* hash ip[2] */
hash0 = hash1;
@@ -282,36 +315,19 @@ _start: /* Requires: ip0 */
current0 = (U32)(ip0 - base);
hashTable[hash0] = current0;
/* load match for ip[0] */
if (idx >= prefixStartIndex) {
mval = MEM_read32(base + idx);
} else {
mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
}
/* check match at ip[0] */
if (MEM_read32(ip0) == mval) {
/* found a match! */
/* first write next hash table entry; we've already calculated it */
if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) {
/* Write next hash table entry, since it's already calculated */
if (step <= 4) {
/* We need to avoid writing an index into the hash table >= the
* position at which we will pick up our searching after we've
* taken this match.
*
* The minimum possible match has length 4, so the earliest ip0
* can be after we take this match will be the current ip0 + 4.
* ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely
* write this position.
*/
/* Avoid writing an index if it's >= position where search will resume.
* The minimum possible match has length 4, so search can resume at ip0 + 4.
*/
hashTable[hash1] = (U32)(ip1 - base);
}
goto _offset;
}
/* lookup ip[1] */
idx = hashTable[hash1];
matchIdx = hashTable[hash1];
/* hash ip[2] */
hash0 = hash1;
@@ -333,7 +349,7 @@ _start: /* Requires: ip0 */
} while (ip3 < ilimit);
_cleanup:
/* Note that there are probably still a couple positions we could search.
/* Note that there are probably still a couple positions one could search.
* However, it seems to be a meaningful performance hit to try to search
* them. So let's not. */
@@ -362,7 +378,7 @@ _cleanup:
_offset: /* Requires: ip0, idx */
/* Compute the offset code. */
match0 = base + idx;
match0 = base + matchIdx;
rep_offset2 = rep_offset1;
rep_offset1 = (U32)(ip0-match0);
offcode = OFFSET_TO_OFFBASE(rep_offset1);
@@ -407,12 +423,12 @@ _match: /* Requires: ip0, match0, offcode */
goto _start;
}
#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \
static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
#define ZSTD_GEN_FAST_FN(dictMode, mml, cmov) \
static size_t ZSTD_compressBlock_fast_##dictMode##_##mml##_##cmov( \
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
void const* src, size_t srcSize) \
{ \
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mml, cmov); \
}
ZSTD_GEN_FAST_FN(noDict, 4, 1)
@@ -426,13 +442,15 @@ ZSTD_GEN_FAST_FN(noDict, 6, 0)
ZSTD_GEN_FAST_FN(noDict, 7, 0)
size_t ZSTD_compressBlock_fast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
U32 const mml = ms->cParams.minMatch;
/* use cmov when "candidate in range" branch is likely unpredictable */
int const useCmov = ms->cParams.windowLog < 19;
assert(ms->dictMatchState == NULL);
if (ms->cParams.targetLength > 1) {
switch(mls)
if (useCmov) {
switch(mml)
{
default: /* includes case 3 */
case 4 :
@@ -445,7 +463,8 @@ size_t ZSTD_compressBlock_fast(
return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize);
}
} else {
switch(mls)
/* use a branch instead */
switch(mml)
{
default: /* includes case 3 */
case 4 :
@@ -457,14 +476,13 @@ size_t ZSTD_compressBlock_fast(
case 7 :
return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize);
}
}
}
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_dictMatchState_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -483,7 +501,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=rep[0], offset_2=rep[1];
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
const U32* const dictHashTable = dms->hashTable;
const U32 dictStartIndex = dms->window.dictLimit;
@@ -547,8 +565,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls);
hashTable[hash0] = curr; /* update hash table */
if (((U32) ((prefixStartIndex - 1) - repIndex) >=
3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
if ((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) {
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4;
@@ -581,8 +598,8 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
}
}
if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) {
/* found a regular match */
if (ZSTD_match4Found_cmov(ip0, match, matchIndex, prefixStartIndex)) {
/* found a regular match of size >= 4 */
U32 const offset = (U32) (ip0 - match);
mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4;
while (((ip0 > anchor) & (match > prefixStart))
@@ -632,7 +649,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
dictBase - dictIndexDelta + repIndex2 :
base + repIndex2;
if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
if ( (ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
&& (MEM_read32(repMatch2) == MEM_read32(ip0))) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
@@ -668,7 +685,7 @@ ZSTD_GEN_FAST_FN(dictMatchState, 6, 0)
ZSTD_GEN_FAST_FN(dictMatchState, 7, 0)
size_t ZSTD_compressBlock_fast_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
@@ -691,7 +708,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_extDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -926,7 +943,7 @@ _match: /* Requires: ip0, match0, offcode, matchEnd */
while (ip0 <= ilimit) {
U32 const repIndex2 = (U32)(ip0-base) - offset_2;
const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0))
&& (MEM_read32(repMatch2) == MEM_read32(ip0)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
@@ -949,7 +966,7 @@ ZSTD_GEN_FAST_FN(extDict, 6, 0)
ZSTD_GEN_FAST_FN(extDict, 7, 0)
size_t ZSTD_compressBlock_fast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;

16
vendor/github.com/DataDog/zstd/zstd_fast.h generated vendored
View File

@@ -12,30 +12,22 @@
#ifndef ZSTD_FAST_H
#define ZSTD_FAST_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "mem.h" /* U32 */
#include "zstd_compress_internal.h"
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
void ZSTD_fillHashTable(ZSTD_MatchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp);
size_t ZSTD_compressBlock_fast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_fast_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_fast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_FAST_H */
#endif /* USE_EXTERNAL_ZSTD */

70
vendor/github.com/DataDog/zstd/zstd_internal.h generated vendored
View File

@@ -40,10 +40,6 @@
# define ZSTD_TRACE 0
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/* ---- static assert (debug) --- */
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
#define ZSTD_isError ERR_isError /* for inlining */
@@ -96,7 +92,7 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */
#define MIN_LITERALS_FOR_4_STREAMS 6
typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e;
#define LONGNBSEQ 0x7F00
@@ -279,62 +275,6 @@ typedef enum {
/*-*******************************************
* Private declarations
*********************************************/
typedef struct seqDef_s {
U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
U16 litLength;
U16 mlBase; /* mlBase == matchLength - MINMATCH */
} seqDef;
/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
typedef enum {
ZSTD_llt_none = 0, /* no longLengthType */
ZSTD_llt_literalLength = 1, /* represents a long literal */
ZSTD_llt_matchLength = 2 /* represents a long match */
} ZSTD_longLengthType_e;
typedef struct {
seqDef* sequencesStart;
seqDef* sequences; /* ptr to end of sequences */
BYTE* litStart;
BYTE* lit; /* ptr to end of literals */
BYTE* llCode;
BYTE* mlCode;
BYTE* ofCode;
size_t maxNbSeq;
size_t maxNbLit;
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
* the existing value of the litLength or matchLength by 0x10000.
*/
ZSTD_longLengthType_e longLengthType;
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
} seqStore_t;
typedef struct {
U32 litLength;
U32 matchLength;
} ZSTD_sequenceLength;
/**
* Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
*/
MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
{
ZSTD_sequenceLength seqLen;
seqLen.litLength = seq->litLength;
seqLen.matchLength = seq->mlBase + MINMATCH;
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
seqLen.litLength += 0x10000;
}
if (seqStore->longLengthType == ZSTD_llt_matchLength) {
seqLen.matchLength += 0x10000;
}
}
return seqLen;
}
/**
* Contains the compressed frame size and an upper-bound for the decompressed frame size.
@@ -348,10 +288,6 @@ typedef struct {
unsigned long long decompressedBound;
} ZSTD_frameSizeInfo; /* decompress & legacy */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
/* ZSTD_invalidateRepCodes() :
* ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant;
@@ -386,10 +322,6 @@ MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
}
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_CCOMMON_H_MODULE */
#endif /* USE_EXTERNAL_ZSTD */

128
vendor/github.com/DataDog/zstd/zstd_lazy.c generated vendored
View File

@@ -27,7 +27,7 @@
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_updateDUBT(ZSTD_matchState_t* ms,
void ZSTD_updateDUBT(ZSTD_MatchState_t* ms,
const BYTE* ip, const BYTE* iend,
U32 mls)
{
@@ -72,7 +72,7 @@ void ZSTD_updateDUBT(ZSTD_matchState_t* ms,
* doesn't fail */
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_insertDUBT1(const ZSTD_matchState_t* ms,
void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms,
U32 curr, const BYTE* inputEnd,
U32 nbCompares, U32 btLow,
const ZSTD_dictMode_e dictMode)
@@ -163,7 +163,7 @@ void ZSTD_insertDUBT1(const ZSTD_matchState_t* ms,
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_DUBT_findBetterDictMatch (
const ZSTD_matchState_t* ms,
const ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
size_t bestLength,
@@ -171,7 +171,7 @@ size_t ZSTD_DUBT_findBetterDictMatch (
U32 const mls,
const ZSTD_dictMode_e dictMode)
{
const ZSTD_matchState_t * const dms = ms->dictMatchState;
const ZSTD_MatchState_t * const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
const U32 * const dictHashTable = dms->hashTable;
U32 const hashLog = dmsCParams->hashLog;
@@ -241,7 +241,7 @@ size_t ZSTD_DUBT_findBetterDictMatch (
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
size_t* offBasePtr,
U32 const mls,
@@ -393,7 +393,7 @@ size_t ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offBasePtr,
const U32 mls /* template */,
@@ -409,7 +409,7 @@ size_t ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
* Dedicated dict search
***********************************/
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip)
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip)
{
const BYTE* const base = ms->window.base;
U32 const target = (U32)(ip - base);
@@ -528,7 +528,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B
*/
FORCE_INLINE_TEMPLATE
size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts,
const ZSTD_matchState_t* const dms,
const ZSTD_MatchState_t* const dms,
const BYTE* const ip, const BYTE* const iLimit,
const BYTE* const prefixStart, const U32 curr,
const U32 dictLimit, const size_t ddsIdx) {
@@ -631,7 +631,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_insertAndFindFirstIndex_internal(
ZSTD_matchState_t* ms,
ZSTD_MatchState_t* ms,
const ZSTD_compressionParameters* const cParams,
const BYTE* ip, U32 const mls, U32 const lazySkipping)
{
@@ -657,7 +657,7 @@ U32 ZSTD_insertAndFindFirstIndex_internal(
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
}
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0);
}
@@ -666,7 +666,7 @@ U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_HcFindBestMatch(
ZSTD_matchState_t* ms,
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_dictMode_e dictMode)
@@ -690,7 +690,7 @@ size_t ZSTD_HcFindBestMatch(
U32 nbAttempts = 1U << cParams->searchLog;
size_t ml=4-1;
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch
? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch
@@ -835,7 +835,7 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* t
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base,
void ZSTD_row_fillHashCache(ZSTD_MatchState_t* ms, const BYTE* base,
U32 const rowLog, U32 const mls,
U32 idx, const BYTE* const iLimit)
{
@@ -883,7 +883,7 @@ U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms,
void ZSTD_row_update_internalImpl(ZSTD_MatchState_t* ms,
U32 updateStartIdx, U32 const updateEndIdx,
U32 const mls, U32 const rowLog,
U32 const rowMask, U32 const useCache)
@@ -914,7 +914,7 @@ void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms,
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip,
void ZSTD_row_update_internal(ZSTD_MatchState_t* ms, const BYTE* ip,
U32 const mls, U32 const rowLog,
U32 const rowMask, U32 const useCache)
{
@@ -947,7 +947,7 @@ void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip,
* External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary
* processing.
*/
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip) {
const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
const U32 rowMask = (1u << rowLog) - 1;
const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
@@ -1124,9 +1124,9 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
/* The high-level approach of the SIMD row based match finder is as follows:
* - Figure out where to insert the new entry:
* - Generate a hash for current input posistion and split it into a one byte of tag and `rowHashLog` bits of index.
* - The hash is salted by a value that changes on every contex reset, so when the same table is used
* we will avoid collisions that would otherwise slow us down by intorducing phantom matches.
* - Generate a hash for current input position and split it into a one byte of tag and `rowHashLog` bits of index.
* - The hash is salted by a value that changes on every context reset, so when the same table is used
* we will avoid collisions that would otherwise slow us down by introducing phantom matches.
* - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines
* which row to insert into.
* - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can
@@ -1140,7 +1140,7 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_RowFindBestMatch(
ZSTD_matchState_t* ms,
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_dictMode_e dictMode,
@@ -1172,7 +1172,7 @@ size_t ZSTD_RowFindBestMatch(
U32 hash;
/* DMS/DDS variables that may be referenced laster */
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
/* Initialize the following variables to satisfy static analyzer */
size_t ddsIdx = 0;
@@ -1341,7 +1341,7 @@ size_t ZSTD_RowFindBestMatch(
* ZSTD_searchMax() dispatches to the correct implementation function.
*
* TODO: The start of the search function involves loading and calculating a
* bunch of constants from the ZSTD_matchState_t. These computations could be
* bunch of constants from the ZSTD_MatchState_t. These computations could be
* done in an initialization function, and saved somewhere in the match state.
* Then we could pass a pointer to the saved state instead of the match state,
* and avoid duplicate computations.
@@ -1365,7 +1365,7 @@ size_t ZSTD_RowFindBestMatch(
#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \
ZSTD_matchState_t* ms, \
ZSTD_MatchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offBasePtr) \
{ \
@@ -1375,7 +1375,7 @@ size_t ZSTD_RowFindBestMatch(
#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \
ZSTD_matchState_t* ms, \
ZSTD_MatchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offsetPtr) \
{ \
@@ -1385,7 +1385,7 @@ size_t ZSTD_RowFindBestMatch(
#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \
ZSTD_matchState_t* ms, \
ZSTD_MatchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offsetPtr) \
{ \
@@ -1486,7 +1486,7 @@ typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searc
* If a match is found its offset is stored in @p offsetPtr.
*/
FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
ZSTD_matchState_t* ms,
ZSTD_MatchState_t* ms,
const BYTE* ip,
const BYTE* iend,
size_t* offsetPtr,
@@ -1515,7 +1515,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_lazy_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const searchMethod_e searchMethod, const U32 depth,
@@ -1538,7 +1538,7 @@ size_t ZSTD_compressBlock_lazy_generic(
const int isDMS = dictMode == ZSTD_dictMatchState;
const int isDDS = dictMode == ZSTD_dedicatedDictSearch;
const int isDxS = isDMS || isDDS;
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0;
const BYTE* const dictBase = isDxS ? dms->window.base : NULL;
const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL;
@@ -1591,7 +1591,7 @@ size_t ZSTD_compressBlock_lazy_generic(
&& repIndex < prefixLowestIndex) ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1643,7 +1643,7 @@ size_t ZSTD_compressBlock_lazy_generic(
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1679,7 +1679,7 @@ size_t ZSTD_compressBlock_lazy_generic(
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1741,7 +1741,7 @@ _storeSequence:
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase - dictIndexDelta + repIndex :
base + repIndex;
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */)
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
@@ -1783,42 +1783,42 @@ _storeSequence:
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict);
}
size_t ZSTD_compressBlock_greedy_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_greedy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict);
}
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch);
@@ -1827,42 +1827,42 @@ size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_lazy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch);
@@ -1871,42 +1871,42 @@ size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy2_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_lazy2_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch);
@@ -1915,14 +1915,14 @@ size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState);
@@ -1936,7 +1936,7 @@ size_t ZSTD_compressBlock_btlazy2_dictMatchState(
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_lazy_extDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const searchMethod_e searchMethod, const U32 depth)
@@ -1987,7 +1987,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr+1 - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */
if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
@@ -2028,7 +2028,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
@@ -2060,7 +2060,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
@@ -2114,7 +2114,7 @@ _storeSequence:
const U32 repIndex = repCurrent - offset_2;
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
@@ -2140,14 +2140,14 @@ _storeSequence:
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0);
}
size_t ZSTD_compressBlock_greedy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0);
@@ -2156,7 +2156,7 @@ size_t ZSTD_compressBlock_greedy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
@@ -2164,7 +2164,7 @@ size_t ZSTD_compressBlock_lazy_extDict(
}
size_t ZSTD_compressBlock_lazy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
@@ -2174,7 +2174,7 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
@@ -2182,7 +2182,7 @@ size_t ZSTD_compressBlock_lazy2_extDict(
}
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
@@ -2191,7 +2191,7 @@ size_t ZSTD_compressBlock_lazy2_extDict_row(
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{

69
vendor/github.com/DataDog/zstd/zstd_lazy.h generated vendored
View File

@@ -12,10 +12,6 @@
#ifndef ZSTD_LAZY_H
#define ZSTD_LAZY_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_compress_internal.h"
/**
@@ -32,38 +28,38 @@ extern "C" {
|| !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR)
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip);
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip);
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip);
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
#endif
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_GREEDY ZSTD_compressBlock_greedy
@@ -87,28 +83,28 @@ size_t ZSTD_compressBlock_greedy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_LAZY ZSTD_compressBlock_lazy
@@ -132,28 +128,28 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_LAZY2 ZSTD_compressBlock_lazy2
@@ -177,13 +173,13 @@ size_t ZSTD_compressBlock_lazy2_extDict_row(
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_BTLAZY2 ZSTD_compressBlock_btlazy2
@@ -195,11 +191,6 @@ size_t ZSTD_compressBlock_btlazy2_extDict(
#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT NULL
#endif
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_LAZY_H */
#endif /* USE_EXTERNAL_ZSTD */

83
vendor/github.com/DataDog/zstd/zstd_ldm.c generated vendored
View File

@@ -17,7 +17,7 @@
#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
#include "zstd_ldm_geartab.h"
#define LDM_BUCKET_SIZE_LOG 3
#define LDM_BUCKET_SIZE_LOG 4
#define LDM_MIN_MATCH_LENGTH 64
#define LDM_HASH_RLOG 7
@@ -134,21 +134,35 @@ done:
}
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
ZSTD_compressionParameters const* cParams)
const ZSTD_compressionParameters* cParams)
{
params->windowLog = cParams->windowLog;
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
if (params->hashLog == 0) {
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
}
if (params->hashRateLog == 0) {
params->hashRateLog = params->windowLog < params->hashLog
? 0
: params->windowLog - params->hashLog;
if (params->hashLog > 0) {
/* if params->hashLog is set, derive hashRateLog from it */
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
if (params->windowLog > params->hashLog) {
params->hashRateLog = params->windowLog - params->hashLog;
}
} else {
assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9);
/* mapping from [fast, rate7] to [btultra2, rate4] */
params->hashRateLog = 7 - (cParams->strategy/3);
}
}
if (params->hashLog == 0) {
params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX);
}
if (params->minMatchLength == 0) {
params->minMatchLength = LDM_MIN_MATCH_LENGTH;
if (cParams->strategy >= ZSTD_btultra)
params->minMatchLength /= 2;
}
if (params->bucketSizeLog==0) {
assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9);
params->bucketSizeLog = BOUNDED(LDM_BUCKET_SIZE_LOG, (U32)cParams->strategy, ZSTD_LDM_BUCKETSIZELOG_MAX);
}
params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog);
}
@@ -171,22 +185,22 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
/** ZSTD_ldm_getBucket() :
* Returns a pointer to the start of the bucket associated with hash. */
static ldmEntry_t* ZSTD_ldm_getBucket(
ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams)
const ldmState_t* ldmState, size_t hash, U32 const bucketSizeLog)
{
return ldmState->hashTable + (hash << ldmParams.bucketSizeLog);
return ldmState->hashTable + (hash << bucketSizeLog);
}
/** ZSTD_ldm_insertEntry() :
* Insert the entry with corresponding hash into the hash table */
static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
size_t const hash, const ldmEntry_t entry,
ldmParams_t const ldmParams)
U32 const bucketSizeLog)
{
BYTE* const pOffset = ldmState->bucketOffsets + hash;
unsigned const offset = *pOffset;
*(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
*pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
*(ZSTD_ldm_getBucket(ldmState, hash, bucketSizeLog) + offset) = entry;
*pOffset = (BYTE)((offset + 1) & ((1u << bucketSizeLog) - 1));
}
@@ -235,7 +249,7 @@ static size_t ZSTD_ldm_countBackwardsMatch_2segments(
*
* The tables for the other strategies are filled within their
* block compressors. */
static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
static size_t ZSTD_ldm_fillFastTables(ZSTD_MatchState_t* ms,
void const* end)
{
const BYTE* const iend = (const BYTE*)end;
@@ -274,7 +288,8 @@ void ZSTD_ldm_fillHashTable(
const BYTE* iend, ldmParams_t const* params)
{
U32 const minMatchLength = params->minMatchLength;
U32 const hBits = params->hashLog - params->bucketSizeLog;
U32 const bucketSizeLog = params->bucketSizeLog;
U32 const hBits = params->hashLog - bucketSizeLog;
BYTE const* const base = ldmState->window.base;
BYTE const* const istart = ip;
ldmRollingHashState_t hashState;
@@ -289,7 +304,7 @@ void ZSTD_ldm_fillHashTable(
unsigned n;
numSplits = 0;
hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
hashed = ZSTD_ldm_gear_feed(&hashState, ip, (size_t)(iend - ip), splits, &numSplits);
for (n = 0; n < numSplits; n++) {
if (ip + splits[n] >= istart + minMatchLength) {
@@ -300,7 +315,7 @@ void ZSTD_ldm_fillHashTable(
entry.offset = (U32)(split - base);
entry.checksum = (U32)(xxhash >> 32);
ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
ZSTD_ldm_insertEntry(ldmState, hash, entry, params->bucketSizeLog);
}
}
@@ -314,7 +329,7 @@ void ZSTD_ldm_fillHashTable(
* Sets cctx->nextToUpdate to a position corresponding closer to anchor
* if it is far way
* (after a long match, only update tables a limited amount). */
static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
static void ZSTD_ldm_limitTableUpdate(ZSTD_MatchState_t* ms, const BYTE* anchor)
{
U32 const curr = (U32)(anchor - ms->window.base);
if (curr > ms->nextToUpdate + 1024) {
@@ -326,7 +341,7 @@ static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_ldm_generateSequences_internal(
ldmState_t* ldmState, rawSeqStore_t* rawSeqStore,
ldmState_t* ldmState, RawSeqStore_t* rawSeqStore,
ldmParams_t const* params, void const* src, size_t srcSize)
{
/* LDM parameters */
@@ -380,7 +395,7 @@ size_t ZSTD_ldm_generateSequences_internal(
candidates[n].split = split;
candidates[n].hash = hash;
candidates[n].checksum = (U32)(xxhash >> 32);
candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, params->bucketSizeLog);
PREFETCH_L1(candidates[n].bucket);
}
@@ -403,7 +418,7 @@ size_t ZSTD_ldm_generateSequences_internal(
* the previous one, we merely register it in the hash table and
* move on */
if (split < anchor) {
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
continue;
}
@@ -450,7 +465,7 @@ size_t ZSTD_ldm_generateSequences_internal(
/* No match found -- insert an entry into the hash table
* and process the next candidate match */
if (bestEntry == NULL) {
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
continue;
}
@@ -471,7 +486,7 @@ size_t ZSTD_ldm_generateSequences_internal(
/* Insert the current entry into the hash table --- it must be
* done after the previous block to avoid clobbering bestEntry */
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
anchor = split + forwardMatchLength;
@@ -510,7 +525,7 @@ static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
}
size_t ZSTD_ldm_generateSequences(
ldmState_t* ldmState, rawSeqStore_t* sequences,
ldmState_t* ldmState, RawSeqStore_t* sequences,
ldmParams_t const* params, void const* src, size_t srcSize)
{
U32 const maxDist = 1U << params->windowLog;
@@ -587,7 +602,7 @@ size_t ZSTD_ldm_generateSequences(
}
void
ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
{
while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
@@ -623,7 +638,7 @@ ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const min
* Returns the current sequence to handle, or if the rest of the block should
* be literals, it returns a sequence with offset == 0.
*/
static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
static rawSeq maybeSplitSequence(RawSeqStore_t* rawSeqStore,
U32 const remaining, U32 const minMatch)
{
rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos];
@@ -647,7 +662,7 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
return sequence;
}
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) {
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
@@ -664,14 +679,14 @@ void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
}
}
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_paramSwitch_e useRowMatchFinder,
size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_ParamSwitch_e useRowMatchFinder,
void const* src, size_t srcSize)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
unsigned const minMatch = cParams->minMatch;
ZSTD_blockCompressor const blockCompressor =
ZSTD_BlockCompressor_f const blockCompressor =
ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
/* Input bounds */
BYTE const* const istart = (BYTE const*)src;

20
vendor/github.com/DataDog/zstd/zstd_ldm.h generated vendored
View File

@@ -12,10 +12,6 @@
#ifndef ZSTD_LDM_H
#define ZSTD_LDM_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_compress_internal.h" /* ldmParams_t, U32 */
#include "zstd.h" /* ZSTD_CCtx, size_t */
@@ -44,7 +40,7 @@ void ZSTD_ldm_fillHashTable(
* sequences.
*/
size_t ZSTD_ldm_generateSequences(
ldmState_t* ldms, rawSeqStore_t* sequences,
ldmState_t* ldms, RawSeqStore_t* sequences,
ldmParams_t const* params, void const* src, size_t srcSize);
/**
@@ -65,9 +61,9 @@ size_t ZSTD_ldm_generateSequences(
* two. We handle that case correctly, and update `rawSeqStore` appropriately.
* NOTE: This function does not return any errors.
*/
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_paramSwitch_e useRowMatchFinder,
size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_ParamSwitch_e useRowMatchFinder,
void const* src, size_t srcSize);
/**
@@ -77,7 +73,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
* Avoids emitting matches less than `minMatch` bytes.
* Must be called for data that is not passed to ZSTD_ldm_blockCompress().
*/
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
void ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize,
U32 const minMatch);
/* ZSTD_ldm_skipRawSeqStoreBytes():
@@ -85,7 +81,7 @@ void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
* Not to be used in conjunction with ZSTD_ldm_skipSequences().
* Must be called for data with is not passed to ZSTD_ldm_blockCompress().
*/
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes);
void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes);
/** ZSTD_ldm_getTableSize() :
* Estimate the space needed for long distance matching tables or 0 if LDM is
@@ -111,10 +107,6 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
ZSTD_compressionParameters const* cParams);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_FAST_H */
#endif /* USE_EXTERNAL_ZSTD */

106
vendor/github.com/DataDog/zstd/zstd_opt.c generated vendored
View File

@@ -409,7 +409,7 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
Assumption : always within prefix (i.e. not within extDict) */
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms,
U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_MatchState_t* ms,
U32* nextToUpdate3,
const BYTE* const ip)
{
@@ -441,7 +441,7 @@ U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms,
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_insertBt1(
const ZSTD_matchState_t* ms,
const ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
U32 const target,
U32 const mls, const int extDict)
@@ -561,7 +561,7 @@ U32 ZSTD_insertBt1(
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_updateTree_internal(
ZSTD_matchState_t* ms,
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
const U32 mls, const ZSTD_dictMode_e dictMode)
{
@@ -581,7 +581,7 @@ void ZSTD_updateTree_internal(
ms->nextToUpdate = target;
}
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend) {
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
}
@@ -590,7 +590,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32
ZSTD_insertBtAndGetAllMatches (
ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */
ZSTD_matchState_t* ms,
ZSTD_MatchState_t* ms,
U32* nextToUpdate3,
const BYTE* const ip, const BYTE* const iLimit,
const ZSTD_dictMode_e dictMode,
@@ -626,7 +626,7 @@ ZSTD_insertBtAndGetAllMatches (
U32 mnum = 0;
U32 nbCompares = 1U << cParams->searchLog;
const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
const ZSTD_MatchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
const ZSTD_compressionParameters* const dmsCParams =
dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL;
const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL;
@@ -665,13 +665,13 @@ ZSTD_insertBtAndGetAllMatches (
assert(curr >= windowLow);
if ( dictMode == ZSTD_extDict
&& ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */
& (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)
& (ZSTD_index_overlap_check(dictLimit, repIndex)) )
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
}
if (dictMode == ZSTD_dictMatchState
&& ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */
& ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */
& (ZSTD_index_overlap_check(dictLimit, repIndex)) )
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch;
} }
@@ -820,7 +820,7 @@ ZSTD_insertBtAndGetAllMatches (
typedef U32 (*ZSTD_getAllMatchesFn)(
ZSTD_match_t*,
ZSTD_matchState_t*,
ZSTD_MatchState_t*,
U32*,
const BYTE*,
const BYTE*,
@@ -832,7 +832,7 @@ FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_btGetAllMatches_internal(
ZSTD_match_t* matches,
ZSTD_matchState_t* ms,
ZSTD_MatchState_t* ms,
U32* nextToUpdate3,
const BYTE* ip,
const BYTE* const iHighLimit,
@@ -855,7 +855,7 @@ U32 ZSTD_btGetAllMatches_internal(
#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \
static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \
ZSTD_match_t* matches, \
ZSTD_matchState_t* ms, \
ZSTD_MatchState_t* ms, \
U32* nextToUpdate3, \
const BYTE* ip, \
const BYTE* const iHighLimit, \
@@ -887,7 +887,7 @@ GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState)
}
static ZSTD_getAllMatchesFn
ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode)
ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode)
{
ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = {
ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict),
@@ -906,7 +906,7 @@ ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const di
/* Struct containing info needed to make decision about ldm inclusion */
typedef struct {
rawSeqStore_t seqStore; /* External match candidates store for this block */
RawSeqStore_t seqStore; /* External match candidates store for this block */
U32 startPosInBlock; /* Start position of the current match candidate */
U32 endPosInBlock; /* End position of the current match candidate */
U32 offset; /* Offset of the match candidate */
@@ -916,7 +916,7 @@ typedef struct {
* Moves forward in @rawSeqStore by @nbBytes,
* which will update the fields 'pos' and 'posInSequence'.
*/
static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes)
static void ZSTD_optLdm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes)
{
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
@@ -973,7 +973,7 @@ ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock
return;
}
/* Matches may be < MINMATCH by this process. In that case, we will reject them
/* Matches may be < minMatch by this process. In that case, we will reject them
when we are deciding whether or not to add the ldm */
optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining;
optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining;
@@ -995,7 +995,8 @@ ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock
* into 'matches'. Maintains the correct ordering of 'matches'.
*/
static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
const ZSTD_optLdm_t* optLdm, U32 currPosInBlock)
const ZSTD_optLdm_t* optLdm, U32 currPosInBlock,
U32 minMatch)
{
U32 const posDiff = currPosInBlock - optLdm->startPosInBlock;
/* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */
@@ -1004,7 +1005,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
/* Ensure that current block position is not outside of the match */
if (currPosInBlock < optLdm->startPosInBlock
|| currPosInBlock >= optLdm->endPosInBlock
|| candidateMatchLength < MINMATCH) {
|| candidateMatchLength < minMatch) {
return;
}
@@ -1024,7 +1025,8 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
static void
ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
ZSTD_match_t* matches, U32* nbMatches,
U32 currPosInBlock, U32 remainingBytes)
U32 currPosInBlock, U32 remainingBytes,
U32 minMatch)
{
if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
return;
@@ -1041,7 +1043,7 @@ ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
}
ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes);
}
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock);
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock, minMatch);
}
@@ -1073,8 +1075,8 @@ listStats(const U32* table, int lastEltID)
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t
ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
SeqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const int optLevel,
@@ -1123,7 +1125,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 const ll0 = !litlen;
U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch);
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
(U32)(ip-istart), (U32)(iend-ip));
(U32)(ip-istart), (U32)(iend-ip),
minMatch);
if (!nbMatches) {
DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart));
ip++;
@@ -1198,7 +1201,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
for (cur = 1; cur <= last_pos; cur++) {
const BYTE* const inr = ip + cur;
assert(cur <= ZSTD_OPT_NUM);
DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur);
DEBUGLOG(7, "cPos:%i==rPos:%u", (int)(inr-istart), cur);
/* Fix current position with one literal if cheaper */
{ U32 const litlen = opt[cur-1].litlen + 1;
@@ -1208,8 +1211,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
assert(price < 1000000000); /* overflow check */
if (price <= opt[cur].price) {
ZSTD_optimal_t const prevMatch = opt[cur];
DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
DEBUGLOG(7, "cPos:%i==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
(int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]);
opt[cur] = opt[cur-1];
opt[cur].litlen = litlen;
@@ -1228,34 +1231,34 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
&& (with1literal < opt[cur+1].price) ) {
/* update offset history - before it disappears */
U32 const prev = cur - prevMatch.mlen;
repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0);
Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0);
assert(cur >= prevMatch.mlen);
DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !",
ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals),
newReps.rep[0], newReps.rep[1], newReps.rep[2] );
opt[cur+1] = prevMatch; /* mlen & offbase */
ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(repcodes_t));
ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(Repcodes_t));
opt[cur+1].litlen = 1;
opt[cur+1].price = with1literal;
if (last_pos < cur+1) last_pos = cur+1;
}
}
} else {
DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f)",
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
DEBUGLOG(7, "cPos:%i==rPos:%u : literal would cost more (%.2f>%.2f)",
(int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
}
}
/* Offset history is not updated during match comparison.
* Do it here, now that the match is selected and confirmed.
*/
ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(Repcodes_t));
assert(cur >= opt[cur].mlen);
if (opt[cur].litlen == 0) {
/* just finished a match => alter offset history */
U32 const prev = cur - opt[cur].mlen;
repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(Repcodes_t));
}
/* last match must start at a minimum distance of 8 from oend */
@@ -1277,7 +1280,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 matchNb;
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
(U32)(inr-istart), (U32)(iend-inr));
(U32)(inr-istart), (U32)(iend-inr),
minMatch);
if (!nbMatches) {
DEBUGLOG(7, "rPos:%u : no match found", cur);
@@ -1285,8 +1289,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
}
{ U32 const longestML = matches[nbMatches-1].len;
DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of longest ML=%u",
inr-istart, cur, nbMatches, longestML);
DEBUGLOG(7, "cPos:%i==rPos:%u, found %u matches, of longest ML=%u",
(int)(inr-istart), cur, nbMatches, longestML);
if ( (longestML > sufficient_len)
|| (cur + longestML >= ZSTD_OPT_NUM)
@@ -1354,10 +1358,10 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
/* Update offset history */
if (lastStretch.litlen == 0) {
/* finishing on a match : update offset history */
repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0);
ZSTD_memcpy(rep, &reps, sizeof(repcodes_t));
Repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0);
ZSTD_memcpy(rep, &reps, sizeof(Repcodes_t));
} else {
ZSTD_memcpy(rep, lastStretch.rep, sizeof(repcodes_t));
ZSTD_memcpy(rep, lastStretch.rep, sizeof(Repcodes_t));
assert(cur >= lastStretch.litlen);
cur -= lastStretch.litlen;
}
@@ -1412,8 +1416,8 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
U32 const mlen = opt[storePos].mlen;
U32 const offBase = opt[storePos].off;
U32 const advance = llen + mlen;
DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
anchor - istart, (unsigned)llen, (unsigned)mlen);
DEBUGLOG(6, "considering seq starting at %i, llen=%u, mlen=%u",
(int)(anchor - istart), (unsigned)llen, (unsigned)mlen);
if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
assert(storePos == storeEnd); /* must be last sequence */
@@ -1441,7 +1445,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
static size_t ZSTD_compressBlock_opt0(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode);
@@ -1450,7 +1454,7 @@ static size_t ZSTD_compressBlock_opt0(
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
static size_t ZSTD_compressBlock_opt2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode);
@@ -1459,7 +1463,7 @@ static size_t ZSTD_compressBlock_opt2(
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btopt(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btopt");
@@ -1478,8 +1482,8 @@ size_t ZSTD_compressBlock_btopt(
*/
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
void ZSTD_initStats_ultra(ZSTD_MatchState_t* ms,
SeqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
@@ -1504,7 +1508,7 @@ void ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
}
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
@@ -1512,7 +1516,7 @@ size_t ZSTD_compressBlock_btultra(
}
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 const curr = (U32)((const BYTE*)src - ms->window.base);
@@ -1542,14 +1546,14 @@ size_t ZSTD_compressBlock_btultra2(
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_btopt_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
@@ -1558,14 +1562,14 @@ size_t ZSTD_compressBlock_btopt_extDict(
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btultra_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict);

24
vendor/github.com/DataDog/zstd/zstd_opt.h generated vendored
View File

@@ -12,28 +12,24 @@
#ifndef ZSTD_OPT_H
#define ZSTD_OPT_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_compress_internal.h"
#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
/* used in ZSTD_loadDictionaryContent() */
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend);
void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend);
#endif
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btopt(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_BTOPT ZSTD_compressBlock_btopt
@@ -47,20 +43,20 @@ size_t ZSTD_compressBlock_btopt_extDict(
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_BTULTRA ZSTD_compressBlock_btultra
@@ -74,10 +70,6 @@ size_t ZSTD_compressBlock_btultra2(
#define ZSTD_COMPRESSBLOCK_BTULTRA2 NULL
#endif
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_OPT_H */
#endif /* USE_EXTERNAL_ZSTD */

241
vendor/github.com/DataDog/zstd/zstd_preSplit.c generated vendored Normal file
View File

@@ -0,0 +1,241 @@
#ifndef USE_EXTERNAL_ZSTD
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "compiler.h" /* ZSTD_ALIGNOF */
#include "mem.h" /* S64 */
#include "zstd_deps.h" /* ZSTD_memset */
#include "zstd_internal.h" /* ZSTD_STATIC_ASSERT */
#include "hist.h" /* HIST_add */
#include "zstd_preSplit.h"
#define BLOCKSIZE_MIN 3500
#define THRESHOLD_PENALTY_RATE 16
#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2)
#define THRESHOLD_PENALTY 3
#define HASHLENGTH 2
#define HASHLOG_MAX 10
#define HASHTABLESIZE (1 << HASHLOG_MAX)
#define HASHMASK (HASHTABLESIZE - 1)
#define KNUTH 0x9e3779b9
/* for hashLog > 8, hash 2 bytes.
* for hashLog == 8, just take the byte, no hashing.
* The speed of this method relies on compile-time constant propagation */
FORCE_INLINE_TEMPLATE unsigned hash2(const void *p, unsigned hashLog)
{
assert(hashLog >= 8);
if (hashLog == 8) return (U32)((const BYTE*)p)[0];
assert(hashLog <= HASHLOG_MAX);
return (U32)(MEM_read16(p)) * KNUTH >> (32 - hashLog);
}
typedef struct {
unsigned events[HASHTABLESIZE];
size_t nbEvents;
} Fingerprint;
typedef struct {
Fingerprint pastEvents;
Fingerprint newEvents;
} FPStats;
static void initStats(FPStats* fpstats)
{
ZSTD_memset(fpstats, 0, sizeof(FPStats));
}
FORCE_INLINE_TEMPLATE void
addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
{
const char* p = (const char*)src;
size_t limit = srcSize - HASHLENGTH + 1;
size_t n;
assert(srcSize >= HASHLENGTH);
for (n = 0; n < limit; n+=samplingRate) {
fp->events[hash2(p+n, hashLog)]++;
}
fp->nbEvents += limit/samplingRate;
}
FORCE_INLINE_TEMPLATE void
recordFingerprint_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
{
ZSTD_memset(fp, 0, sizeof(unsigned) * ((size_t)1 << hashLog));
fp->nbEvents = 0;
addEvents_generic(fp, src, srcSize, samplingRate, hashLog);
}
typedef void (*RecordEvents_f)(Fingerprint* fp, const void* src, size_t srcSize);
#define FP_RECORD(_rate) ZSTD_recordFingerprint_##_rate
#define ZSTD_GEN_RECORD_FINGERPRINT(_rate, _hSize) \
static void FP_RECORD(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \
{ \
recordFingerprint_generic(fp, src, srcSize, _rate, _hSize); \
}
ZSTD_GEN_RECORD_FINGERPRINT(1, 10)
ZSTD_GEN_RECORD_FINGERPRINT(5, 10)
ZSTD_GEN_RECORD_FINGERPRINT(11, 9)
ZSTD_GEN_RECORD_FINGERPRINT(43, 8)
static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); }
static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2, unsigned hashLog)
{
U64 distance = 0;
size_t n;
assert(hashLog <= HASHLOG_MAX);
for (n = 0; n < ((size_t)1 << hashLog); n++) {
distance +=
abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents);
}
return distance;
}
/* Compare newEvents with pastEvents
* return 1 when considered "too different"
*/
static int compareFingerprints(const Fingerprint* ref,
const Fingerprint* newfp,
int penalty,
unsigned hashLog)
{
assert(ref->nbEvents > 0);
assert(newfp->nbEvents > 0);
{ U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents;
U64 deviation = fpDistance(ref, newfp, hashLog);
U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
return deviation >= threshold;
}
}
static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp)
{
size_t n;
for (n = 0; n < HASHTABLESIZE; n++) {
acc->events[n] += newfp->events[n];
}
acc->nbEvents += newfp->nbEvents;
}
static void flushEvents(FPStats* fpstats)
{
size_t n;
for (n = 0; n < HASHTABLESIZE; n++) {
fpstats->pastEvents.events[n] = fpstats->newEvents.events[n];
}
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents;
ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents));
}
static void removeEvents(Fingerprint* acc, const Fingerprint* slice)
{
size_t n;
for (n = 0; n < HASHTABLESIZE; n++) {
assert(acc->events[n] >= slice->events[n]);
acc->events[n] -= slice->events[n];
}
acc->nbEvents -= slice->nbEvents;
}
#define CHUNKSIZE (8 << 10)
static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize,
int level,
void* workspace, size_t wkspSize)
{
static const RecordEvents_f records_fs[] = {
FP_RECORD(43), FP_RECORD(11), FP_RECORD(5), FP_RECORD(1)
};
static const unsigned hashParams[] = { 8, 9, 10, 10 };
const RecordEvents_f record_f = (assert(0<=level && level<=3), records_fs[level]);
FPStats* const fpstats = (FPStats*)workspace;
const char* p = (const char*)blockStart;
int penalty = THRESHOLD_PENALTY;
size_t pos = 0;
assert(blockSize == (128 << 10));
assert(workspace != NULL);
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
initStats(fpstats);
record_f(&fpstats->pastEvents, p, CHUNKSIZE);
for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) {
record_f(&fpstats->newEvents, p + pos, CHUNKSIZE);
if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty, hashParams[level])) {
return pos;
} else {
mergeEvents(&fpstats->pastEvents, &fpstats->newEvents);
if (penalty > 0) penalty--;
}
}
assert(pos == blockSize);
return blockSize;
(void)flushEvents; (void)removeEvents;
}
/* ZSTD_splitBlock_fromBorders(): very fast strategy :
* compare fingerprint from beginning and end of the block,
* derive from their difference if it's preferable to split in the middle,
* repeat the process a second time, for finer grained decision.
* 3 times did not brought improvements, so I stopped at 2.
* Benefits are good enough for a cheap heuristic.
* More accurate splitting saves more, but speed impact is also more perceptible.
* For better accuracy, use more elaborate variant *_byChunks.
*/
static size_t ZSTD_splitBlock_fromBorders(const void* blockStart, size_t blockSize,
void* workspace, size_t wkspSize)
{
#define SEGMENT_SIZE 512
FPStats* const fpstats = (FPStats*)workspace;
Fingerprint* middleEvents = (Fingerprint*)(void*)((char*)workspace + 512 * sizeof(unsigned));
assert(blockSize == (128 << 10));
assert(workspace != NULL);
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
initStats(fpstats);
HIST_add(fpstats->pastEvents.events, blockStart, SEGMENT_SIZE);
HIST_add(fpstats->newEvents.events, (const char*)blockStart + blockSize - SEGMENT_SIZE, SEGMENT_SIZE);
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents = SEGMENT_SIZE;
if (!compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, 0, 8))
return blockSize;
HIST_add(middleEvents->events, (const char*)blockStart + blockSize/2 - SEGMENT_SIZE/2, SEGMENT_SIZE);
middleEvents->nbEvents = SEGMENT_SIZE;
{ U64 const distFromBegin = fpDistance(&fpstats->pastEvents, middleEvents, 8);
U64 const distFromEnd = fpDistance(&fpstats->newEvents, middleEvents, 8);
U64 const minDistance = SEGMENT_SIZE * SEGMENT_SIZE / 3;
if (abs64((S64)distFromBegin - (S64)distFromEnd) < minDistance)
return 64 KB;
return (distFromBegin > distFromEnd) ? 32 KB : 96 KB;
}
}
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
int level,
void* workspace, size_t wkspSize)
{
DEBUGLOG(6, "ZSTD_splitBlock (level=%i)", level);
assert(0<=level && level<=4);
if (level == 0)
return ZSTD_splitBlock_fromBorders(blockStart, blockSize, workspace, wkspSize);
/* level >= 1*/
return ZSTD_splitBlock_byChunks(blockStart, blockSize, level-1, workspace, wkspSize);
}
#endif /* USE_EXTERNAL_ZSTD */

36
vendor/github.com/DataDog/zstd/zstd_preSplit.h generated vendored Normal file
View File

@@ -0,0 +1,36 @@
#ifndef USE_EXTERNAL_ZSTD
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_PRESPLIT_H
#define ZSTD_PRESPLIT_H
#include <stddef.h> /* size_t */
#define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208
/* ZSTD_splitBlock():
* @level must be a value between 0 and 4.
* higher levels spend more energy to detect block boundaries.
* @workspace must be aligned for size_t.
* @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE
* note:
* For the time being, this function only accepts full 128 KB blocks.
* Therefore, @blockSize must be == 128 KB.
* While this could be extended to smaller sizes in the future,
* it is not yet clear if this would be useful. TBD.
*/
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
int level,
void* workspace, size_t wkspSize);
#endif /* ZSTD_PRESPLIT_H */
#endif /* USE_EXTERNAL_ZSTD */

104
vendor/github.com/DataDog/zstd/zstd_stream.go generated vendored
View File

@@ -80,7 +80,6 @@ type Writer struct {
ctx *C.ZSTD_CCtx
dict []byte
srcBuffer []byte
dstBuffer []byte
firstError error
underlyingWriter io.Writer
@@ -138,7 +137,6 @@ func NewWriterLevelDict(w io.Writer, level int, dict []byte) *Writer {
CompressionLevel: level,
ctx: ctx,
dict: dict,
srcBuffer: make([]byte, 0),
dstBuffer: make([]byte, CompressBound(1024)),
firstError: err,
underlyingWriter: w,
@@ -154,65 +152,59 @@ func (w *Writer) Write(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
total := len(p)
// Check if dstBuffer is enough
w.dstBuffer = w.dstBuffer[0:cap(w.dstBuffer)]
if len(w.dstBuffer) < CompressBound(len(p)) {
w.dstBuffer = make([]byte, CompressBound(len(p)))
}
// Do not do an extra memcopy if zstd ingest all input data
srcData := p
fastPath := len(w.srcBuffer) == 0
if !fastPath {
w.srcBuffer = append(w.srcBuffer, p...)
srcData = w.srcBuffer
}
dstoff := 0
consumed := 0
for len(p) > 0 {
C.ZSTD_compressStream2_wrapper(
w.resultBuffer,
w.ctx,
unsafe.Pointer(&w.dstBuffer[dstoff]),
C.size_t(len(w.dstBuffer[dstoff:])),
unsafe.Pointer(&p[0]),
C.size_t(len(p)),
)
ret := int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
// The stream is dead after this.
w.firstError = err
return 0, err
}
p = p[w.resultBuffer.bytes_consumed:]
dstoff += int(w.resultBuffer.bytes_written)
consumed += int(w.resultBuffer.bytes_consumed)
if len(p) > 0 && dstoff == len(w.dstBuffer) {
// We have bytes remaining to compress and our output buffer
// filled up. This shouldn't happen since we calculated it
// in advance using CompressBound, but we need to handle it
// in case there was some miscalculation, or the internal
// stream buffer contained enough data from previous writes
// to overflow dstBuffer. (it's not clear from the docs
// whether this is possible)
//
// Allocate space for whatever we haven't compressed yet.
newbuf := make([]byte, len(w.dstBuffer)+CompressBound(total-consumed))
copy(newbuf, w.dstBuffer)
w.dstBuffer = newbuf
if len(srcData) == 0 {
// this is technically unnecessary: srcData is p or w.srcBuffer, and len() > 0 checked above
// but this ensures the code can change without dereferencing an srcData[0]
return 0, nil
}
C.ZSTD_compressStream2_wrapper(
w.resultBuffer,
w.ctx,
unsafe.Pointer(&w.dstBuffer[0]),
C.size_t(len(w.dstBuffer)),
unsafe.Pointer(&srcData[0]),
C.size_t(len(srcData)),
)
ret := int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
return 0, err
}
consumed := int(w.resultBuffer.bytes_consumed)
if !fastPath {
w.srcBuffer = w.srcBuffer[consumed:]
} else {
remaining := len(p) - consumed
if remaining > 0 {
// We still have some non-consumed data, copy remaining data to srcBuffer
// Try to not reallocate w.srcBuffer if we already have enough space
if cap(w.srcBuffer) >= remaining {
w.srcBuffer = w.srcBuffer[0:remaining]
} else {
w.srcBuffer = make([]byte, remaining)
}
copy(w.srcBuffer, p[consumed:])
}
}
written := int(w.resultBuffer.bytes_written)
// Write to underlying buffer
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
_, err := w.underlyingWriter.Write(w.dstBuffer[:dstoff])
// Same behaviour as zlib, we can't know how much data we wrote, only
// if there was an error
if err != nil {
return 0, err
}
return len(p), err
return total, err
}
// Flush writes any unwritten data to the underlying io.Writer.
@@ -223,24 +215,18 @@ func (w *Writer) Flush() error {
ret := 1 // So we loop at least once
for ret > 0 {
var srcPtr *byte // Do not point anywhere, if src is empty
if len(w.srcBuffer) > 0 {
srcPtr = &w.srcBuffer[0]
}
C.ZSTD_compressStream2_flush(
w.resultBuffer,
w.ctx,
unsafe.Pointer(&w.dstBuffer[0]),
C.size_t(len(w.dstBuffer)),
unsafe.Pointer(srcPtr),
C.size_t(len(w.srcBuffer)),
unsafe.Pointer(uintptr(0)),
C.size_t(0),
)
ret = int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
return err
}
w.srcBuffer = w.srcBuffer[w.resultBuffer.bytes_consumed:]
written := int(w.resultBuffer.bytes_written)
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
if err != nil {
@@ -267,24 +253,18 @@ func (w *Writer) Close() error {
ret := 1 // So we loop at least once
for ret > 0 {
var srcPtr *byte // Do not point anywhere, if src is empty
if len(w.srcBuffer) > 0 {
srcPtr = &w.srcBuffer[0]
}
C.ZSTD_compressStream2_finish(
w.resultBuffer,
w.ctx,
unsafe.Pointer(&w.dstBuffer[0]),
C.size_t(len(w.dstBuffer)),
unsafe.Pointer(srcPtr),
C.size_t(len(w.srcBuffer)),
unsafe.Pointer(uintptr(0)),
C.size_t(0),
)
ret = int(w.resultBuffer.return_code)
if err := getError(ret); err != nil {
return err
}
w.srcBuffer = w.srcBuffer[w.resultBuffer.bytes_consumed:]
written := int(w.resultBuffer.bytes_written)
_, err := w.underlyingWriter.Write(w.dstBuffer[:written])
if err != nil {
@@ -486,7 +466,7 @@ func (r *reader) Read(p []byte) (int, error) {
n, err = r.underlyingReader.Read(src[r.compressionLeft:])
}
if err != nil && err != io.EOF { // Handle underlying reader errors first
return 0, fmt.Errorf("failed to read from underlying reader: %s", err)
return 0, fmt.Errorf("failed to read from underlying reader: %w", err)
}
if n == 0 {
// Ideally, we'd return with ErrUnexpectedEOF in all cases where the stream was unexpectedly EOF'd
@@ -524,7 +504,7 @@ func (r *reader) Read(p []byte) (int, error) {
// Keep src here even though we reuse later, the code might be deleted at some point
runtime.KeepAlive(src)
if err := getError(retCode); err != nil {
return 0, fmt.Errorf("failed to decompress: %s", err)
return 0, fmt.Errorf("failed to decompress: %w", err)
}
// Put everything in buffer

17
vendor/github.com/DataDog/zstd/zstd_trace.h generated vendored
View File

@@ -12,23 +12,20 @@
#ifndef ZSTD_TRACE_H
#define ZSTD_TRACE_H
#if defined (__cplusplus)
extern "C" {
#endif
#include <stddef.h>
/* weak symbol support
* For now, enable conservatively:
* - Only GNUC
* - Only ELF
* - Only x86-64, i386 and aarch64
* - Only x86-64, i386, aarch64 and risc-v.
* Also, explicitly disable on platforms known not to work so they aren't
* forgotten in the future.
*/
#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
defined(__GNUC__) && defined(__ELF__) && \
(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \
(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \
!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
!defined(__CYGWIN__) && !defined(_AIX)
# define ZSTD_HAVE_WEAK_SYMBOLS 1
@@ -65,7 +62,7 @@ typedef struct {
/**
* Non-zero if streaming (de)compression is used.
*/
unsigned streaming;
int streaming;
/**
* The dictionary ID.
*/
@@ -74,7 +71,7 @@ typedef struct {
* Is the dictionary cold?
* Only set on decompression.
*/
unsigned dictionaryIsCold;
int dictionaryIsCold;
/**
* The dictionary size or zero if no dictionary.
*/
@@ -157,10 +154,6 @@ ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
#endif /* ZSTD_TRACE */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_TRACE_H */
#endif /* USE_EXTERNAL_ZSTD */

4
vendor/github.com/DataDog/zstd/zstd_v01.c generated vendored
View File

@@ -1384,7 +1384,7 @@ typedef struct {
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
} SeqStore_t;
typedef struct ZSTD_Cctx_s
@@ -1392,7 +1392,7 @@ typedef struct ZSTD_Cctx_s
const BYTE* base;
U32 current;
U32 nextUpdate;
seqStore_t seqStore;
SeqStore_t seqStore;
#ifdef __AVX2__
__m256i hashTable[HASH_TABLESIZE>>3];
#else

2
vendor/github.com/DataDog/zstd/zstd_v02.c generated vendored
View File

@@ -2723,7 +2723,7 @@ typedef struct {
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
} SeqStore_t;
/* *************************************

2
vendor/github.com/DataDog/zstd/zstd_v03.c generated vendored
View File

@@ -2363,7 +2363,7 @@ typedef struct {
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
} SeqStore_t;
/* *************************************

2
vendor/github.com/DataDog/zstd/zstd_v05.c generated vendored
View File

@@ -492,7 +492,7 @@ typedef struct {
U32 litLengthSum;
U32 litSum;
U32 offCodeSum;
} seqStore_t;
} SeqStore_t;

8
vendor/github.com/DataDog/zstd/zstd_v06.c generated vendored
View File

@@ -553,9 +553,9 @@ typedef struct {
U32 cachedLitLength;
const BYTE* cachedLiterals;
ZSTDv06_stats_t stats;
} seqStore_t;
} SeqStore_t;
void ZSTDv06_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
void ZSTDv06_seqToCodes(const SeqStore_t* seqStorePtr, size_t const nbSeq);
#endif /* ZSTDv06_CCOMMON_H_MODULE */
@@ -3920,6 +3920,10 @@ ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void)
if (zbd==NULL) return NULL;
memset(zbd, 0, sizeof(*zbd));
zbd->zd = ZSTDv06_createDCtx();
if (zbd->zd==NULL) {
ZBUFFv06_freeDCtx(zbd); /* avoid leaking the context */
return NULL;
}
zbd->stage = ZBUFFds_init;
return zbd;
}

4
vendor/github.com/DataDog/zstd/zstd_v07.c generated vendored
View File

@@ -2788,9 +2788,9 @@ typedef struct {
U32 cachedLitLength;
const BYTE* cachedLiterals;
ZSTDv07_stats_t stats;
} seqStore_t;
} SeqStore_t;
void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
void ZSTDv07_seqToCodes(const SeqStore_t* seqStorePtr, size_t const nbSeq);
/* custom memory allocation functions */
static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL };

227
vendor/github.com/DataDog/zstd/zstdmt_compress.c generated vendored
View File

@@ -91,9 +91,9 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
typedef struct buffer_s {
void* start;
size_t capacity;
} buffer_t;
} Buffer;
static const buffer_t g_nullBuffer = { NULL, 0 };
static const Buffer g_nullBuffer = { NULL, 0 };
typedef struct ZSTDMT_bufferPool_s {
ZSTD_pthread_mutex_t poolMutex;
@@ -101,7 +101,7 @@ typedef struct ZSTDMT_bufferPool_s {
unsigned totalBuffers;
unsigned nbBuffers;
ZSTD_customMem cMem;
buffer_t* buffers;
Buffer* buffers;
} ZSTDMT_bufferPool;
static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
@@ -129,7 +129,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_cu
ZSTD_customFree(bufPool, cMem);
return NULL;
}
bufPool->buffers = (buffer_t*)ZSTD_customCalloc(maxNbBuffers * sizeof(buffer_t), cMem);
bufPool->buffers = (Buffer*)ZSTD_customCalloc(maxNbBuffers * sizeof(Buffer), cMem);
if (bufPool->buffers==NULL) {
ZSTDMT_freeBufferPool(bufPool);
return NULL;
@@ -145,7 +145,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_cu
static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
{
size_t const poolSize = sizeof(*bufPool);
size_t const arraySize = bufPool->totalBuffers * sizeof(buffer_t);
size_t const arraySize = bufPool->totalBuffers * sizeof(Buffer);
unsigned u;
size_t totalBufferSize = 0;
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
@@ -190,13 +190,13 @@ static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool,
* assumption : bufPool must be valid
* @return : a buffer, with start pointer and size
* note: allocation may fail, in this case, start==NULL and size==0 */
static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
static Buffer ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
{
size_t const bSize = bufPool->bufferSize;
DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize);
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
if (bufPool->nbBuffers) { /* try to use an existing buffer */
buffer_t const buf = bufPool->buffers[--(bufPool->nbBuffers)];
Buffer const buf = bufPool->buffers[--(bufPool->nbBuffers)];
size_t const availBufferSize = buf.capacity;
bufPool->buffers[bufPool->nbBuffers] = g_nullBuffer;
if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) {
@@ -213,7 +213,7 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
/* create new buffer */
DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer");
{ buffer_t buffer;
{ Buffer buffer;
void* const start = ZSTD_customMalloc(bSize, bufPool->cMem);
buffer.start = start; /* note : start can be NULL if malloc fails ! */
buffer.capacity = (start==NULL) ? 0 : bSize;
@@ -232,12 +232,12 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
* @return : a buffer that is at least the buffer pool buffer size.
* If a reallocation happens, the data in the input buffer is copied.
*/
static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer)
static Buffer ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, Buffer buffer)
{
size_t const bSize = bufPool->bufferSize;
if (buffer.capacity < bSize) {
void* const start = ZSTD_customMalloc(bSize, bufPool->cMem);
buffer_t newBuffer;
Buffer newBuffer;
newBuffer.start = start;
newBuffer.capacity = start == NULL ? 0 : bSize;
if (start != NULL) {
@@ -253,7 +253,7 @@ static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer)
#endif
/* store buffer for later re-use, up to pool capacity */
static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, Buffer buf)
{
DEBUGLOG(5, "ZSTDMT_releaseBuffer");
if (buf.start == NULL) return; /* compatible with release on NULL */
@@ -291,23 +291,23 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool)
return ZSTDMT_sizeof_bufferPool(seqPool);
}
static rawSeqStore_t bufferToSeq(buffer_t buffer)
static RawSeqStore_t bufferToSeq(Buffer buffer)
{
rawSeqStore_t seq = kNullRawSeqStore;
RawSeqStore_t seq = kNullRawSeqStore;
seq.seq = (rawSeq*)buffer.start;
seq.capacity = buffer.capacity / sizeof(rawSeq);
return seq;
}
static buffer_t seqToBuffer(rawSeqStore_t seq)
static Buffer seqToBuffer(RawSeqStore_t seq)
{
buffer_t buffer;
Buffer buffer;
buffer.start = seq.seq;
buffer.capacity = seq.capacity * sizeof(rawSeq);
return buffer;
}
static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool)
static RawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool)
{
if (seqPool->bufferSize == 0) {
return kNullRawSeqStore;
@@ -316,13 +316,13 @@ static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool)
}
#if ZSTD_RESIZE_SEQPOOL
static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq)
static RawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq)
{
return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq)));
}
#endif
static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq)
static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq)
{
ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq));
}
@@ -467,7 +467,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
typedef struct {
void const* start;
size_t size;
} range_t;
} Range;
typedef struct {
/* All variables in the struct are protected by mutex. */
@@ -483,10 +483,10 @@ typedef struct {
ZSTD_pthread_mutex_t ldmWindowMutex;
ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */
ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */
} serialState_t;
} SerialState;
static int
ZSTDMT_serialState_reset(serialState_t* serialState,
ZSTDMT_serialState_reset(SerialState* serialState,
ZSTDMT_seqPool* seqPool,
ZSTD_CCtx_params params,
size_t jobSize,
@@ -556,7 +556,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState,
return 0;
}
static int ZSTDMT_serialState_init(serialState_t* serialState)
static int ZSTDMT_serialState_init(SerialState* serialState)
{
int initError = 0;
ZSTD_memset(serialState, 0, sizeof(*serialState));
@@ -567,7 +567,7 @@ static int ZSTDMT_serialState_init(serialState_t* serialState)
return initError;
}
static void ZSTDMT_serialState_free(serialState_t* serialState)
static void ZSTDMT_serialState_free(SerialState* serialState)
{
ZSTD_customMem cMem = serialState->params.customMem;
ZSTD_pthread_mutex_destroy(&serialState->mutex);
@@ -578,9 +578,10 @@ static void ZSTDMT_serialState_free(serialState_t* serialState)
ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem);
}
static void ZSTDMT_serialState_update(serialState_t* serialState,
ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore,
range_t src, unsigned jobID)
static void
ZSTDMT_serialState_genSequences(SerialState* serialState,
RawSeqStore_t* seqStore,
Range src, unsigned jobID)
{
/* Wait for our turn */
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex);
@@ -593,12 +594,13 @@ static void ZSTDMT_serialState_update(serialState_t* serialState,
/* It is now our turn, do any processing necessary */
if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) {
size_t error;
assert(seqStore.seq != NULL && seqStore.pos == 0 &&
seqStore.size == 0 && seqStore.capacity > 0);
DEBUGLOG(6, "ZSTDMT_serialState_genSequences: LDM update");
assert(seqStore->seq != NULL && seqStore->pos == 0 &&
seqStore->size == 0 && seqStore->capacity > 0);
assert(src.size <= serialState->params.jobSize);
ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0);
error = ZSTD_ldm_generateSequences(
&serialState->ldmState, &seqStore,
&serialState->ldmState, seqStore,
&serialState->params.ldmParams, src.start, src.size);
/* We provide a large enough buffer to never fail. */
assert(!ZSTD_isError(error)); (void)error;
@@ -617,14 +619,22 @@ static void ZSTDMT_serialState_update(serialState_t* serialState,
serialState->nextJobID++;
ZSTD_pthread_cond_broadcast(&serialState->cond);
ZSTD_pthread_mutex_unlock(&serialState->mutex);
}
if (seqStore.size > 0) {
ZSTD_referenceExternalSequences(jobCCtx, seqStore.seq, seqStore.size);
assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable);
static void
ZSTDMT_serialState_applySequences(const SerialState* serialState, /* just for an assert() check */
ZSTD_CCtx* jobCCtx,
const RawSeqStore_t* seqStore)
{
if (seqStore->size > 0) {
DEBUGLOG(5, "ZSTDMT_serialState_applySequences: uploading %u external sequences", (unsigned)seqStore->size);
assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); (void)serialState;
assert(jobCCtx);
ZSTD_referenceExternalSequences(jobCCtx, seqStore->seq, seqStore->size);
}
}
static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState,
static void ZSTDMT_serialState_ensureFinished(SerialState* serialState,
unsigned jobID, size_t cSize)
{
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex);
@@ -648,28 +658,28 @@ static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState,
/* ===== Worker thread ===== */
/* ------------------------------------------ */
static const range_t kNullRange = { NULL, 0 };
static const Range kNullRange = { NULL, 0 };
typedef struct {
size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */
size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */
ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */
ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */
ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */
ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */
ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */
serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */
buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */
range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */
range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */
unsigned jobID; /* set by mtctx, then read by worker => no barrier */
unsigned firstJob; /* set by mtctx, then read by worker => no barrier */
unsigned lastJob; /* set by mtctx, then read by worker => no barrier */
ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */
const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */
unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */
size_t dstFlushed; /* used only by mtctx */
unsigned frameChecksumNeeded; /* used only by mtctx */
size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */
size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */
ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */
ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */
ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */
ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */
ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */
SerialState* serial; /* Thread-safe - used by mtctx and (all) workers */
Buffer dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */
Range prefix; /* set by mtctx, then read by worker & mtctx => no barrier */
Range src; /* set by mtctx, then read by worker & mtctx => no barrier */
unsigned jobID; /* set by mtctx, then read by worker => no barrier */
unsigned firstJob; /* set by mtctx, then read by worker => no barrier */
unsigned lastJob; /* set by mtctx, then read by worker => no barrier */
ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */
const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */
unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */
size_t dstFlushed; /* used only by mtctx */
unsigned frameChecksumNeeded; /* used only by mtctx */
} ZSTDMT_jobDescription;
#define JOB_ERROR(e) \
@@ -686,10 +696,11 @@ static void ZSTDMT_compressionJob(void* jobDescription)
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */
ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool);
rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool);
buffer_t dstBuff = job->dstBuff;
RawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool);
Buffer dstBuff = job->dstBuff;
size_t lastCBlockSize = 0;
DEBUGLOG(5, "ZSTDMT_compressionJob: job %u", job->jobID);
/* resources */
if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation));
if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */
@@ -711,11 +722,15 @@ static void ZSTDMT_compressionJob(void* jobDescription)
/* init */
/* Perform serial step as early as possible */
ZSTDMT_serialState_genSequences(job->serial, &rawSeqStore, job->src, job->jobID);
if (job->cdict) {
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize);
assert(job->firstJob); /* only allowed for first job */
if (ZSTD_isError(initError)) JOB_ERROR(initError);
} else { /* srcStart points at reloaded section */
} else {
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
{ size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
@@ -724,16 +739,17 @@ static void ZSTDMT_compressionJob(void* jobDescription)
size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0);
if (ZSTD_isError(err)) JOB_ERROR(err);
}
DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size);
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
job->prefix.start, job->prefix.size, ZSTD_dct_rawContent,
ZSTD_dtlm_fast,
NULL, /*cdict*/
&jobParams, pledgedSrcSize);
if (ZSTD_isError(initError)) JOB_ERROR(initError);
} }
/* Perform serial step as early as possible, but after CCtx initialization */
ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID);
/* External Sequences can only be applied after CCtx initialization */
ZSTDMT_serialState_applySequences(job->serial, cctx, &rawSeqStore);
if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */
size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0);
@@ -742,7 +758,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
ZSTD_invalidateRepCodes(cctx);
}
/* compress */
/* compress the entire job by smaller chunks, for better granularity */
{ size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX;
int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize);
const BYTE* ip = (const BYTE*) job->src.start;
@@ -810,10 +826,10 @@ _endJob:
/* ------------------------------------------ */
typedef struct {
range_t prefix; /* read-only non-owned prefix buffer */
buffer_t buffer;
Range prefix; /* read-only non-owned prefix buffer */
Buffer buffer;
size_t filled;
} inBuff_t;
} InBuff_t;
typedef struct {
BYTE* buffer; /* The round input buffer. All jobs get references
@@ -827,9 +843,9 @@ typedef struct {
* the inBuff is sent to the worker thread.
* pos <= capacity.
*/
} roundBuff_t;
} RoundBuff_t;
static const roundBuff_t kNullRoundBuff = {NULL, 0, 0};
static const RoundBuff_t kNullRoundBuff = {NULL, 0, 0};
#define RSYNC_LENGTH 32
/* Don't create chunks smaller than the zstd block size.
@@ -846,7 +862,7 @@ typedef struct {
U64 hash;
U64 hitMask;
U64 primePower;
} rsyncState_t;
} RSyncState_t;
struct ZSTDMT_CCtx_s {
POOL_ctx* factory;
@@ -858,10 +874,10 @@ struct ZSTDMT_CCtx_s {
size_t targetSectionSize;
size_t targetPrefixSize;
int jobReady; /* 1 => one job is already prepared, but pool has shortage of workers. Don't create a new job. */
inBuff_t inBuff;
roundBuff_t roundBuff;
serialState_t serial;
rsyncState_t rsync;
InBuff_t inBuff;
RoundBuff_t roundBuff;
SerialState serial;
RSyncState_t rsync;
unsigned jobIDMask;
unsigned doneJobID;
unsigned nextJobID;
@@ -1246,13 +1262,11 @@ size_t ZSTDMT_initCStream_internal(
/* init */
if (params.nbWorkers != mtctx->params.nbWorkers)
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , "");
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, (unsigned)params.nbWorkers) , "");
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX;
DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers);
if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */
ZSTDMT_waitForAllJobsCompleted(mtctx);
ZSTDMT_releaseAllJobResources(mtctx);
@@ -1261,15 +1275,14 @@ size_t ZSTDMT_initCStream_internal(
mtctx->params = params;
mtctx->frameContentSize = pledgedSrcSize;
ZSTD_freeCDict(mtctx->cdictLocal);
if (dict) {
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
} else {
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = NULL;
mtctx->cdict = cdict;
}
@@ -1335,9 +1348,32 @@ size_t ZSTDMT_initCStream_internal(
mtctx->allJobsCompleted = 0;
mtctx->consumed = 0;
mtctx->produced = 0;
/* update dictionary */
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = NULL;
mtctx->cdict = NULL;
if (dict) {
if (dictContentType == ZSTD_dct_rawContent) {
mtctx->inBuff.prefix.start = (const BYTE*)dict;
mtctx->inBuff.prefix.size = dictSize;
} else {
/* note : a loadPrefix becomes an internal CDict */
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byRef, dictContentType,
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
}
} else {
mtctx->cdict = cdict;
}
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
dict, dictSize, dictContentType))
return ERROR(memory_allocation);
return 0;
}
@@ -1404,7 +1440,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS
mtctx->roundBuff.pos += srcSize;
mtctx->inBuff.buffer = g_nullBuffer;
mtctx->inBuff.filled = 0;
/* Set the prefix */
/* Set the prefix for next job */
if (!endFrame) {
size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize);
mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize;
@@ -1541,12 +1577,17 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
* If the data of the first job is broken up into two segments, we cover both
* sections.
*/
static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
{
unsigned const firstJobID = mtctx->doneJobID;
unsigned const lastJobID = mtctx->nextJobID;
unsigned jobID;
/* no need to check during first round */
size_t roundBuffCapacity = mtctx->roundBuff.capacity;
size_t nbJobs1stRoundMin = roundBuffCapacity / mtctx->targetSectionSize;
if (lastJobID < nbJobs1stRoundMin) return kNullRange;
for (jobID = firstJobID; jobID < lastJobID; ++jobID) {
unsigned const wJobID = jobID & mtctx->jobIDMask;
size_t consumed;
@@ -1556,7 +1597,7 @@ static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
if (consumed < mtctx->jobs[wJobID].src.size) {
range_t range = mtctx->jobs[wJobID].prefix;
Range range = mtctx->jobs[wJobID].prefix;
if (range.size == 0) {
/* Empty prefix */
range = mtctx->jobs[wJobID].src;
@@ -1572,7 +1613,7 @@ static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
/**
* Returns non-zero iff buffer and range overlap.
*/
static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
static int ZSTDMT_isOverlapped(Buffer buffer, Range range)
{
BYTE const* const bufferStart = (BYTE const*)buffer.start;
BYTE const* const rangeStart = (BYTE const*)range.start;
@@ -1592,10 +1633,10 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
}
}
static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window)
static int ZSTDMT_doesOverlapWindow(Buffer buffer, ZSTD_window_t window)
{
range_t extDict;
range_t prefix;
Range extDict;
Range prefix;
DEBUGLOG(5, "ZSTDMT_doesOverlapWindow");
extDict.start = window.dictBase + window.lowLimit;
@@ -1614,7 +1655,7 @@ static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window)
|| ZSTDMT_isOverlapped(buffer, prefix);
}
static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer)
static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, Buffer buffer)
{
if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) {
ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex;
@@ -1639,16 +1680,16 @@ static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer)
*/
static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
{
range_t const inUse = ZSTDMT_getInputDataInUse(mtctx);
Range const inUse = ZSTDMT_getInputDataInUse(mtctx);
size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos;
size_t const target = mtctx->targetSectionSize;
buffer_t buffer;
size_t const spaceNeeded = mtctx->targetSectionSize;
Buffer buffer;
DEBUGLOG(5, "ZSTDMT_tryGetInputRange");
assert(mtctx->inBuff.buffer.start == NULL);
assert(mtctx->roundBuff.capacity >= target);
assert(mtctx->roundBuff.capacity >= spaceNeeded);
if (spaceLeft < target) {
if (spaceLeft < spaceNeeded) {
/* ZSTD_invalidateRepCodes() doesn't work for extDict variants.
* Simply copy the prefix to the beginning in that case.
*/
@@ -1667,7 +1708,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
mtctx->roundBuff.pos = prefixSize;
}
buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos;
buffer.capacity = target;
buffer.capacity = spaceNeeded;
if (ZSTDMT_isOverlapped(buffer, inUse)) {
DEBUGLOG(5, "Waiting for buffer...");
@@ -1694,7 +1735,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
typedef struct {
size_t toLoad; /* The number of bytes to load from the input. */
int flush; /* Boolean declaring if we must flush because we found a synchronization point. */
} syncPoint_t;
} SyncPoint;
/**
* Searches through the input for a synchronization point. If one is found, we
@@ -1702,14 +1743,14 @@ typedef struct {
* Otherwise, we will load as many bytes as possible and instruct the caller
* to continue as normal.
*/
static syncPoint_t
static SyncPoint
findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
{
BYTE const* const istart = (BYTE const*)input.src + input.pos;
U64 const primePower = mtctx->rsync.primePower;
U64 const hitMask = mtctx->rsync.hitMask;
syncPoint_t syncPoint;
SyncPoint syncPoint;
U64 hash;
BYTE const* prev;
size_t pos;
@@ -1841,7 +1882,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
}
if (mtctx->inBuff.buffer.start != NULL) {
syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input);
SyncPoint const syncPoint = findSynchronizationPoint(mtctx, *input);
if (syncPoint.flush && endOp == ZSTD_e_continue) {
endOp = ZSTD_e_flush;
}

19
vendor/github.com/DataDog/zstd/zstdmt_compress.h generated vendored
View File

@@ -12,10 +12,10 @@
#ifndef ZSTDMT_COMPRESS_H
#define ZSTDMT_COMPRESS_H
#if defined (__cplusplus)
extern "C" {
#endif
/* === Dependencies === */
#include "zstd_deps.h" /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
/* Note : This is an internal API.
* These APIs used to be exposed with ZSTDLIB_API,
@@ -26,12 +26,6 @@
* otherwise ZSTDMT_createCCtx*() will fail.
*/
/* === Dependencies === */
#include "zstd_deps.h" /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
/* === Constants === */
#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */
# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256)
@@ -106,11 +100,6 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
*/
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDMT_COMPRESS_H */
#endif /* USE_EXTERNAL_ZSTD */

36
vendor/github.com/briandowns/openweathermap/current.go generated vendored
View File

@@ -25,24 +25,24 @@ import (
// CurrentWeatherData struct contains an aggregate view of the structs
// defined above for JSON to be unmarshaled into.
type CurrentWeatherData struct {
GeoPos Coordinates `json:"coord"`
Sys Sys `json:"sys"`
Base string `json:"base"`
Weather []Weather `json:"weather"`
Main Main `json:"main"`
Visibility int `json:"visibility"`
Wind Wind `json:"wind"`
Clouds Clouds `json:"clouds"`
Rain Rain `json:"rain"`
Snow Snow `json:"snow"`
Dt int `json:"dt"`
ID int `json:"id"`
Name string `json:"name"`
Cod int `json:"cod"`
Timezone int `json:"timezone"`
Unit string
Lang string
Key string
GeoPos Coordinates `json:"coord"`
Sys Sys `json:"sys"`
Base string `json:"base"`
Weather []Weather `json:"weather"`
Main Main `json:"main"`
Visibility int `json:"visibility"`
Wind Wind `json:"wind"`
Clouds Clouds `json:"clouds"`
Rain Rain `json:"rain"`
Snow Snow `json:"snow"`
Dt int `json:"dt"`
ID int `json:"id"`
Name string `json:"name"`
Cod int `json:"cod"`
Timezone int `json:"timezone"`
Unit string
Lang string
Key string
*Settings
}

2
vendor/github.com/briandowns/openweathermap/history.go generated vendored
View File

@@ -68,7 +68,7 @@ type HistoricalWeatherData struct {
}
// NewHistorical returns a new HistoricalWeatherData pointer with
//the supplied arguments.
// the supplied arguments.
func NewHistorical(unit, key string, options ...Option) (*HistoricalWeatherData, error) {
h := &HistoricalWeatherData{
Settings: NewSettings(),

52
vendor/github.com/briandowns/openweathermap/onecall.go generated vendored
View File

@@ -17,20 +17,22 @@ import (
"encoding/json"
"fmt"
"strings"
"time"
)
// OneCallData struct contains an aggregate view of the structs
// defined above for JSON to be unmarshaled into.
type OneCallData struct {
Latitude float64 `json:"lat"`
Longitude float64 `json:"lon"`
Timezone string `json:"timezone"`
TimezoneOffset int `json:"timezone_offset"`
Current OneCallCurrentData `json:"current,omitempty"`
Minutely []OneCallMinutelyData `json:"minutely,omitempty"`
Hourly []OneCallHourlyData `json:"hourly,omitempty"`
Daily []OneCallDailyData `json:"daily,omitempty"`
Alerts []OneCallAlertData `json:"alerts,omitempty"`
Latitude float64 `json:"lat"`
Longitude float64 `json:"lon"`
Timezone string `json:"timezone"`
TimezoneOffset int `json:"timezone_offset"`
Current OneCallCurrentData `json:"current,omitempty"`
Minutely []OneCallMinutelyData `json:"minutely,omitempty"`
Hourly []OneCallHourlyData `json:"hourly,omitempty"`
Daily []OneCallDailyData `json:"daily,omitempty"`
Alerts []OneCallAlertData `json:"alerts,omitempty"`
Data []OneCallTimeMachineData `json:"data,omitempty"`
Unit string
Lang string
@@ -39,6 +41,24 @@ type OneCallData struct {
*Settings
}
type OneCallTimeMachineData struct {
Dt int `json:"dt"`
Sunrise int `json:"sunrise"`
Sunset int `json:"sunset"`
Temp float64 `json:"temp"`
FeelsLike float64 `json:"feels_like"`
Pressure int `json:"pressure"`
Humidity int `json:"humidity"`
DewPoint float64 `json:"dew_point"`
Clouds int `json:"clouds"`
UVI float64 `json:"uvi"`
Visibility int `json:"visibility"`
WindSpeed float64 `json:"wind_speed"`
WindGust float64 `json:"wind_gust,omitempty"`
WindDeg float64 `json:"wind_deg"`
Weather []Weather `json:"weather"`
}
type OneCallCurrentData struct {
Dt int `json:"dt"`
Sunrise int `json:"sunrise"`
@@ -161,7 +181,19 @@ func NewOneCall(unit, lang, key string, excludes []string, options ...Option) (*
// OneCallByCoordinates will provide the onecall weather with the
// provided location coordinates.
func (w *OneCallData) OneCallByCoordinates(location *Coordinates) error {
response, err := w.client.Get(fmt.Sprintf(fmt.Sprintf(onecallURL, "appid=%s&lat=%f&lon=%f&units=%s&lang=%s&exclude=%s"), w.Key, location.Latitude, location.Longitude, w.Unit, w.Lang, w.Excludes))
response, err := w.client.Get(fmt.Sprintf(fmt.Sprintf(onecallURL, "?appid=%s&lat=%f&lon=%f&units=%s&lang=%s&exclude=%s"), w.Key, location.Latitude, location.Longitude, w.Unit, w.Lang, w.Excludes))
if err != nil {
return err
}
defer response.Body.Close()
return json.NewDecoder(response.Body).Decode(&w)
}
// OneCallTimeMachine will provide the onecall timemachine weather with the
// provided location coordinates and unix timestamp
func (w *OneCallData) OneCallTimeMachine(location *Coordinates, datetime time.Time) error {
response, err := w.client.Get(fmt.Sprintf(fmt.Sprintf(onecallURL, "/timemachine?appid=%s&lat=%f&lon=%f&units=%s&lang=%s&dt=%d"), w.Key, location.Latitude, location.Longitude, w.Unit, w.Lang, datetime.Unix()))
if err != nil {
return err
}

2
vendor/github.com/briandowns/openweathermap/openweathermap.go generated vendored
View File

@@ -35,7 +35,7 @@ var (
var DataUnits = map[string]string{"C": "metric", "F": "imperial", "K": "internal"}
var (
baseURL = "https://api.openweathermap.org/data/2.5/weather?%s"
onecallURL = "https://api.openweathermap.org/data/3.0/onecall?%s"
onecallURL = "https://api.openweathermap.org/data/3.0/onecall%s"
iconURL = "https://openweathermap.org/img/w/%s"
groupURL = "http://api.openweathermap.org/data/2.5/group?%s"
stationURL = "https://api.openweathermap.org/data/2.5/station?id=%d"

30
vendor/github.com/briandowns/openweathermap/uv.go generated vendored
View File

@@ -106,33 +106,33 @@ type UVIndexInfo struct {
// UVData contains data in regards to UV index ranges, rankings, and steps for protection
var UVData = []UVIndexInfo{
{
UVIndex: []float64{0, 2.9},
MGC: "Green",
Risk: "Low",
UVIndex: []float64{0, 2.9},
MGC: "Green",
Risk: "Low",
RecommendedProtection: "Wear sunglasses on bright days; use sunscreen if there is snow on the ground, which reflects UV radiation, or if you have particularly fair skin.",
},
{
UVIndex: []float64{3, 5.9},
MGC: "Yellow",
Risk: "Moderate",
UVIndex: []float64{3, 5.9},
MGC: "Yellow",
Risk: "Moderate",
RecommendedProtection: "Take precautions, such as covering up, if you will be outside. Stay in shade near midday when the sun is strongest.",
},
{
UVIndex: []float64{6, 7.9},
MGC: "Orange",
Risk: "High",
UVIndex: []float64{6, 7.9},
MGC: "Orange",
Risk: "High",
RecommendedProtection: "Cover the body with sun protective clothing, use SPF 30+ sunscreen, wear a hat, reduce time in the sun within three hours of solar noon, and wear sunglasses.",
},
{
UVIndex: []float64{8, 10.9},
MGC: "Red",
Risk: "Very high",
UVIndex: []float64{8, 10.9},
MGC: "Red",
Risk: "Very high",
RecommendedProtection: "Wear SPF 30+ sunscreen, a shirt, sunglasses, and a wide-brimmed hat. Do not stay in the sun for too long.",
},
{
UVIndex: []float64{11},
MGC: "Violet",
Risk: "Extreme",
UVIndex: []float64{11},
MGC: "Violet",
Risk: "Extreme",
RecommendedProtection: "Take all precautions: Wear SPF 30+ sunscreen, a long-sleeved shirt and trousers, sunglasses, and a very broad hat. Avoid the sun within three hours of solar noon.",
},
}

Some files were not shown because too many files have changed in this diff Show More