aleesa-bot/aleesa-telegram-go
0
Fork 0
Code Activity

[WIP] lint a bit, update deps

Browse Source
This commit is contained in:
eleksir
2024-02-27 00:34:44 +03:00
parent 1892960896
commit 1f97424f31
201 changed files with 21474 additions and 9282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
vendor/github.com/cockroachdb/pebble/options.go generated vendored
View File

@@ -17,8 +17,10 @@ import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/rangekey"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
)
@@ -117,11 +119,24 @@ type IterOptions struct {
// false to skip scanning. This function must be thread-safe since the same
// function can be used by multiple iterators, if the iterator is cloned.
TableFilter func(userProps map[string]string) bool
// SkipPoint may be used to skip over point keys that don't match an
// arbitrary predicate during iteration. If set, the Iterator invokes
// SkipPoint for keys encountered. If SkipPoint returns true, the iterator
// will skip the key without yielding it to the iterator operation in
// progress.
//
// SkipPoint must be a pure function and always return the same result when
// provided the same arguments. The iterator may call SkipPoint multiple
// times for the same user key.
SkipPoint func(userKey []byte) bool
// PointKeyFilters can be used to avoid scanning tables and blocks in tables
// when iterating over point keys. It is requires that this slice is sorted in
// increasing order of the BlockPropertyFilter.ShortID. This slice represents
// an intersection across all filters, i.e., all filters must indicate that the
// block is relevant.
// when iterating over point keys. This slice represents an intersection
// across all filters, i.e., all filters must indicate that the block is
// relevant.
//
// Performance note: When len(PointKeyFilters) > 0, the caller should ensure
// that cap(PointKeyFilters) is at least len(PointKeyFilters)+1. This helps
// avoid allocations in Pebble internal code that mutates the slice.
PointKeyFilters []BlockPropertyFilter
// RangeKeyFilters can be usefd to avoid scanning tables and blocks in tables
// when iterating over range keys. The same requirements that apply to
@@ -180,6 +195,10 @@ type IterOptions struct {
level manifest.Level
// disableLazyCombinedIteration is an internal testing option.
disableLazyCombinedIteration bool
// snapshotForHideObsoletePoints is specified for/by levelIter when opening
// files and is used to decide whether to hide obsolete points. A value of 0
// implies obsolete points should not be hidden.
snapshotForHideObsoletePoints uint64
// NB: If adding new Options, you must account for them in iterator
// construction and Iterator.SetOptions.
@@ -222,6 +241,38 @@ func (o *IterOptions) getLogger() Logger {
return o.logger
}
// SpanIterOptions creates a SpanIterOptions from this IterOptions.
func (o *IterOptions) SpanIterOptions() keyspan.SpanIterOptions {
if o == nil {
return keyspan.SpanIterOptions{}
}
return keyspan.SpanIterOptions{
RangeKeyFilters: o.RangeKeyFilters,
}
}
// scanInternalOptions is similar to IterOptions, meant for use with
// scanInternalIterator.
type scanInternalOptions struct {
IterOptions
visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error
visitRangeDel func(start, end []byte, seqNum uint64) error
visitRangeKey func(start, end []byte, keys []rangekey.Key) error
visitSharedFile func(sst *SharedSSTMeta) error
// skipSharedLevels skips levels that are shareable (level >=
// sharedLevelStart).
skipSharedLevels bool
// includeObsoleteKeys specifies whether keys shadowed by newer internal keys
// are exposed. If false, only one internal key per user key is exposed.
includeObsoleteKeys bool
// rateLimitFunc is used to limit the amount of bytes read per second.
rateLimitFunc func(key *InternalKey, value LazyValue) error
}
// RangeKeyMasking configures automatic hiding of point keys by range keys. A
// non-nil Suffix enables range-key masking. When enabled, range keys with
// suffixes ≥ Suffix behave as masks. All point keys that are contained within a
@@ -497,18 +548,12 @@ type Options struct {
// concurrent compaction is added. This works "on top" of
// L0CompactionConcurrency, so the higher of the count of compaction
// concurrency slots as determined by the two options is chosen.
CompactionDebtConcurrency int
CompactionDebtConcurrency uint64
// MinDeletionRate is the minimum number of bytes per second that would
// be deleted. Deletion pacing is used to slow down deletions when
// compactions finish up or readers close, and newly-obsolete files need
// cleaning up. Deleting lots of files at once can cause disk latency to
// go up on some SSDs, which this functionality guards against. This is a
// minimum as the maximum is theoretically unlimited; pacing is disabled
// when there are too many obsolete files relative to live bytes, or
// there isn't enough disk space available. Setting this to 0 disables
// deletion pacing, which is also the default.
MinDeletionRate int
// IngestSplit, if it returns true, allows for ingest-time splitting of
// existing sstables into two virtual sstables to allow ingestion sstables to
// slot into a lower level than they otherwise would have.
IngestSplit func() bool
// ReadCompactionRate controls the frequency of read triggered
// compactions by adjusting `AllowedSeeks` in manifest.FileMetadata:
@@ -573,10 +618,10 @@ type Options struct {
// desired size of each level of the LSM. Defaults to 10.
LevelMultiplier int
// MultiLevelCompactionHueristic determines whether to add an additional
// MultiLevelCompactionHeuristic determines whether to add an additional
// level to a conventional two level compaction. If nil, a multilevel
// compaction will never get triggered.
MultiLevelCompactionHueristic MultiLevelHeuristic
MultiLevelCompactionHeuristic MultiLevelHeuristic
// MaxWriterConcurrency is used to indicate the maximum number of
// compression workers the compression queue is allowed to use. If
@@ -597,8 +642,11 @@ type Options struct {
CPUWorkPermissionGranter CPUWorkPermissionGranter
// EnableValueBlocks is used to decide whether to enable writing
// TableFormatPebblev3 sstables. WARNING: do not return true yet, since
// support for TableFormatPebblev3 is incomplete and not production ready.
// TableFormatPebblev3 sstables. This setting is only respected by a
// specific subset of format major versions: FormatSSTableValueBlocks,
// FormatFlushableIngest and FormatPrePebblev1MarkedCompacted. In lower
// format major versions, value blocks are never enabled. In higher
// format major versions, value blocks are always enabled.
EnableValueBlocks func() bool
// ShortAttributeExtractor is used iff EnableValueBlocks() returns true
@@ -607,18 +655,24 @@ type Options struct {
ShortAttributeExtractor ShortAttributeExtractor
// RequiredInPlaceValueBound specifies an optional span of user key
// prefixes for which the values must be stored with the key. This is
// useful for statically known exclusions to value separation. In
// CockroachDB, this will be used for the lock table key space that has
// non-empty suffixes, but those locks don't represent actual MVCC
// versions (the suffix ordering is arbitrary). We will also need to add
// support for dynamically configured exclusions (we want the default to
// be to allow Pebble to decide whether to separate the value or not,
// hence this is structured as exclusions), for example, for users of
// CockroachDB to dynamically exclude certain tables.
// prefixes that are not-MVCC, but have a suffix. For these the values
// must be stored with the key, since the concept of "older versions" is
// not defined. It is also useful for statically known exclusions to value
// separation. In CockroachDB, this will be used for the lock table key
// space that has non-empty suffixes, but those locks don't represent
// actual MVCC versions (the suffix ordering is arbitrary). We will also
// need to add support for dynamically configured exclusions (we want the
// default to be to allow Pebble to decide whether to separate the value
// or not, hence this is structured as exclusions), for example, for users
// of CockroachDB to dynamically exclude certain tables.
//
// Any change in exclusion behavior takes effect only on future written
// sstables, and does not start rewriting existing sstables.
//
// Even ignoring changes in this setting, exclusions are interpreted as a
// guidance by Pebble, and not necessarily honored. Specifically, user
// keys with multiple Pebble-versions *may* have the older versions stored
// in value blocks.
RequiredInPlaceValueBound UserKeyPrefixBound
// DisableIngestAsFlushable disables lazy ingestion of sstables through
@@ -626,15 +680,49 @@ type Options struct {
// major version is at least `FormatFlushableIngest`.
DisableIngestAsFlushable func() bool
// SharedStorage is a second FS-like storage medium that can be shared
// between multiple Pebble instances. It is used to store sstables only, and
// is managed by objstorage.Provider. Each sstable might only be written to
// by one Pebble instance, but other Pebble instances can possibly read the
// same files if they have the path to get to them. The pebble instance that
// wrote a file should not delete it if other Pebble instances are known to
// be reading this file. This FS is expected to have slower read/write
// performance than the default FS above.
SharedStorage shared.Storage
// RemoteStorage enables use of remote storage (e.g. S3) for storing
// sstables. Setting this option enables use of CreateOnShared option and
// allows ingestion of external files.
RemoteStorage remote.StorageFactory
// If CreateOnShared is non-zero, new sstables are created on remote storage
// (using CreateOnSharedLocator and with the appropriate
// CreateOnSharedStrategy). These sstables can be shared between different
// Pebble instances; the lifecycle of such objects is managed by the
// remote.Storage constructed by options.RemoteStorage.
//
// Can only be used when RemoteStorage is set (and recognizes
// CreateOnSharedLocator).
CreateOnShared remote.CreateOnSharedStrategy
CreateOnSharedLocator remote.Locator
// CacheSizeBytesBytes is the size of the on-disk block cache for objects
// on shared storage in bytes. If it is 0, no cache is used.
SecondaryCacheSizeBytes int64
// IneffectualPointDeleteCallback is called in compactions/flushes if any
// single delete is being elided without deleting a point set/merge.
IneffectualSingleDeleteCallback func(userKey []byte)
// SingleDeleteInvariantViolationCallback is called in compactions/flushes if any
// single delete has consumed a Set/Merge, and there is another immediately older
// Set/SetWithDelete/Merge. The user of Pebble has violated the invariant under
// which SingleDelete can be used correctly.
//
// Consider the sequence SingleDelete#3, Set#2, Set#1. There are three
// ways some of these keys can first meet in a compaction.
//
// - All 3 keys in the same compaction: this callback will detect the
// violation.
//
// - SingleDelete#3, Set#2 meet in a compaction first: Both keys will
// disappear. The violation will not be detected, and the DB will have
// Set#1 which is likely incorrect (from the user's perspective).
//
// - Set#2, Set#1 meet in a compaction first: The output will be Set#2,
// which will later be consumed by SingleDelete#3. The violation will
// not be detected and the DB will be correct.
SingleDeleteInvariantViolationCallback func(userKey []byte)
}
// Filters is a map from filter policy name to filter policy. It is used for
@@ -749,12 +837,18 @@ type Options struct {
// writing the contents of the old one in the
// background. MemTableStopWritesThreshold places a hard limit on the size of
// the queued MemTables.
MemTableSize int
//
// The default value is 4MB.
MemTableSize uint64
// Hard limit on the size of queued of MemTables. Writes are stopped when the
// sum of the queued memtable sizes exceeds
// MemTableStopWritesThreshold*MemTableSize. This value should be at least 2
// or writes will stop whenever a MemTable is being flushed.
// Hard limit on the number of queued of MemTables. Writes are stopped when
// the sum of the queued memtable sizes exceeds:
// MemTableStopWritesThreshold * MemTableSize.
//
// This value should be at least 2 or writes will stop whenever a MemTable is
// being flushed.
//
// The default value is 2.
MemTableStopWritesThreshold int
// Merger defines the associative merge operation to use for merging values
@@ -839,6 +933,20 @@ type Options struct {
// changing options dynamically?
WALMinSyncInterval func() time.Duration
// TargetByteDeletionRate is the rate (in bytes per second) at which sstable file
// deletions are limited to (under normal circumstances).
//
// Deletion pacing is used to slow down deletions when compactions finish up
// or readers close and newly-obsolete files need cleaning up. Deleting lots
// of files at once can cause disk latency to go up on some SSDs, which this
// functionality guards against.
//
// This value is only a best-effort target; the effective rate can be
// higher if deletions are falling behind or disk space is running low.
//
// Setting this to 0 disables deletion pacing, which is also the default.
TargetByteDeletionRate int
// private options are only used by internal tests or are used internally
// for facilitating upgrade paths of unconfigurable functionality.
private struct {
@@ -872,6 +980,10 @@ type Options struct {
// A private option to disable stats collection.
disableTableStats bool
// testingAlwaysWaitForCleanup is set by some tests to force waiting for
// obsolete file deletion (to make events deterministic).
testingAlwaysWaitForCleanup bool
// fsCloser holds a closer that should be invoked after a DB using these
// Options is closed. This is used to automatically stop the
// long-running goroutine associated with the disk-health-checking FS.
@@ -983,7 +1095,7 @@ func (o *Options) EnsureDefaults() *Options {
o.MaxOpenFiles = 1000
}
if o.MemTableSize <= 0 {
o.MemTableSize = 4 << 20
o.MemTableSize = 4 << 20 // 4 MB
}
if o.MemTableStopWritesThreshold <= 0 {
o.MemTableStopWritesThreshold = 2
@@ -1024,8 +1136,8 @@ func (o *Options) EnsureDefaults() *Options {
if o.Experimental.CPUWorkPermissionGranter == nil {
o.Experimental.CPUWorkPermissionGranter = defaultCPUWorkGranter{}
}
if o.Experimental.MultiLevelCompactionHueristic == nil {
o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{}
if o.Experimental.MultiLevelCompactionHeuristic == nil {
o.Experimental.MultiLevelCompactionHeuristic = WriteAmpHeuristic{}
}
o.initMaps()
@@ -1144,7 +1256,7 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, " max_open_files=%d\n", o.MaxOpenFiles)
fmt.Fprintf(&buf, " mem_table_size=%d\n", o.MemTableSize)
fmt.Fprintf(&buf, " mem_table_stop_writes_threshold=%d\n", o.MemTableStopWritesThreshold)
fmt.Fprintf(&buf, " min_deletion_rate=%d\n", o.Experimental.MinDeletionRate)
fmt.Fprintf(&buf, " min_deletion_rate=%d\n", o.TargetByteDeletionRate)
fmt.Fprintf(&buf, " merger=%s\n", o.Merger.Name)
fmt.Fprintf(&buf, " read_compaction_rate=%d\n", o.Experimental.ReadCompactionRate)
fmt.Fprintf(&buf, " read_sampling_multiplier=%d\n", o.Experimental.ReadSamplingMultiplier)
@@ -1165,6 +1277,8 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, " wal_bytes_per_sync=%d\n", o.WALBytesPerSync)
fmt.Fprintf(&buf, " max_writer_concurrency=%d\n", o.Experimental.MaxWriterConcurrency)
fmt.Fprintf(&buf, " force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism)
fmt.Fprintf(&buf, " secondary_cache_size_bytes=%d\n", o.Experimental.SecondaryCacheSizeBytes)
fmt.Fprintf(&buf, " create_on_shared=%d\n", o.Experimental.CreateOnShared)
// Private options.
//
@@ -1322,7 +1436,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
}
}
case "compaction_debt_concurrency":
o.Experimental.CompactionDebtConcurrency, err = strconv.Atoi(value)
o.Experimental.CompactionDebtConcurrency, err = strconv.ParseUint(value, 10, 64)
case "delete_range_flush_delay":
// NB: This is a deprecated serialization of the
// `flush_delay_delete_range`.
@@ -1354,7 +1468,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
// version is valid right here.
var v uint64
v, err = strconv.ParseUint(value, 10, 64)
if vers := FormatMajorVersion(v); vers > FormatNewest || vers == FormatDefault {
if vers := FormatMajorVersion(v); vers > internalFormatNewest || vers == FormatDefault {
err = errors.Newf("unknown format major version %d", o.FormatMajorVersion)
}
if err == nil {
@@ -1387,14 +1501,14 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
case "max_open_files":
o.MaxOpenFiles, err = strconv.Atoi(value)
case "mem_table_size":
o.MemTableSize, err = strconv.Atoi(value)
o.MemTableSize, err = strconv.ParseUint(value, 10, 64)
case "mem_table_stop_writes_threshold":
o.MemTableStopWritesThreshold, err = strconv.Atoi(value)
case "min_compaction_rate":
// Do nothing; option existed in older versions of pebble, and
// may be meaningful again eventually.
case "min_deletion_rate":
o.Experimental.MinDeletionRate, err = strconv.Atoi(value)
o.TargetByteDeletionRate, err = strconv.Atoi(value)
case "min_flush_rate":
// Do nothing; option existed in older versions of pebble, and
// may be meaningful again eventually.
@@ -1438,6 +1552,12 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
o.Experimental.MaxWriterConcurrency, err = strconv.Atoi(value)
case "force_writer_parallelism":
o.Experimental.ForceWriterParallelism, err = strconv.ParseBool(value)
case "secondary_cache_size_bytes":
o.Experimental.SecondaryCacheSizeBytes, err = strconv.ParseInt(value, 10, 64)
case "create_on_shared":
var createOnSharedInt int64
createOnSharedInt, err = strconv.ParseInt(value, 10, 64)
o.Experimental.CreateOnShared = remote.CreateOnSharedStrategy(createOnSharedInt)
default:
if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
return nil
@@ -1568,15 +1688,15 @@ func (o *Options) Validate() error {
}
if uint64(o.MemTableSize) >= maxMemTableSize {
fmt.Fprintf(&buf, "MemTableSize (%s) must be < %s\n",
humanize.Uint64(uint64(o.MemTableSize)), humanize.Uint64(maxMemTableSize))
humanize.Bytes.Uint64(uint64(o.MemTableSize)), humanize.Bytes.Uint64(maxMemTableSize))
}
if o.MemTableStopWritesThreshold < 2 {
fmt.Fprintf(&buf, "MemTableStopWritesThreshold (%d) must be >= 2\n",
o.MemTableStopWritesThreshold)
}
if o.FormatMajorVersion > FormatNewest {
if o.FormatMajorVersion > internalFormatNewest {
fmt.Fprintf(&buf, "FormatMajorVersion (%d) must be <= %d\n",
o.FormatMajorVersion, FormatNewest)
o.FormatMajorVersion, internalFormatNewest)
}
if o.TableCache != nil && o.Cache != o.TableCache.cache {
fmt.Fprintf(&buf, "underlying cache in the TableCache and the Cache dont match\n")
@@ -1596,6 +1716,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions {
readerOpts.Comparer = o.Comparer
readerOpts.Filters = o.Filters
if o.Merger != nil {
readerOpts.Merge = o.Merger.Merge
readerOpts.MergerName = o.Merger.Name
}
readerOpts.LoggerAndTracer = o.LoggerAndTracer
@@ -1620,6 +1741,9 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab
if format >= sstable.TableFormatPebblev3 {
writerOpts.ShortAttributeExtractor = o.Experimental.ShortAttributeExtractor
writerOpts.RequiredInPlaceValueBound = o.Experimental.RequiredInPlaceValueBound
if format >= sstable.TableFormatPebblev4 && level == numLevels-1 {
writerOpts.WritingToLowestLevel = true
}
}
levelOpts := o.Level(level)
writerOpts.BlockRestartInterval = levelOpts.BlockRestartInterval