aleesa-bot/aleesa-phrases-go
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgrade versions of vendored libs

Browse Source
This commit is contained in:
eleksir
2022-11-04 16:29:14 +03:00
parent fba732cc24
commit a7ae372459
941 changed files with 219674 additions and 23611 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
vendor/github.com/cockroachdb/pebble/compaction_picker.go generated vendored
View File

@@ -19,10 +19,7 @@ import (
// heuristic.
const minIntraL0Count = 4
const levelMultiplier = 10
type compactionEnv struct {
bytesCompacted *uint64
earliestUnflushedSeqNum uint64
earliestSnapshotSeqNum uint64
inProgressCompactions []compactionInfo
@@ -146,10 +143,18 @@ type pickedCompaction struct {
// startLevel is the level that is being compacted. Inputs from startLevel
// and outputLevel will be merged to produce a set of outputLevel files.
startLevel *compactionLevel
// outputLevel is the level that files are being produced in. In default
// compactions, outputLevel is equal to startLevel+1 except when startLevel
// is 0 in which case it is equal to compactionPicker.baseLevel().
// outputLevel is the level that files are being produced in. outputLevel is
// equal to startLevel+1 except when:
// - if startLevel is 0, the output level equals compactionPicker.baseLevel().
// - in multilevel compaction, the output level is the lowest level involved in
// the compaction
outputLevel *compactionLevel
// extraLevels contain additional levels in between the input and output
// levels that get compacted in multi level compactions
extraLevels []*compactionLevel
// adjustedOutputLevel is the output level used for the purpose of
// determining the target output file size, overlap bytes, and expanded
// bytes, taking into account the base level.
@@ -161,7 +166,7 @@ type pickedCompaction struct {
// where startLevel == 0 that were generated by L0Sublevels.
lcf *manifest.L0CompactionFiles
// L0 sublevel info is used for compactions out of L0. It is nil for all
// L0SublevelInfo is used for compactions out of L0. It is nil for all
// other compactions.
l0SublevelInfo []sublevelInfo
@@ -252,37 +257,71 @@ func newPickedCompactionFromL0(
return pc
}
func (pc *pickedCompaction) setupInputs(opts *Options, diskAvailBytes uint64) bool {
// maybeExpandedBounds is a helper function for setupInputs which ensures the
// pickedCompaction's smallest and largest internal keys are updated iff
// the candidate keys expand the key span. This avoids a bug for multi-level
// compactions: during the second call to setupInputs, the picked compaction's
// smallest and largest keys should not decrease the key span.
func (pc *pickedCompaction) maybeExpandBounds(smallest InternalKey, largest InternalKey) {
emptyKey := InternalKey{}
if base.InternalCompare(pc.cmp, smallest, emptyKey) == 0 {
if base.InternalCompare(pc.cmp, largest, emptyKey) != 0 {
panic("either both candidate keys are empty or neither are empty")
}
return
}
if base.InternalCompare(pc.cmp, pc.smallest, emptyKey) == 0 {
if base.InternalCompare(pc.cmp, pc.largest, emptyKey) != 0 {
panic("either both pc keys are empty or neither are empty")
}
pc.smallest = smallest
pc.largest = largest
return
}
if base.InternalCompare(pc.cmp, pc.smallest, smallest) >= 0 {
pc.smallest = smallest
}
if base.InternalCompare(pc.cmp, pc.largest, largest) <= 0 {
pc.largest = largest
}
}
func (pc *pickedCompaction) setupInputs(
opts *Options, diskAvailBytes uint64, startLevel *compactionLevel,
) bool {
// maxExpandedBytes is the maximum size of an expanded compaction. If
// growing a compaction results in a larger size, the original compaction
// is used instead.
maxExpandedBytes := expandedCompactionByteSizeLimit(opts, pc.adjustedOutputLevel, diskAvailBytes)
maxExpandedBytes := expandedCompactionByteSizeLimit(
opts, pc.adjustedOutputLevel, diskAvailBytes,
)
// Expand the initial inputs to a clean cut.
var isCompacting bool
pc.startLevel.files, isCompacting = expandToAtomicUnit(pc.cmp, pc.startLevel.files, false /* disableIsCompacting */)
startLevel.files, isCompacting = expandToAtomicUnit(pc.cmp, startLevel.files, false /* disableIsCompacting */)
if isCompacting {
return false
}
pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter())
pc.maybeExpandBounds(manifest.KeyRange(pc.cmp, startLevel.files.Iter()))
// Determine the sstables in the output level which overlap with the input
// sstables, and then expand those tables to a clean cut. No need to do
// this for intra-L0 compactions; outputLevel.files is left empty for those.
if pc.startLevel.level != pc.outputLevel.level {
if startLevel.level != pc.outputLevel.level {
pc.outputLevel.files = pc.version.Overlaps(pc.outputLevel.level, pc.cmp, pc.smallest.UserKey,
pc.largest.UserKey, pc.largest.IsExclusiveSentinel())
pc.outputLevel.files, isCompacting = expandToAtomicUnit(pc.cmp, pc.outputLevel.files, false /* disableIsCompacting */)
pc.outputLevel.files, isCompacting = expandToAtomicUnit(pc.cmp, pc.outputLevel.files,
false /* disableIsCompacting */)
if isCompacting {
return false
}
pc.smallest, pc.largest = manifest.KeyRange(pc.cmp,
pc.startLevel.files.Iter(), pc.outputLevel.files.Iter())
pc.maybeExpandBounds(manifest.KeyRange(pc.cmp,
startLevel.files.Iter(), pc.outputLevel.files.Iter()))
}
// Grow the sstables in pc.startLevel.level as long as it doesn't affect the number
// Grow the sstables in startLevel.level as long as it doesn't affect the number
// of sstables included from pc.outputLevel.level.
if pc.lcf != nil && pc.startLevel.level == 0 && pc.outputLevel.level != 0 {
if pc.lcf != nil && startLevel.level == 0 && pc.outputLevel.level != 0 {
// Call the L0-specific compaction extension method. Similar logic as
// pc.grow. Additional L0 files are optionally added to the compaction at
// this step. Note that the bounds passed in are not the bounds of the
@@ -336,16 +375,16 @@ func (pc *pickedCompaction) setupInputs(opts *Options, diskAvailBytes uint64) bo
}
}
if sizeSum+pc.outputLevel.files.SizeSum() < maxExpandedBytes {
pc.startLevel.files = manifest.NewLevelSliceSeqSorted(newStartLevelFiles)
startLevel.files = manifest.NewLevelSliceSeqSorted(newStartLevelFiles)
pc.smallest, pc.largest = manifest.KeyRange(pc.cmp,
pc.startLevel.files.Iter(), pc.outputLevel.files.Iter())
startLevel.files.Iter(), pc.outputLevel.files.Iter())
} else {
*pc.lcf = oldLcf
}
}
} else if pc.grow(pc.smallest, pc.largest, maxExpandedBytes) {
pc.smallest, pc.largest = manifest.KeyRange(pc.cmp,
pc.startLevel.files.Iter(), pc.outputLevel.files.Iter())
} else if pc.grow(pc.smallest, pc.largest, maxExpandedBytes, startLevel) {
pc.maybeExpandBounds(manifest.KeyRange(pc.cmp,
startLevel.files.Iter(), pc.outputLevel.files.Iter()))
}
if pc.startLevel.level == 0 {
@@ -359,23 +398,27 @@ func (pc *pickedCompaction) setupInputs(opts *Options, diskAvailBytes uint64) bo
// grow grows the number of inputs at c.level without changing the number of
// c.level+1 files in the compaction, and returns whether the inputs grew. sm
// and la are the smallest and largest InternalKeys in all of the inputs.
func (pc *pickedCompaction) grow(sm, la InternalKey, maxExpandedBytes uint64) bool {
func (pc *pickedCompaction) grow(
sm, la InternalKey, maxExpandedBytes uint64, startLevel *compactionLevel,
) bool {
if pc.outputLevel.files.Empty() {
return false
}
grow0 := pc.version.Overlaps(pc.startLevel.level, pc.cmp, sm.UserKey,
grow0 := pc.version.Overlaps(startLevel.level, pc.cmp, sm.UserKey,
la.UserKey, la.IsExclusiveSentinel())
grow0, isCompacting := expandToAtomicUnit(pc.cmp, grow0, false /* disableIsCompacting */)
if isCompacting {
return false
}
if grow0.Len() <= pc.startLevel.files.Len() {
if grow0.Len() <= startLevel.files.Len() {
return false
}
if grow0.SizeSum()+pc.outputLevel.files.SizeSum() >= maxExpandedBytes {
return false
}
sm1, la1 := manifest.KeyRange(pc.cmp, grow0.Iter())
// We need to include the outputLevel iter because without it, in a multiLevel scenario,
// sm1 and la1 could shift the output level keyspace when pc.outputLevel.files is set to grow1.
sm1, la1 := manifest.KeyRange(pc.cmp, grow0.Iter(), pc.outputLevel.files.Iter())
grow1 := pc.version.Overlaps(pc.outputLevel.level, pc.cmp, sm1.UserKey,
la1.UserKey, la1.IsExclusiveSentinel())
grow1, isCompacting = expandToAtomicUnit(pc.cmp, grow1, false /* disableIsCompacting */)
@@ -385,11 +428,19 @@ func (pc *pickedCompaction) grow(sm, la InternalKey, maxExpandedBytes uint64) bo
if grow1.Len() != pc.outputLevel.files.Len() {
return false
}
pc.startLevel.files = grow0
startLevel.files = grow0
pc.outputLevel.files = grow1
return true
}
// initMultiLevelCompaction returns true if it initiated a multilevel input
// compaction. This currently never inits a multiLevel compaction.
func (pc *pickedCompaction) initMultiLevelCompaction(
opts *Options, vers *version, levelMaxBytes [7]int64, diskAvailBytes uint64,
) bool {
return false
}
// expandToAtomicUnit expands the provided level slice within its level both
// forwards and backwards to its "atomic compaction unit" boundaries, if
// necessary.
@@ -453,7 +504,7 @@ func expandToAtomicUnit(
iter := start.Clone()
iter.Prev()
for cur, prev := start.Current(), iter.Current(); prev != nil; cur, prev = start.Prev(), iter.Prev() {
if cur.Compacting {
if cur.IsCompacting() {
isCompacting = true
}
if cmp(prev.Largest.UserKey, cur.Smallest.UserKey) < 0 {
@@ -471,7 +522,7 @@ func expandToAtomicUnit(
iter = end.Clone()
iter.Next()
for cur, next := end.Current(), iter.Current(); next != nil; cur, next = end.Next(), iter.Next() {
if cur.Compacting {
if cur.IsCompacting() {
isCompacting = true
}
if cmp(cur.Largest.UserKey, next.Smallest.UserKey) < 0 {
@@ -487,7 +538,8 @@ func expandToAtomicUnit(
}
})
inputIter := inputs.Iter()
isCompacting = !disableIsCompacting && (isCompacting || inputIter.First().Compacting || inputIter.Last().Compacting)
isCompacting = !disableIsCompacting &&
(isCompacting || inputIter.First().IsCompacting() || inputIter.Last().IsCompacting())
return inputs, isCompacting
}
@@ -525,11 +577,11 @@ type candidateLevelInfo struct {
// compensatedSize returns f's file size, inflated according to compaction
// priorities.
func compensatedSize(f *fileMetadata) uint64 {
func compensatedSize(f *fileMetadata, pointTombstoneWeight float64) uint64 {
sz := f.Size
// Add in the estimate of disk space that may be reclaimed by compacting
// the file's tombstones.
sz += f.Stats.PointDeletionsBytesEstimate
sz += uint64(float64(f.Stats.PointDeletionsBytesEstimate) * pointTombstoneWeight)
sz += f.Stats.RangeDeletionsBytesEstimate
return sz
}
@@ -539,7 +591,9 @@ func compensatedSize(f *fileMetadata) uint64 {
// a *uint64. Compensated sizes may change once a table's stats are loaded
// asynchronously, so its values are marked as cacheable only if a file's
// stats have been loaded.
type compensatedSizeAnnotator struct{}
type compensatedSizeAnnotator struct {
pointTombstoneWeight float64
}
var _ manifest.Annotator = compensatedSizeAnnotator{}
@@ -556,8 +610,8 @@ func (a compensatedSizeAnnotator) Accumulate(
f *fileMetadata, dst interface{},
) (v interface{}, cacheOK bool) {
vptr := dst.(*uint64)
*vptr = *vptr + compensatedSize(f)
return vptr, f.Stats.Valid
*vptr = *vptr + compensatedSize(f, a.pointTombstoneWeight)
return vptr, f.StatsValidLocked()
}
func (a compensatedSizeAnnotator) Merge(src interface{}, dst interface{}) interface{} {
@@ -571,10 +625,10 @@ func (a compensatedSizeAnnotator) Merge(src interface{}, dst interface{}) interf
// iterator. Note that this function is linear in the files available to the
// iterator. Use the compensatedSizeAnnotator if querying the total
// compensated size of a level.
func totalCompensatedSize(iter manifest.LevelIterator) uint64 {
func totalCompensatedSize(iter manifest.LevelIterator, pointTombstoneWeight float64) uint64 {
var sz uint64
for f := iter.First(); f != nil; f = iter.Next() {
sz += compensatedSize(f)
sz += compensatedSize(f, pointTombstoneWeight)
}
return sz
}
@@ -732,11 +786,11 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp
}
dbSize += p.levelSizes[0]
bottomLevelSize := dbSize - dbSize/levelMultiplier
bottomLevelSize := dbSize - dbSize/int64(p.opts.Experimental.LevelMultiplier)
curLevelSize := bottomLevelSize
for level := numLevels - 2; level >= firstNonEmptyLevel; level-- {
curLevelSize = int64(float64(curLevelSize) / levelMultiplier)
curLevelSize = int64(float64(curLevelSize) / float64(p.opts.Experimental.LevelMultiplier))
}
// Compute base level (where L0 data is compacted to).
@@ -744,7 +798,7 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp
p.baseLevel = firstNonEmptyLevel
for p.baseLevel > 1 && curLevelSize > baseBytesMax {
p.baseLevel--
curLevelSize = int64(float64(curLevelSize) / levelMultiplier)
curLevelSize = int64(float64(curLevelSize) / float64(p.opts.Experimental.LevelMultiplier))
}
smoothedLevelMultiplier := 1.0
@@ -772,7 +826,9 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp
}
}
func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]int64 {
func calculateSizeAdjust(
inProgressCompactions []compactionInfo, pointTombstoneWeight float64,
) [numLevels]int64 {
// Compute a size adjustment for each level based on the in-progress
// compactions. We subtract the compensated size of start level inputs.
// Since compensated file sizes may be compensated because they reclaim
@@ -785,7 +841,7 @@ func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]int6
for _, input := range c.inputs {
real := int64(input.files.SizeSum())
compensated := int64(totalCompensatedSize(input.files.Iter()))
compensated := int64(totalCompensatedSize(input.files.Iter(), pointTombstoneWeight))
if input.level != c.outputLevel {
sizeAdjust[input.level] -= compensated
@@ -812,7 +868,10 @@ func (p *compactionPickerByScore) calculateScores(
}
scores[0] = p.calculateL0Score(inProgressCompactions)
sizeAdjust := calculateSizeAdjust(inProgressCompactions)
sizeAdjust := calculateSizeAdjust(
inProgressCompactions,
p.opts.Experimental.PointTombstoneWeight,
)
for level := 1; level < numLevels; level++ {
levelSize := int64(levelCompensatedSize(p.vers.Levels[level])) + sizeAdjust[level]
scores[level].score = float64(levelSize) / float64(p.levelMaxBytes[level])
@@ -936,10 +995,10 @@ func (p *compactionPickerByScore) pickFile(
outputFile = outputIter.Next()
}
compacting := f.Compacting
compacting := f.IsCompacting()
for outputFile != nil && base.InternalCompare(cmp, outputFile.Smallest, f.Largest) < 0 {
overlappingBytes += outputFile.Size
compacting = compacting || outputFile.Compacting
compacting = compacting || outputFile.IsCompacting()
// For files in the bottommost level of the LSM, the
// Stats.RangeDeletionsBytesEstimate field is set to the estimate
@@ -969,8 +1028,9 @@ func (p *compactionPickerByScore) pickFile(
continue
}
scaledRatio := overlappingBytes * 1024 / compensatedSize(f)
if scaledRatio < smallestRatio && !f.Compacting {
compSz := compensatedSize(f, p.opts.Experimental.PointTombstoneWeight)
scaledRatio := overlappingBytes * 1024 / compSz
if scaledRatio < smallestRatio && !f.IsCompacting() {
smallestRatio = scaledRatio
file = startIter.Take()
}
@@ -1031,7 +1091,9 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
}
fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %8s %8s",
marker, info.level, info.score, info.origScore,
humanize.Int64(int64(totalCompensatedSize(p.vers.Levels[info.level].Iter()))),
humanize.Int64(int64(totalCompensatedSize(
p.vers.Levels[info.level].Iter(), p.opts.Experimental.PointTombstoneWeight,
))),
humanize.Int64(p.levelMaxBytes[info.level]),
)
@@ -1092,7 +1154,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
continue
}
pc := pickAutoLPositive(env, p.opts, p.vers, *info, p.baseLevel, p.diskAvailBytes)
pc := pickAutoLPositive(env, p.opts, p.vers, *info, p.baseLevel, p.diskAvailBytes, p.levelMaxBytes)
// Fail-safe to protect against compacting the same sstable concurrently.
if pc != nil && !inputRangeAlreadyCompacting(env, pc) {
pc.score = info.score
@@ -1172,10 +1234,10 @@ func (a elisionOnlyAnnotator) Zero(interface{}) interface{} {
}
func (a elisionOnlyAnnotator) Accumulate(f *fileMetadata, dst interface{}) (interface{}, bool) {
if f.Compacting {
if f.IsCompacting() {
return dst, true
}
if !f.Stats.Valid {
if !f.StatsValidLocked() {
return dst, false
}
// Bottommost files are large and not worthwhile to compact just
@@ -1187,8 +1249,13 @@ func (a elisionOnlyAnnotator) Accumulate(f *fileMetadata, dst interface{}) (inte
// which may be collapsed. Ideally, we would have 'obsolete keys'
// statistics that would include tombstones, the keys that are
// dropped by tombstones and duplicated user keys. See #847.
//
// Note that tables that contain exclusively range keys (i.e. no point keys,
// `NumEntries` and `RangeDeletionsBytesEstimate` are both zero) are excluded
// from elision-only compactions.
// TODO(travers): Consider an alternative heuristic for elision of range-keys.
if f.Stats.RangeDeletionsBytesEstimate*10 < f.Size &&
f.Stats.NumDeletions*10 < f.Stats.NumEntries {
f.Stats.NumDeletions*10 <= f.Stats.NumEntries {
return dst, true
}
if dst == nil {
@@ -1266,7 +1333,7 @@ func (p *compactionPickerByScore) pickElisionOnlyCompaction(
return nil
}
candidate := v.(*fileMetadata)
if candidate.Compacting || candidate.LargestSeqNum >= env.earliestSnapshotSeqNum {
if candidate.IsCompacting() || candidate.LargestSeqNum >= env.earliestSnapshotSeqNum {
return nil
}
lf := p.vers.Levels[numLevels-1].Find(p.opts.Comparer.Compare, candidate)
@@ -1304,7 +1371,7 @@ func (p *compactionPickerByScore) pickRewriteCompaction(env compactionEnv) (pc *
continue
}
candidate := v.(*fileMetadata)
if candidate.Compacting {
if candidate.IsCompacting() {
// Try the next level.
continue
}
@@ -1364,6 +1431,7 @@ func pickAutoLPositive(
cInfo candidateLevelInfo,
baseLevel int,
diskAvailBytes func() uint64,
levelMaxBytes [7]int64,
) (pc *pickedCompaction) {
if cInfo.level == 0 {
panic("pebble: pickAutoLPositive called for L0")
@@ -1385,9 +1453,15 @@ func pickAutoLPositive(
}
}
if !pc.setupInputs(opts, diskAvailBytes()) {
if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) {
return nil
}
if opts.Experimental.MultiLevelCompaction &&
pc.initMultiLevelCompaction(opts, vers, levelMaxBytes, diskAvailBytes()) {
if !pc.setupInputs(opts, diskAvailBytes(), pc.extraLevels[len(pc.extraLevels)-1]) {
return nil
}
}
return pc
}
@@ -1410,7 +1484,7 @@ func pickL0(
}
if lcf != nil {
pc = newPickedCompactionFromL0(lcf, opts, vers, baseLevel, true)
pc.setupInputs(opts, diskAvailBytes())
pc.setupInputs(opts, diskAvailBytes(), pc.startLevel)
if pc.startLevel.files.Empty() {
opts.Logger.Fatalf("empty compaction chosen")
}
@@ -1428,7 +1502,7 @@ func pickL0(
}
if lcf != nil {
pc = newPickedCompactionFromL0(lcf, opts, vers, 0, false)
if !pc.setupInputs(opts, diskAvailBytes()) {
if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) {
return nil
}
if pc.startLevel.files.Empty() {
@@ -1474,12 +1548,16 @@ func (p *compactionPickerByScore) pickManual(
if conflictsWithInProgress(manual, outputLevel, env.inProgressCompactions, p.opts.Comparer.Compare) {
return nil, true
}
pc = pickManualHelper(p.opts, manual, p.vers, p.baseLevel, p.diskAvailBytes)
pc = pickManualHelper(p.opts, manual, p.vers, p.baseLevel, p.diskAvailBytes, p.levelMaxBytes)
if pc == nil {
return nil, false
}
if pc.outputLevel.level != outputLevel {
panic("pebble: compaction picked unexpected output level")
if len(pc.extraLevels) > 0 {
// multilevel compactions relax this invariant
} else {
panic("pebble: compaction picked unexpected output level")
}
}
// Fail-safe to protect against compacting the same sstable concurrently.
if inputRangeAlreadyCompacting(env, pc) {
@@ -1494,6 +1572,7 @@ func pickManualHelper(
vers *version,
baseLevel int,
diskAvailBytes func() uint64,
levelMaxBytes [7]int64,
) (pc *pickedCompaction) {
pc = newPickedCompaction(opts, vers, manual.level, defaultOutputLevel(manual.level, baseLevel), baseLevel)
manual.outputLevel = pc.outputLevel.level
@@ -1503,9 +1582,15 @@ func pickManualHelper(
// Nothing to do
return nil
}
if !pc.setupInputs(opts, diskAvailBytes()) {
if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) {
return nil
}
if opts.Experimental.MultiLevelCompaction && pc.startLevel.level > 0 &&
pc.initMultiLevelCompaction(opts, vers, levelMaxBytes, diskAvailBytes()) {
if !pc.setupInputs(opts, diskAvailBytes(), pc.extraLevels[len(pc.extraLevels)-1]) {
return nil
}
}
return pc
}
@@ -1554,7 +1639,7 @@ func pickReadTriggeredCompactionHelper(
pc = newPickedCompaction(p.opts, p.vers, rc.level, defaultOutputLevel(rc.level, p.baseLevel), p.baseLevel)
pc.startLevel.files = overlapSlice
if !pc.setupInputs(p.opts, p.diskAvailBytes()) {
if !pc.setupInputs(p.opts, p.diskAvailBytes(), pc.startLevel) {
return nil
}
if inputRangeAlreadyCompacting(env, pc) {
@@ -1588,7 +1673,7 @@ func inputRangeAlreadyCompacting(env compactionEnv, pc *pickedCompaction) bool {
for _, cl := range pc.inputs {
iter := cl.files.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Compacting {
if f.IsCompacting() {
return true
}
}