aleesa-bot/aleesa-jabber-go
0
Fork 0
Code Activity

Update deps

Browse Source
This commit is contained in:
eleksir
2024-04-19 21:30:16 +03:00
parent 3b6dd3fdef
commit 442142291c
585 changed files with 21099 additions and 15328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

378
vendor/github.com/cockroachdb/pebble/compaction_picker.go generated vendored
View File

@@ -90,18 +90,38 @@ func (info compactionInfo) String() string {
return buf.String()
}
type sortCompactionLevelsDecreasingScore []candidateLevelInfo
type sortCompactionLevelsByPriority []candidateLevelInfo
func (s sortCompactionLevelsDecreasingScore) Len() int {
func (s sortCompactionLevelsByPriority) Len() int {
return len(s)
}
func (s sortCompactionLevelsDecreasingScore) Less(i, j int) bool {
if s[i].score != s[j].score {
return s[i].score > s[j].score
// A level should be picked for compaction if the compensatedScoreRatio is >= the
// compactionScoreThreshold.
const compactionScoreThreshold = 1
// Less should return true if s[i] must be placed earlier than s[j] in the final
// sorted list. The candidateLevelInfo for the level placed earlier is more likely
// to be picked for a compaction.
func (s sortCompactionLevelsByPriority) Less(i, j int) bool {
iShouldCompact := s[i].compensatedScoreRatio >= compactionScoreThreshold
jShouldCompact := s[j].compensatedScoreRatio >= compactionScoreThreshold
// Ordering is defined as decreasing on (shouldCompact, uncompensatedScoreRatio)
// where shouldCompact is 1 for true and 0 for false.
if iShouldCompact && !jShouldCompact {
return true
}
if !iShouldCompact && jShouldCompact {
return false
}
if s[i].uncompensatedScoreRatio != s[j].uncompensatedScoreRatio {
return s[i].uncompensatedScoreRatio > s[j].uncompensatedScoreRatio
}
return s[i].level < s[j].level
}
func (s sortCompactionLevelsDecreasingScore) Swap(i, j int) {
func (s sortCompactionLevelsByPriority) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
@@ -151,13 +171,22 @@ func generateSublevelInfo(cmp base.Compare, levelFiles manifest.LevelSlice) []su
return levelSlices
}
// compactionPickerMetrics holds metrics related to the compaction picking process
type compactionPickerMetrics struct {
// scores contains the compensatedScoreRatio from the candidateLevelInfo.
scores []float64
singleLevelOverlappingRatio float64
multiLevelOverlappingRatio float64
}
// pickedCompaction contains information about a compaction that has already
// been chosen, and is being constructed. Compaction construction info lives in
// this struct, and is copied over into the compaction struct when that's
// created.
type pickedCompaction struct {
cmp Compare
// score of the chosen compaction. Taken from candidateLevelInfo.
// score of the chosen compaction. This is the same as the
// compensatedScoreRatio in the candidateLevelInfo.
score float64
// kind indicates the kind of compaction.
kind compactionKind
@@ -173,17 +202,12 @@ type pickedCompaction struct {
// extraLevels contain additional levels in between the input and output
// levels that get compacted in multi level compactions
extraLevels []*compactionLevel
// adjustedOutputLevel is the output level used for the purpose of
// determining the target output file size, overlap bytes, and expanded
// bytes, taking into account the base level.
adjustedOutputLevel int
inputs []compactionLevel
inputs []compactionLevel
// LBase at the time of compaction picking.
baseLevel int
// L0-specific compaction info. Set to a non-nil value for all compactions
// where startLevel == 0 that were generated by L0Sublevels.
lcf *manifest.L0CompactionFiles
// L0SublevelInfo is used for compactions out of L0. It is nil for all
// other compactions.
l0SublevelInfo []sublevelInfo
// maxOutputFileSize is the maximum size of an individual table created
// during compaction.
maxOutputFileSize uint64
@@ -195,9 +219,10 @@ type pickedCompaction struct {
// maxReadCompaction bytes, then we don't proceed with the compaction.
maxReadCompactionBytes uint64
// The boundaries of the input data.
smallest InternalKey
largest InternalKey
version *version
smallest InternalKey
largest InternalKey
version *version
pickerMetrics compactionPickerMetrics
}
func defaultOutputLevel(startLevel, baseLevel int) int {
@@ -219,28 +244,35 @@ func newPickedCompaction(
startLevel, baseLevel))
}
adjustedOutputLevel := outputLevel
if adjustedOutputLevel > 0 {
// Output level is in the range [baseLevel,numLevels]. For the purpose of
// determining the target output file size, overlap bytes, and expanded
// bytes, we want to adjust the range to [1,numLevels].
adjustedOutputLevel = 1 + outputLevel - baseLevel
}
adjustedLevel := adjustedOutputLevel(outputLevel, baseLevel)
pc := &pickedCompaction{
cmp: opts.Comparer.Compare,
version: cur,
baseLevel: baseLevel,
inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}},
adjustedOutputLevel: adjustedOutputLevel,
maxOutputFileSize: uint64(opts.Level(adjustedOutputLevel).TargetFileSize),
maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedOutputLevel),
maxReadCompactionBytes: maxReadCompactionBytes(opts, adjustedOutputLevel),
maxOutputFileSize: uint64(opts.Level(adjustedLevel).TargetFileSize),
maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedLevel),
maxReadCompactionBytes: maxReadCompactionBytes(opts, adjustedLevel),
}
pc.startLevel = &pc.inputs[0]
pc.outputLevel = &pc.inputs[1]
return pc
}
// adjustedOutputLevel is the output level used for the purpose of
// determining the target output file size, overlap bytes, and expanded
// bytes, taking into account the base level.
func adjustedOutputLevel(outputLevel int, baseLevel int) int {
adjustedOutputLevel := outputLevel
if adjustedOutputLevel > 0 {
// Output level is in the range [baseLevel, numLevels]. For the purpose of
// determining the target output file size, overlap bytes, and expanded
// bytes, we want to adjust the range to [1,numLevels].
adjustedOutputLevel = 1 + outputLevel - baseLevel
}
return adjustedOutputLevel
}
func newPickedCompactionFromL0(
lcf *manifest.L0CompactionFiles, opts *Options, vers *version, baseLevel int, isBase bool,
) *pickedCompaction {
@@ -273,7 +305,7 @@ func (pc *pickedCompaction) String() string {
var builder strings.Builder
builder.WriteString(fmt.Sprintf(`Score=%f, `, pc.score))
builder.WriteString(fmt.Sprintf(`Kind=%s, `, pc.kind))
builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, pc.adjustedOutputLevel))
builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel)))
builder.WriteString(fmt.Sprintf(`maxOutputFileSize=%d, `, pc.maxOutputFileSize))
builder.WriteString(fmt.Sprintf(`maxReadCompactionBytes=%d, `, pc.maxReadCompactionBytes))
builder.WriteString(fmt.Sprintf(`smallest=%s, `, pc.smallest))
@@ -283,7 +315,7 @@ func (pc *pickedCompaction) String() string {
builder.WriteString(fmt.Sprintf(`startlevel=%s, `, pc.startLevel))
builder.WriteString(fmt.Sprintf(`outputLevel=%s, `, pc.outputLevel))
builder.WriteString(fmt.Sprintf(`extraLevels=%s, `, pc.extraLevels))
builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.l0SublevelInfo))
builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.startLevel.l0SublevelInfo))
builder.WriteString(fmt.Sprintf(`lcf=%s`, pc.lcf))
return builder.String()
}
@@ -297,13 +329,16 @@ func (pc *pickedCompaction) clone() *pickedCompaction {
cmp: pc.cmp,
score: pc.score,
kind: pc.kind,
adjustedOutputLevel: pc.adjustedOutputLevel,
baseLevel: pc.baseLevel,
maxOutputFileSize: pc.maxOutputFileSize,
maxOverlapBytes: pc.maxOverlapBytes,
maxReadCompactionBytes: pc.maxReadCompactionBytes,
smallest: pc.smallest.Clone(),
largest: pc.largest.Clone(),
// TODO(msbutler): properly clone picker metrics
pickerMetrics: pc.pickerMetrics,
// Both copies see the same manifest, therefore, it's ok for them to se
// share the same pc. version.
version: pc.version,
@@ -322,9 +357,11 @@ func (pc *pickedCompaction) clone() *pickedCompaction {
}
}
newPC.l0SublevelInfo = make([]sublevelInfo, len(pc.l0SublevelInfo))
for i := range pc.l0SublevelInfo {
newPC.l0SublevelInfo[i] = pc.l0SublevelInfo[i].Clone()
if len(pc.startLevel.l0SublevelInfo) > 0 {
newPC.startLevel.l0SublevelInfo = make([]sublevelInfo, len(pc.startLevel.l0SublevelInfo))
for i := range pc.startLevel.l0SublevelInfo {
newPC.startLevel.l0SublevelInfo[i] = pc.startLevel.l0SublevelInfo[i].Clone()
}
}
if pc.lcf != nil {
newPC.lcf = pc.lcf.Clone()
@@ -370,7 +407,7 @@ func (pc *pickedCompaction) setupInputs(
// growing a compaction results in a larger size, the original compaction
// is used instead.
maxExpandedBytes := expandedCompactionByteSizeLimit(
opts, pc.adjustedOutputLevel, diskAvailBytes,
opts, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel), diskAvailBytes,
)
// Expand the initial inputs to a clean cut.
@@ -465,7 +502,7 @@ func (pc *pickedCompaction) setupInputs(
if pc.startLevel.level == 0 {
// We don't change the input files for the compaction beyond this point.
pc.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
pc.startLevel.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
}
return true
@@ -528,8 +565,6 @@ func (pc *pickedCompaction) setupMultiLevelCandidate(opts *Options, diskAvailByt
pc.startLevel = &pc.inputs[0]
pc.extraLevels = []*compactionLevel{&pc.inputs[1]}
pc.outputLevel = &pc.inputs[2]
pc.adjustedOutputLevel++
return pc.setupInputs(opts, diskAvailBytes, pc.extraLevels[len(pc.extraLevels)-1])
}
@@ -653,16 +688,23 @@ func newCompactionPicker(
// Information about a candidate compaction level that has been identified by
// the compaction picker.
type candidateLevelInfo struct {
// The score of the level to be compacted, with compensated file sizes and
// adjustments.
score float64
// The original score of the level to be compacted, before adjusting
// according to other levels' sizes.
origScore float64
// The raw score of the level to be compacted, calculated using
// uncompensated file sizes and without any adjustments.
rawScore float64
level int
// The compensatedScore of the level after adjusting according to the other
// levels' sizes. For L0, the compensatedScoreRatio is equivalent to the
// uncompensatedScoreRatio as we don't account for level size compensation in
// L0.
compensatedScoreRatio float64
// The score of the level after accounting for level size compensation before
// adjusting according to other levels' sizes. For L0, the compensatedScore
// is equivalent to the uncompensatedScore as we don't account for level
// size compensation in L0.
compensatedScore float64
// The score of the level to be compacted, calculated using uncompensated file
// sizes and without any adjustments.
uncompensatedScore float64
// uncompensatedScoreRatio is the uncompensatedScore adjusted according to
// the other levels' sizes.
uncompensatedScoreRatio float64
level int
// The level to compact to.
outputLevel int
// The file in level that will be compacted. Additional files may be
@@ -671,6 +713,10 @@ type candidateLevelInfo struct {
file manifest.LevelFile
}
func (c *candidateLevelInfo) shouldCompact() bool {
return c.compensatedScoreRatio >= compactionScoreThreshold
}
func fileCompensation(f *fileMetadata) uint64 {
return uint64(f.Stats.PointDeletionsBytesEstimate) + f.Stats.RangeDeletionsBytesEstimate
}
@@ -748,7 +794,7 @@ var _ compactionPicker = &compactionPickerByScore{}
func (p *compactionPickerByScore) getScores(inProgress []compactionInfo) [numLevels]float64 {
var scores [numLevels]float64
for _, info := range p.calculateLevelScores(inProgress) {
scores[info.level] = info.score
scores[info.level] = info.compensatedScoreRatio
}
return scores
}
@@ -967,68 +1013,83 @@ func (p *compactionPickerByScore) calculateLevelScores(
scores[i].level = i
scores[i].outputLevel = i + 1
}
l0UncompensatedScore := calculateL0UncompensatedScore(p.vers, p.opts, inProgressCompactions)
scores[0] = candidateLevelInfo{
outputLevel: p.baseLevel,
score: calculateL0Score(p.vers, p.opts, inProgressCompactions),
outputLevel: p.baseLevel,
uncompensatedScore: l0UncompensatedScore,
compensatedScore: l0UncompensatedScore, /* No level size compensation for L0 */
}
sizeAdjust := calculateSizeAdjust(inProgressCompactions)
for level := 1; level < numLevels; level++ {
compensatedLevelSize := levelCompensatedSize(p.vers.Levels[level]) + sizeAdjust[level].compensated()
scores[level].score = float64(compensatedLevelSize) / float64(p.levelMaxBytes[level])
scores[level].origScore = scores[level].score
// In addition to the compensated score, we calculate a separate score
// that uses actual file sizes, not compensated sizes. This is used
// during score smoothing down below to prevent excessive
// prioritization of reclaiming disk space.
scores[level].rawScore = float64(p.vers.Levels[level].Size()+sizeAdjust[level].actual()) / float64(p.levelMaxBytes[level])
scores[level].compensatedScore = float64(compensatedLevelSize) / float64(p.levelMaxBytes[level])
scores[level].uncompensatedScore = float64(p.vers.Levels[level].Size()+sizeAdjust[level].actual()) / float64(p.levelMaxBytes[level])
}
// Adjust each level's score by the score of the next level. If the next
// level has a high score, and is thus a priority for compaction, this
// reduces the priority for compacting the current level. If the next level
// has a low score (i.e. it is below its target size), this increases the
// priority for compacting the current level.
// Adjust each level's {compensated, uncompensated}Score by the uncompensatedScore
// of the next level to get a {compensated, uncompensated}ScoreRatio. If the
// next level has a high uncompensatedScore, and is thus a priority for compaction,
// this reduces the priority for compacting the current level. If the next level
// has a low uncompensatedScore (i.e. it is below its target size), this increases
// the priority for compacting the current level.
//
// The effect of this adjustment is to help prioritize compactions in lower
// levels. The following shows the new score and original score. In this
// scenario, L0 has 68 sublevels. L3 (a.k.a. Lbase) is significantly above
// its target size. The original score prioritizes compactions from those two
// levels, but doing so ends up causing a future problem: data piles up in
// the higher levels, starving L5->L6 compactions, and to a lesser degree
// starving L4->L5 compactions.
// levels. The following example shows the compensatedScoreRatio and the
// compensatedScore. In this scenario, L0 has 68 sublevels. L3 (a.k.a. Lbase)
// is significantly above its target size. The original score prioritizes
// compactions from those two levels, but doing so ends up causing a future
// problem: data piles up in the higher levels, starving L5->L6 compactions,
// and to a lesser degree starving L4->L5 compactions.
//
// adjusted original
// score score size max-size
// L0 3.2 68.0 2.2 G -
// L3 3.2 21.1 1.3 G 64 M
// L4 3.4 6.7 3.1 G 467 M
// L5 3.4 2.0 6.6 G 3.3 G
// L6 0.6 0.6 14 G 24 G
// Note that in the example shown there is no level size compensation so the
// compensatedScore and the uncompensatedScore is the same for each level.
//
// compensatedScoreRatio compensatedScore uncompensatedScore size max-size
// L0 3.2 68.0 68.0 2.2 G -
// L3 3.2 21.1 21.1 1.3 G 64 M
// L4 3.4 6.7 6.7 3.1 G 467 M
// L5 3.4 2.0 2.0 6.6 G 3.3 G
// L6 0.6 0.6 0.6 14 G 24 G
var prevLevel int
for level := p.baseLevel; level < numLevels; level++ {
if scores[prevLevel].score >= 1 {
// Avoid absurdly large scores by placing a floor on the score that we'll
// adjust a level by. The value of 0.01 was chosen somewhat arbitrarily
const minScore = 0.01
if scores[level].rawScore >= minScore {
scores[prevLevel].score /= scores[level].rawScore
// The compensated scores, and uncompensated scores will be turned into
// ratios as they're adjusted according to other levels' sizes.
scores[prevLevel].compensatedScoreRatio = scores[prevLevel].compensatedScore
scores[prevLevel].uncompensatedScoreRatio = scores[prevLevel].uncompensatedScore
// Avoid absurdly large scores by placing a floor on the score that we'll
// adjust a level by. The value of 0.01 was chosen somewhat arbitrarily.
const minScore = 0.01
if scores[prevLevel].compensatedScoreRatio >= compactionScoreThreshold {
if scores[level].uncompensatedScore >= minScore {
scores[prevLevel].compensatedScoreRatio /= scores[level].uncompensatedScore
} else {
scores[prevLevel].score /= minScore
scores[prevLevel].compensatedScoreRatio /= minScore
}
}
if scores[prevLevel].uncompensatedScoreRatio >= compactionScoreThreshold {
if scores[level].uncompensatedScore >= minScore {
scores[prevLevel].uncompensatedScoreRatio /= scores[level].uncompensatedScore
} else {
scores[prevLevel].uncompensatedScoreRatio /= minScore
}
}
prevLevel = level
}
// Set the score ratios for the lowest level.
// INVARIANT: prevLevel == numLevels-1
scores[prevLevel].compensatedScoreRatio = scores[prevLevel].compensatedScore
scores[prevLevel].uncompensatedScoreRatio = scores[prevLevel].uncompensatedScore
sort.Sort(sortCompactionLevelsDecreasingScore(scores[:]))
sort.Sort(sortCompactionLevelsByPriority(scores[:]))
return scores
}
// calculateL0Score calculates a float score representing the relative priority
// of compacting L0. Level L0 is special in that files within L0 may overlap one
// another, so a different set of heuristics that take into account
// read amplification apply.
func calculateL0Score(
// calculateL0UncompensatedScore calculates a float score representing the
// relative priority of compacting L0. Level L0 is special in that files within
// L0 may overlap one another, so a different set of heuristics that take into
// account read amplification apply.
func calculateL0UncompensatedScore(
vers *version, opts *Options, inProgressCompactions []compactionInfo,
) float64 {
// Use the sublevel count to calculate the score. The base vs intra-L0
@@ -1231,8 +1292,9 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
if pc.startLevel.level == info.level {
marker = "*"
}
fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %5.1f %8s %8s",
marker, info.level, info.score, info.origScore, info.rawScore,
fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %5.1f %5.1f %8s %8s",
marker, info.level, info.compensatedScoreRatio, info.compensatedScore,
info.uncompensatedScoreRatio, info.uncompensatedScore,
humanize.Bytes.Int64(int64(totalCompensatedSize(
p.vers.Levels[info.level].Iter(),
))),
@@ -1262,12 +1324,12 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
pc.startLevel.level, pc.outputLevel.level, buf.String())
}
// Check for a score-based compaction. "scores" has been sorted in order of
// decreasing score. For each level with a score >= 1, we attempt to find a
// compaction anchored at at that level.
// Check for a score-based compaction. candidateLevelInfos are first sorted
// by whether they should be compacted, so if we find a level which shouldn't
// be compacted, we can break early.
for i := range scores {
info := &scores[i]
if info.score < 1 {
if !info.shouldCompact() {
break
}
if info.level == numLevels-1 {
@@ -1279,7 +1341,8 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
// Fail-safe to protect against compacting the same sstable
// concurrently.
if pc != nil && !inputRangeAlreadyCompacting(env, pc) {
pc.score = info.score
p.addScoresToPickedCompactionMetrics(pc, scores)
pc.score = info.compensatedScoreRatio
// TODO(bananabrick): Create an EventListener for logCompaction.
if false {
logCompaction(pc)
@@ -1299,7 +1362,8 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
pc := pickAutoLPositive(env, p.opts, p.vers, *info, p.baseLevel, p.levelMaxBytes)
// Fail-safe to protect against compacting the same sstable concurrently.
if pc != nil && !inputRangeAlreadyCompacting(env, pc) {
pc.score = info.score
p.addScoresToPickedCompactionMetrics(pc, scores)
pc.score = info.compensatedScoreRatio
// TODO(bananabrick): Create an EventListener for logCompaction.
if false {
logCompaction(pc)
@@ -1360,6 +1424,30 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
return nil
}
func (p *compactionPickerByScore) addScoresToPickedCompactionMetrics(
pc *pickedCompaction, candInfo [numLevels]candidateLevelInfo,
) {
// candInfo is sorted by score, not by compaction level.
infoByLevel := [numLevels]candidateLevelInfo{}
for i := range candInfo {
level := candInfo[i].level
infoByLevel[level] = candInfo[i]
}
// Gather the compaction scores for the levels participating in the compaction.
pc.pickerMetrics.scores = make([]float64, len(pc.inputs))
inputIdx := 0
for i := range infoByLevel {
if pc.inputs[inputIdx].level == infoByLevel[i].level {
pc.pickerMetrics.scores[inputIdx] = infoByLevel[i].compensatedScoreRatio
inputIdx++
}
if inputIdx == len(pc.inputs) {
break
}
}
}
// elisionOnlyAnnotator implements the manifest.Annotator interface,
// annotating B-Tree nodes with the *fileMetadata of a file meeting the
// obsolete keys criteria for an elision-only compaction within the subtree.
@@ -1558,7 +1646,7 @@ func (p *compactionPickerByScore) pickRewriteCompaction(env compactionEnv) (pc *
// Fail-safe to protect against compacting the same sstable concurrently.
if !inputRangeAlreadyCompacting(env, pc) {
if pc.startLevel.level == 0 {
pc.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
pc.startLevel.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
}
return pc
}
@@ -1605,33 +1693,113 @@ func pickAutoLPositive(
// maybeAddLevel maybe adds a level to the picked compaction.
func (pc *pickedCompaction) maybeAddLevel(opts *Options, diskAvailBytes uint64) *pickedCompaction {
pc.pickerMetrics.singleLevelOverlappingRatio = pc.overlappingRatio()
if pc.outputLevel.level == numLevels-1 {
// Don't add a level if the current output level is in L6
return pc
}
if !opts.Experimental.MultiLevelCompactionHeuristic.allowL0() && pc.startLevel.level == 0 {
return pc
}
if pc.compactionSize() > expandedCompactionByteSizeLimit(
opts, pc.adjustedOutputLevel, diskAvailBytes) {
opts, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel), diskAvailBytes) {
// Don't add a level if the current compaction exceeds the compaction size limit
return pc
}
return opts.Experimental.MultiLevelCompactionHueristic.pick(pc, opts, diskAvailBytes)
return opts.Experimental.MultiLevelCompactionHeuristic.pick(pc, opts, diskAvailBytes)
}
// MultiLevelHeuristic evaluates whether to add files from the next level into the compaction.
type MultiLevelHeuristic interface {
// evaluate returns the preferred compaction.
// Evaluate returns the preferred compaction.
pick(pc *pickedCompaction, opts *Options, diskAvailBytes uint64) *pickedCompaction
// Returns if the heuristic allows L0 to be involved in ML compaction
allowL0() bool
}
// NoMultiLevel will never add an additional level to the compaction.
type NoMultiLevel struct{}
var _ MultiLevelHeuristic = (*NoMultiLevel)(nil)
func (nml NoMultiLevel) pick(
pc *pickedCompaction, opts *Options, diskAvailBytes uint64,
) *pickedCompaction {
return pc
}
func (nml NoMultiLevel) allowL0() bool {
return false
}
func (pc *pickedCompaction) predictedWriteAmp() float64 {
var bytesToCompact uint64
var higherLevelBytes uint64
for i := range pc.inputs {
levelSize := pc.inputs[i].files.SizeSum()
bytesToCompact += levelSize
if i != len(pc.inputs)-1 {
higherLevelBytes += levelSize
}
}
return float64(bytesToCompact) / float64(higherLevelBytes)
}
func (pc *pickedCompaction) overlappingRatio() float64 {
var higherLevelBytes uint64
var lowestLevelBytes uint64
for i := range pc.inputs {
levelSize := pc.inputs[i].files.SizeSum()
if i == len(pc.inputs)-1 {
lowestLevelBytes += levelSize
continue
}
higherLevelBytes += levelSize
}
return float64(lowestLevelBytes) / float64(higherLevelBytes)
}
// WriteAmpHeuristic defines a multi level compaction heuristic which will add
// an additional level to the picked compaction if it reduces predicted write
// amp of the compaction + the addPropensity constant.
type WriteAmpHeuristic struct {
// addPropensity is a constant that affects the propensity to conduct multilevel
// compactions. If positive, a multilevel compaction may get picked even if
// the single level compaction has lower write amp, and vice versa.
AddPropensity float64
// AllowL0 if true, allow l0 to be involved in a ML compaction.
AllowL0 bool
}
var _ MultiLevelHeuristic = (*WriteAmpHeuristic)(nil)
// TODO(msbutler): microbenchmark the extent to which multilevel compaction
// picking slows down the compaction picking process. This should be as fast as
// possible since Compaction-picking holds d.mu, which prevents WAL rotations,
// in-progress flushes and compactions from completing, etc. Consider ways to
// deduplicate work, given that setupInputs has already been called.
func (wa WriteAmpHeuristic) pick(
pcOrig *pickedCompaction, opts *Options, diskAvailBytes uint64,
) *pickedCompaction {
pcMulti := pcOrig.clone()
if !pcMulti.setupMultiLevelCandidate(opts, diskAvailBytes) {
return pcOrig
}
picked := pcOrig
if pcMulti.predictedWriteAmp() <= pcOrig.predictedWriteAmp()+wa.AddPropensity {
picked = pcMulti
}
// Regardless of what compaction was picked, log the multilevelOverlapping ratio.
picked.pickerMetrics.multiLevelOverlappingRatio = pcMulti.overlappingRatio()
return picked
}
func (wa WriteAmpHeuristic) allowL0() bool {
return wa.AllowL0
}
// Helper method to pick compactions originating from L0. Uses information about
// sublevels to generate a compaction.
func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc *pickedCompaction) {
@@ -1653,7 +1821,7 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc
if pc.startLevel.files.Empty() {
opts.Logger.Fatalf("empty compaction chosen")
}
return pc
return pc.maybeAddLevel(opts, env.diskAvailBytes)
}
// Couldn't choose a base compaction. Try choosing an intra-L0