aleesa-bot/aleesa-phrases-go
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgrade versions of vendored libs

Browse Source
This commit is contained in:
eleksir
2022-11-04 16:29:14 +03:00
parent fba732cc24
commit a7ae372459
941 changed files with 219674 additions and 23611 deletions
Download Patch File Download Diff File Expand all files Collapse all files

479
vendor/github.com/cockroachdb/pebble/table_stats.go generated vendored
View File

@@ -5,6 +5,7 @@
package pebble
import (
"fmt"
"math"
"github.com/cockroachdb/pebble/internal/base"
@@ -52,7 +53,7 @@ func (d *DB) maybeCollectTableStatsLocked() {
func (d *DB) updateTableStatsLocked(newFiles []manifest.NewFileEntry) {
var needStats bool
for _, nf := range newFiles {
if !nf.Meta.Stats.Valid {
if !nf.Meta.StatsValidLocked() {
needStats = true
break
}
@@ -125,7 +126,8 @@ func (d *DB) collectTableStats() bool {
maybeCompact := false
for _, c := range collected {
c.fileMetadata.Stats = c.TableStats
maybeCompact = maybeCompact || c.fileMetadata.Stats.RangeDeletionsBytesEstimate > 0
maybeCompact = maybeCompact || c.TableStats.RangeDeletionsBytesEstimate > 0
c.fileMetadata.StatsMarkValid()
}
d.mu.tableStats.cond.Broadcast()
d.maybeCollectTableStatsLocked()
@@ -173,7 +175,7 @@ func (d *DB) loadNewFileStats(
// collectTableStats updates f.Stats for active files, and we
// ensure only one goroutine runs it at a time through
// d.mu.tableStats.loading.
if nf.Meta.Stats.Valid {
if nf.Meta.StatsValidLocked() {
continue
}
@@ -215,9 +217,9 @@ func (d *DB) scanReadStateTableStats(
// NB: We're not holding d.mu which protects f.Stats, but only the
// active stats collection job updates f.Stats for active files,
// and we ensure only one goroutine runs it at a time through
// d.mu.tableStats.loading. This makes it safe to read
// f.Stats.Valid despite not holding d.mu.
if f.Stats.Valid {
// d.mu.tableStats.loading. This makes it safe to read validity
// through f.Stats.ValidLocked despite not holding d.mu.
if f.StatsValidLocked() {
continue
}
@@ -261,7 +263,7 @@ func (d *DB) loadTableStats(
return
}
}
if r.Properties.NumRangeDeletions > 0 {
if r.Properties.NumRangeDeletions > 0 || r.Properties.NumRangeKeyDels > 0 {
if compactionHints, err = d.loadTableRangeDelStats(r, v, level, meta, &stats); err != nil {
return
}
@@ -269,13 +271,12 @@ func (d *DB) loadTableStats(
// TODO(travers): Once we have real-world data, consider collecting
// additional stats that may provide improved heuristics for compaction
// picking.
stats.NumRangeKeys = r.Properties.NumRangeKeys()
stats.NumRangeKeySets = r.Properties.NumRangeKeySets
return
})
if err != nil {
return stats, nil, err
}
stats.Valid = true
return stats, compactionHints, nil
}
@@ -299,68 +300,85 @@ func (d *DB) loadTablePointKeyStats(
return nil
}
// loadTableRangeDelStats calculates the range deletion statistics for the given
// table.
// loadTableRangeDelStats calculates the range deletion and range key deletion
// statistics for the given table.
func (d *DB) loadTableRangeDelStats(
r *sstable.Reader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats,
) ([]deleteCompactionHint, error) {
var compactionHints []deleteCompactionHint
// We iterate over the defragmented range tombstones, which ensures
// we don't double count ranges deleted at different sequence numbers.
// Also, merging abutting tombstones reduces the number of calls to
// estimateSizeBeneath which is costly, and improves the accuracy of
// our overall estimate.
rangeDelIter, err := r.NewRawRangeDelIter()
iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta)
if err != nil {
return nil, err
}
defer rangeDelIter.Close()
// Truncate tombstones to the containing file's bounds if necessary.
// See docs/range_deletions.md for why this is necessary.
rangeDelIter = keyspan.Truncate(
d.cmp, rangeDelIter, meta.Smallest.UserKey, meta.Largest.UserKey, nil, nil)
err = foreachDefragmentedTombstone(rangeDelIter, d.cmp,
func(startUserKey, endUserKey []byte, smallestSeqNum, largestSeqNum uint64) error {
// If the file is in the last level of the LSM, there is no
// data beneath it. The fact that there is still a range
// tombstone in a bottommost file suggests that an open
// snapshot kept the tombstone around. Estimate disk usage
// within the file itself.
if level == numLevels-1 {
size, err := r.EstimateDiskUsage(startUserKey, endUserKey)
if err != nil {
return err
}
stats.RangeDeletionsBytesEstimate += size
return nil
defer iter.Close()
var compactionHints []deleteCompactionHint
// We iterate over the defragmented range tombstones and range key deletions,
// which ensures we don't double count ranges deleted at different sequence
// numbers. Also, merging abutting tombstones reduces the number of calls to
// estimateReclaimedSizeBeneath which is costly, and improves the accuracy of
// our overall estimate.
for s := iter.First(); s != nil; s = iter.Next() {
start, end := s.Start, s.End
// We only need to consider deletion size estimates for tables that contain
// point keys.
var hasPoints bool
for _, k := range s.Keys {
if k.Kind() == base.InternalKeyKindRangeDelete {
hasPoints = true
break
}
}
estimate, hintSeqNum, err := d.estimateSizeBeneath(v, level, meta, startUserKey, endUserKey)
// If the file is in the last level of the LSM, there is no data beneath
// it. The fact that there is still a range tombstone in a bottommost file
// suggests that an open snapshot kept the tombstone around. Estimate disk
// usage within the file itself.
// NOTE: If the span `s` wholly contains a table containing range keys,
// the returned size estimate will be slightly inflated by the range key
// block. However, in practice, range keys are expected to be rare, and
// the size of the range key block relative to the overall size of the
// table is expected to be small.
if hasPoints && level == numLevels-1 {
size, err := r.EstimateDiskUsage(start, end)
if err != nil {
return err
return nil, err
}
stats.RangeDeletionsBytesEstimate += estimate
stats.RangeDeletionsBytesEstimate += size
// If any files were completely contained with the range,
// hintSeqNum is the smallest sequence number contained in any
// such file.
if hintSeqNum == math.MaxUint64 {
return nil
}
hint := deleteCompactionHint{
start: make([]byte, len(startUserKey)),
end: make([]byte, len(endUserKey)),
tombstoneFile: meta,
tombstoneLevel: level,
tombstoneLargestSeqNum: largestSeqNum,
tombstoneSmallestSeqNum: smallestSeqNum,
fileSmallestSeqNum: hintSeqNum,
}
copy(hint.start, startUserKey)
copy(hint.end, endUserKey)
compactionHints = append(compactionHints, hint)
return nil
})
// As the file is in the bottommost level, there is no need to collect a
// deletion hint.
continue
}
// While the size estimates for point keys should only be updated if this
// span contains a range del, the sequence numbers are required for the
// hint. Unconditionally descend, but conditionally update the estimates.
hintType := compactionHintFromKeys(s.Keys)
estimate, hintSeqNum, err := d.estimateReclaimedSizeBeneath(v, level, start, end, hintType)
if err != nil {
return nil, err
}
stats.RangeDeletionsBytesEstimate += estimate
// If any files were completely contained with the range,
// hintSeqNum is the smallest sequence number contained in any
// such file.
if hintSeqNum == math.MaxUint64 {
continue
}
hint := deleteCompactionHint{
hintType: hintType,
start: make([]byte, len(start)),
end: make([]byte, len(end)),
tombstoneFile: meta,
tombstoneLevel: level,
tombstoneLargestSeqNum: s.LargestSeqNum(),
tombstoneSmallestSeqNum: s.SmallestSeqNum(),
fileSmallestSeqNum: hintSeqNum,
}
copy(hint.start, start)
copy(hint.end, end)
compactionHints = append(compactionHints, hint)
}
return compactionHints, err
}
@@ -410,8 +428,8 @@ func (d *DB) averageEntrySizeBeneath(
return avgKeySize, avgValueSize, err
}
func (d *DB) estimateSizeBeneath(
v *version, level int, meta *fileMetadata, start, end []byte,
func (d *DB) estimateReclaimedSizeBeneath(
v *version, level int, start, end []byte, hintType deleteCompactionHintType,
) (estimate uint64, hintSeqNum uint64, err error) {
// Find all files in lower levels that overlap with the deleted range
// [start, end).
@@ -430,13 +448,58 @@ func (d *DB) estimateSizeBeneath(
startCmp := d.cmp(start, file.Smallest.UserKey)
endCmp := d.cmp(file.Largest.UserKey, end)
if startCmp <= 0 && (endCmp < 0 || endCmp == 0 && file.Largest.IsExclusiveSentinel()) {
// The range fully contains the file, so skip looking it up in
// table cache/looking at its indexes and add the full file size.
estimate += file.Size
if hintSeqNum > file.SmallestSeqNum {
// The range fully contains the file, so skip looking it up in table
// cache/looking at its indexes and add the full file size. Whether the
// disk estimate and hint seqnums are updated depends on a) the type of
// hint that requested the estimate and b) the keys contained in this
// current file.
var updateEstimates, updateHints bool
switch hintType {
case deleteCompactionHintTypePointKeyOnly:
// The range deletion byte estimates should only be updated if this
// table contains point keys. This ends up being an overestimate in
// the case that table also has range keys, but such keys are expected
// to contribute a negligible amount of the table's overall size,
// relative to point keys.
if file.HasPointKeys {
updateEstimates = true
}
// As the initiating span contained only range dels, hints can only be
// updated if this table does _not_ contain range keys.
if !file.HasRangeKeys {
updateHints = true
}
case deleteCompactionHintTypeRangeKeyOnly:
// The initiating span contained only range key dels. The estimates
// apply only to point keys, and are therefore not updated.
updateEstimates = false
// As the initiating span contained only range key dels, hints can
// only be updated if this table does _not_ contain point keys.
if !file.HasPointKeys {
updateHints = true
}
case deleteCompactionHintTypePointAndRangeKey:
// Always update the estimates and hints, as this hint type can drop a
// file, irrespective of the mixture of keys. Similar to above, the
// range del bytes estimates is an overestimate.
updateEstimates, updateHints = true, true
default:
panic(fmt.Sprintf("pebble: unknown hint type %s", hintType))
}
if updateEstimates {
estimate += file.Size
}
if updateHints && hintSeqNum > file.SmallestSeqNum {
hintSeqNum = file.SmallestSeqNum
}
} else if d.cmp(file.Smallest.UserKey, end) <= 0 && d.cmp(start, file.Largest.UserKey) <= 0 {
// Partial overlap.
if hintType == deleteCompactionHintTypeRangeKeyOnly {
// If the hint that generated this overlap contains only range keys,
// there is no need to calculate disk usage, as the reclaimable space
// is expected to be minimal relative to point keys.
continue
}
var size uint64
err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) {
size, err = r.EstimateDiskUsage(start, end)
@@ -452,60 +515,19 @@ func (d *DB) estimateSizeBeneath(
return estimate, hintSeqNum, nil
}
func foreachDefragmentedTombstone(
rangeDelIter keyspan.FragmentIterator,
cmp base.Compare,
fn func([]byte, []byte, uint64, uint64) error,
) error {
// Use an equals func that will always merge abutting spans.
equal := func(_ base.Compare, _, _ keyspan.Span) bool { return true }
// Reduce keys by maintaining a slice of length two, corresponding to the
// largest and smallest keys in the defragmented span. This maintains the
// contract that the emitted slice is sorted by (SeqNum, Kind) descending.
reducer := func(current, incoming []keyspan.Key) []keyspan.Key {
if len(current) == 0 && len(incoming) == 0 {
// While this should never occur in practice, a defensive return is used
// here to preserve correctness.
return current
}
// Determine the smallest / largest keys across the two slices. As `current`
// and `incoming` are both sorted by (seqnum, kind) descending, only the
// first and last element of each slice need to be consulted.
var largest, smallest keyspan.Key
var largestSet, smallestSet bool
for _, keys := range [2][]keyspan.Key{current, incoming} {
if len(keys) == 0 {
continue
}
first, last := keys[0], keys[len(keys)-1]
// Update largest.
if !largestSet || first.Trailer > largest.Trailer {
largest, largestSet = first, true
}
// Update smallest.
if !smallestSet || last.Trailer < smallest.Trailer {
smallest, smallestSet = last, true
}
}
return append(current[:0], largest, smallest)
}
iter := keyspan.DefragmentingIter{}
iter.Init(cmp, rangeDelIter, equal, reducer)
for s := iter.First(); s.Valid(); s = iter.Next() {
if err := fn(s.Start, s.End, s.SmallestSeqNum(), s.LargestSeqNum()); err != nil {
_ = iter.Close()
return err
}
}
return iter.Close()
}
func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) bool {
// If a table has range deletions, we can't calculate the
// RangeDeletionsBytesEstimate statistic and can't populate table stats
// from just the properties. The table stats collector goroutine will
// populate the stats.
if props.NumRangeDeletions != 0 {
// If a table contains range deletions or range key deletions, we defer the
// stats collection. There are two main reasons for this:
//
// 1. Estimating the potential for reclaimed space due to a range deletion
// tombstone requires scanning the LSM - a potentially expensive operation
// that should be deferred.
// 2. Range deletions and / or range key deletions present an opportunity to
// compute "deletion hints", which also requires a scan of the LSM to
// compute tables that would be eligible for deletion.
//
// These two tasks are deferred to the table stats collector goroutine.
if props.NumRangeDeletions != 0 || props.NumRangeKeyDels != 0 {
return false
}
@@ -528,14 +550,12 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties)
pointEstimate = pointDeletionsBytesEstimate(props, avgKeySize, avgValSize)
}
meta.Stats = manifest.TableStats{
Valid: true,
NumEntries: props.NumEntries,
NumDeletions: props.NumDeletions,
NumRangeKeys: props.NumRangeKeys(),
PointDeletionsBytesEstimate: pointEstimate,
RangeDeletionsBytesEstimate: 0,
}
meta.Stats.NumEntries = props.NumEntries
meta.Stats.NumDeletions = props.NumDeletions
meta.Stats.NumRangeKeySets = props.NumRangeKeySets
meta.Stats.PointDeletionsBytesEstimate = pointEstimate
meta.Stats.RangeDeletionsBytesEstimate = 0
meta.StatsMarkValid()
return true
}
@@ -581,3 +601,208 @@ func estimateEntrySizes(
avgValSize = props.RawValueSize * fileSizePerEntry / uncompressedSum
return avgKeySize, avgValSize
}
// newCombinedDeletionKeyspanIter returns a keyspan.FragmentIterator that
// returns "ranged deletion" spans for a single table, providing a combined view
// of both range deletion and range key deletion spans. The
// tableRangedDeletionIter is intended for use in the specific case of computing
// the statistics and deleteCompactionHints for a single table.
//
// As an example, consider the following set of spans from the range deletion
// and range key blocks of a table:
//
// |---------| |---------| |-------| RANGEKEYDELs
// |-----------|-------------| |-----| RANGEDELs
// __________________________________________________________
// a b c d e f g h i j k l m n o p q r s t u v w x y z
//
// The tableRangedDeletionIter produces the following set of output spans, where
// '1' indicates a span containing only range deletions, '2' is a span
// containing only range key deletions, and '3' is a span containing a mixture
// of both range deletions and range key deletions.
//
// 1 3 1 3 2 1 3 2
// |-----|---------|-----|---|-----| |---|-|-----|
// __________________________________________________________
// a b c d e f g h i j k l m n o p q r s t u v w x y z
//
// Algorithm.
//
// The iterator first defragments the range deletion and range key blocks
// separately. During this defragmentation, the range key block is also filtered
// so that keys other than range key deletes are ignored. The range delete and
// range key delete keyspaces are then merged.
//
// Note that the only fragmentation introduced by merging is from where a range
// del span overlaps with a range key del span. Within the bounds of any overlap
// there is guaranteed to be no further fragmentation, as the constituent spans
// have already been defragmented. To the left and right of any overlap, the
// same reasoning applies. For example,
//
// |--------| |-------| RANGEKEYDEL
// |---------------------------| RANGEDEL
// |----1---|----3---|----1----|---2---| Merged, fragmented spans.
// __________________________________________________________
// a b c d e f g h i j k l m n o p q r s t u v w x y z
//
// Any fragmented abutting spans produced by the merging iter will be of
// differing types (i.e. a transition from a span with homogenous key kinds to a
// heterogeneous span, or a transition from a span with exclusively range dels
// to a span with exclusively range key dels). Therefore, further
// defragmentation is not required.
//
// Each span returned by the tableRangeDeletionIter will have at most four keys,
// corresponding to the largest and smallest sequence numbers encountered across
// the range deletes and range keys deletes that comprised the merged spans.
func newCombinedDeletionKeyspanIter(
comparer *base.Comparer, r *sstable.Reader, m *fileMetadata,
) (keyspan.FragmentIterator, error) {
// The range del iter and range key iter are each wrapped in their own
// defragmenting iter. For each iter, abutting spans can always be merged.
var equal = keyspan.DefragmentMethodFunc(func(_ base.Equal, a, b *keyspan.Span) bool { return true })
// Reduce keys by maintaining a slice of at most length two, corresponding to
// the largest and smallest keys in the defragmented span. This maintains the
// contract that the emitted slice is sorted by (SeqNum, Kind) descending.
reducer := func(current, incoming []keyspan.Key) []keyspan.Key {
if len(current) == 0 && len(incoming) == 0 {
// While this should never occur in practice, a defensive return is used
// here to preserve correctness.
return current
}
var largest, smallest keyspan.Key
var set bool
for _, keys := range [2][]keyspan.Key{current, incoming} {
if len(keys) == 0 {
continue
}
first, last := keys[0], keys[len(keys)-1]
if !set {
largest, smallest = first, last
set = true
continue
}
if first.Trailer > largest.Trailer {
largest = first
}
if last.Trailer < smallest.Trailer {
smallest = last
}
}
if largest.Equal(comparer.Equal, smallest) {
current = append(current[:0], largest)
} else {
current = append(current[:0], largest, smallest)
}
return current
}
// The separate iters for the range dels and range keys are wrapped in a
// merging iter to join the keyspaces into a single keyspace. The separate
// iters are only added if the particular key kind is present.
mIter := &keyspan.MergingIter{}
var transform = keyspan.TransformerFunc(func(cmp base.Compare, in keyspan.Span, out *keyspan.Span) error {
if in.KeysOrder != keyspan.ByTrailerDesc {
panic("pebble: combined deletion iter encountered keys in non-trailer descending order")
}
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
out.KeysOrder = keyspan.ByTrailerDesc
// NB: The order of by-trailer descending may have been violated,
// because we've layered rangekey and rangedel iterators from the same
// sstable into the same keyspan.MergingIter. The MergingIter will
// return the keys in the order that the child iterators were provided.
// Sort the keys to ensure they're sorted by trailer descending.
keyspan.SortKeysByTrailer(&out.Keys)
return nil
})
mIter.Init(comparer.Compare, transform)
iter, err := r.NewRawRangeDelIter()
if err != nil {
return nil, err
}
if iter != nil {
dIter := &keyspan.DefragmentingIter{}
dIter.Init(comparer, iter, equal, reducer)
iter = dIter
// Truncate tombstones to the containing file's bounds if necessary.
// See docs/range_deletions.md for why this is necessary.
iter = keyspan.Truncate(
comparer.Compare, iter, m.Smallest.UserKey, m.Largest.UserKey, nil, nil,
)
mIter.AddLevel(iter)
}
iter, err = r.NewRawRangeKeyIter()
if err != nil {
return nil, err
}
if iter != nil {
// Wrap the range key iterator in a filter that elides keys other than range
// key deletions.
iter = keyspan.Filter(iter, func(in *keyspan.Span, out *keyspan.Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = out.Keys[:0]
for _, k := range in.Keys {
if k.Kind() != base.InternalKeyKindRangeKeyDelete {
continue
}
out.Keys = append(out.Keys, k)
}
return len(out.Keys) > 0
})
dIter := &keyspan.DefragmentingIter{}
dIter.Init(comparer, iter, equal, reducer)
iter = dIter
mIter.AddLevel(iter)
}
return mIter, nil
}
// rangeKeySetsAnnotator implements manifest.Annotator, annotating B-Tree nodes
// with the sum of the files' counts of range key fragments. Its annotation type
// is a *uint64. The count of range key sets may change once a table's stats are
// loaded asynchronously, so its values are marked as cacheable only if a file's
// stats have been loaded.
type rangeKeySetsAnnotator struct{}
var _ manifest.Annotator = rangeKeySetsAnnotator{}
func (a rangeKeySetsAnnotator) Zero(dst interface{}) interface{} {
if dst == nil {
return new(uint64)
}
v := dst.(*uint64)
*v = 0
return v
}
func (a rangeKeySetsAnnotator) Accumulate(
f *fileMetadata, dst interface{},
) (v interface{}, cacheOK bool) {
vptr := dst.(*uint64)
*vptr = *vptr + f.Stats.NumRangeKeySets
return vptr, f.StatsValidLocked()
}
func (a rangeKeySetsAnnotator) Merge(src interface{}, dst interface{}) interface{} {
srcV := src.(*uint64)
dstV := dst.(*uint64)
*dstV = *dstV + *srcV
return dstV
}
// countRangeKeySetFragments counts the number of RANGEKEYSET keys across all
// files of the LSM. It only counts keys in files for which table stats have
// been loaded. It uses a b-tree annotator to cache intermediate values between
// calculations when possible.
func countRangeKeySetFragments(v *version) (count uint64) {
for l := 0; l < numLevels; l++ {
if v.RangeKeyLevels[l].Empty() {
continue
}
count += *v.RangeKeyLevels[l].Annotation(rangeKeySetsAnnotator{}).(*uint64)
}
return count
}