aleesa-bot/aleesa-irc-go
0
Fork 0
Code Activity

Upgrade versions of vendored libs

Browse Source
This commit is contained in:
eleksir
2023-05-18 23:55:10 +03:00
parent 4869c146fe
commit 7958db9c0d
555 changed files with 36170 additions and 15565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go generated vendored
View File

@@ -66,6 +66,9 @@ type BoundedIter struct {
// indicating whether the iterator is in prefix iteration mode and the prefix
// key if it is. This is used to exclude spans that are outside the iteration
// prefix.
//
// hasPrefix and prefix are allowed to be nil, however if hasPrefix != nil,
// prefix must also not be nil.
func (i *BoundedIter) Init(
cmp base.Compare,
split base.Split,
@@ -146,7 +149,7 @@ func (i *BoundedIter) Next() *Span {
}
// Similarly, if the span extends to the next prefix and we're in prefix
// iteration mode, we can avoid advancing.
if i.iterSpan != nil && *i.hasPrefix {
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
ei := i.split(i.iterSpan.End)
if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 {
i.pos = posAtUpperLimit
@@ -178,7 +181,7 @@ func (i *BoundedIter) Prev() *Span {
}
// Similarly, if the span extends to or beyond the current prefix and
// we're in prefix iteration mode, we can avoid advancing.
if i.iterSpan != nil && *i.hasPrefix {
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(i.iterSpan.Start)
if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 {
i.pos = posAtLowerLimit
@@ -215,7 +218,7 @@ func (i *BoundedIter) SetBounds(lower, upper []byte) {
func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && *i.hasPrefix {
if span != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(span.Start)
if i.cmp(span.Start[:si], *i.prefix) > 0 {
// This span starts at a prefix that sorts after our current prefix.
@@ -242,7 +245,7 @@ func (i *BoundedIter) checkForwardBound(span *Span) *Span {
func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
if span != nil && i.hasPrefix != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
// This span ends before the current prefix.
span = nil
}

141
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored
View File

@@ -8,14 +8,20 @@ import (
"bytes"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/invariants"
)
// bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
// that DefragmentingIter will reuse. Buffers greater than this will be
// that DefragmentingIter will reuse. Buffers larger than this will be
// discarded and reallocated as necessary.
const bufferReuseMaxCapacity = 10 << 10 // 10 KB
// keysReuseMaxCapacity is the maximum capacity of a []keyspan.Key buffer that
// DefragmentingIter will reuse. Buffers larger than this will be discarded and
// reallocated as necessary.
const keysReuseMaxCapacity = 100
// DefragmentMethod configures the defragmentation performed by the
// DefragmentingIter.
type DefragmentMethod interface {
@@ -113,30 +119,21 @@ const (
//
// Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may
// land on a physical fragment in the middle of several fragments that must be
// defragmented. A seek first degfragments in the opposite direction of
// iteration to find the beginning of the defragmented span, and then
// defragments in the iteration direction, ensuring it's found a whole
// defragmented span.
// defragmented. A seek that lands in a fragment straddling the seek key must
// first degfragment in the opposite direction of iteration to find the
// beginning of the defragmented span, and then defragments in the iteration
// direction, ensuring it's found a whole defragmented span.
type DefragmentingIter struct {
// DefragmentingBuffers holds buffers used for copying iterator state.
*DefragmentingBuffers
comparer *base.Comparer
equal base.Equal
iter FragmentIterator
iterSpan *Span
iterPos iterPos
// curr holds the span at the current iterator position. currBuf is a buffer
// for use when copying user keys for curr. keysBuf is a buffer for use when
// copying Keys for curr. currBuf is cleared between positioning methods.
//
// keyBuf is a buffer specifically for the defragmented start key when
// defragmenting backwards or the defragmented end key when defragmenting
// forwards. These bounds are overwritten repeatedly during defragmentation,
// and the defragmentation routines overwrite keyBuf repeatedly to store
// these extended bounds.
curr Span
currBuf []byte
keysBuf []Key
keyBuf []byte
// curr holds the span at the current iterator position.
curr Span
// method is a comparison function for two spans. method is called when two
// spans are abutting to determine whether they may be defragmented.
@@ -148,20 +145,53 @@ type DefragmentingIter struct {
reduce DefragmentReducer
}
// DefragmentingBuffers holds buffers used for copying iterator state.
type DefragmentingBuffers struct {
// currBuf is a buffer for use when copying user keys for curr. currBuf is
// cleared between positioning methods.
currBuf bytealloc.A
// keysBuf is a buffer for use when copying Keys for DefragmentingIter.curr.
keysBuf []Key
// keyBuf is a buffer specifically for the defragmented start key when
// defragmenting backwards or the defragmented end key when defragmenting
// forwards. These bounds are overwritten repeatedly during defragmentation,
// and the defragmentation routines overwrite keyBuf repeatedly to store
// these extended bounds.
keyBuf []byte
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *DefragmentingBuffers) PrepareForReuse() {
if cap(bufs.currBuf) > bufferReuseMaxCapacity {
bufs.currBuf = nil
}
if cap(bufs.keyBuf) > bufferReuseMaxCapacity {
bufs.keyBuf = nil
}
if cap(bufs.keysBuf) > keysReuseMaxCapacity {
bufs.keysBuf = nil
}
}
// Assert that *DefragmentingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*DefragmentingIter)(nil)
// Init initializes the defragmenting iter using the provided defragment
// method.
func (i *DefragmentingIter) Init(
comparer *base.Comparer, iter FragmentIterator, equal DefragmentMethod, reducer DefragmentReducer,
comparer *base.Comparer,
iter FragmentIterator,
equal DefragmentMethod,
reducer DefragmentReducer,
bufs *DefragmentingBuffers,
) {
*i = DefragmentingIter{
comparer: comparer,
equal: comparer.Equal,
iter: iter,
method: equal,
reduce: reducer,
DefragmentingBuffers: bufs,
comparer: comparer,
equal: comparer.Equal,
iter: iter,
method: equal,
reduce: reducer,
}
}
@@ -175,8 +205,9 @@ func (i *DefragmentingIter) Close() error {
return i.iter.Close()
}
// SeekGE seeks the iterator to the first span with a start key greater than or
// equal to key and returns it.
// SeekGE moves the iterator to the first span covering a key greater than or
// equal to the given key. This is equivalent to seeking to the first span with
// an end key greater than the given key.
func (i *DefragmentingIter) SeekGE(key []byte) *Span {
i.iterSpan = i.iter.SeekGE(key)
if i.iterSpan == nil {
@@ -186,30 +217,28 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
i.iterPos = iterPosCurr
return i.iterSpan
}
// Save the current span and peek backwards.
i.saveCurrent()
i.iterSpan = i.iter.Prev()
if i.iterSpan != nil && i.equal(i.curr.Start, i.iterSpan.End) && i.checkEqual(i.iterSpan, &i.curr) {
// A continuation. The span we originally landed on and defragmented
// backwards has a true Start key < key. To obey the FragmentIterator
// contract, we must not return this defragmented span. Defragment
// forward to finish defragmenting the span in the forward direction.
i.defragmentForward()
// Now we must be on a span that truly has a defragmented Start key >
// key.
// If the span starts strictly after key, we know there mustn't be an
// earlier span that ends at i.iterSpan.Start, otherwise i.iter would've
// returned that span instead.
if i.comparer.Compare(i.iterSpan.Start, key) > 0 {
return i.defragmentForward()
}
// The span previous to i.curr does not defragment, so we should return it.
// Next the underlying iterator back onto the span we previously saved to
// i.curr and then defragment forward.
i.iterSpan = i.iter.Next()
// The span we landed on has a Start bound ≤ key. There may be additional
// fragments before this span. Defragment backward to find the start of the
// defragmented span.
i.defragmentBackward()
if i.iterPos == iterPosPrev {
// Next once back onto the span.
i.iterSpan = i.iter.Next()
}
// Defragment the full span from its start.
return i.defragmentForward()
}
// SeekLT seeks the iterator to the last span with a start key less than
// key and returns it.
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
func (i *DefragmentingIter) SeekLT(key []byte) *Span {
i.iterSpan = i.iter.SeekLT(key)
if i.iterSpan == nil {
@@ -219,7 +248,16 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
i.iterPos = iterPosCurr
return i.iterSpan
}
// Defragment forward to find the end of the defragmented span.
// If the span ends strictly before key, we know there mustn't be a later
// span that starts at i.iterSpan.End, otherwise i.iter would've returned
// that span instead.
if i.comparer.Compare(i.iterSpan.End, key) < 0 {
return i.defragmentBackward()
}
// The span we landed on has a End bound ≥ key. There may be additional
// fragments after this span. Defragment forward to find the end of the
// defragmented span.
i.defragmentForward()
if i.iterPos == iterPosNext {
// Prev once back onto the span.
@@ -437,15 +475,9 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
}
func (i *DefragmentingIter) saveCurrent() {
i.currBuf = i.currBuf[:0]
i.currBuf.Reset()
i.keysBuf = i.keysBuf[:0]
i.keyBuf = i.keyBuf[:0]
if cap(i.currBuf) > bufferReuseMaxCapacity {
i.currBuf = nil
}
if cap(i.keyBuf) > bufferReuseMaxCapacity {
i.keyBuf = nil
}
if i.iterSpan == nil {
return
}
@@ -468,7 +500,6 @@ func (i *DefragmentingIter) saveBytes(b []byte) []byte {
if b == nil {
return nil
}
ret := append(i.currBuf, b...)
i.currBuf = ret[len(ret):]
return ret
i.currBuf, b = i.currBuf.Copy(b)
return b
}

124
vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go generated vendored
View File

@@ -108,20 +108,20 @@ func (f *Fragmenter) checkInvariants(buf []Span) {
// increasing start key order. That is, Add must be called with a series
// of spans like:
//
// a---e
// c---g
// c-----i
// j---n
// j-l
// a---e
// c---g
// c-----i
// j---n
// j-l
//
// We need to fragment the spans at overlap points. In the above
// example, we'd create:
//
// a-c-e
// c-e-g
// c-e-g-i
// j-l-n
// j-l
// a-c-e
// c-e-g
// c-e-g-i
// j-l-n
// j-l
//
// The fragments need to be output sorted by start key, and for equal start
// keys, sorted by descending sequence number. This last part requires a mild
@@ -134,42 +134,42 @@ func (f *Fragmenter) checkInvariants(buf []Span) {
//
// Walking through the example above, we start with:
//
// a---e
// a---e
//
// Next we add [c,g) resulting in:
//
// a-c-e
// c---g
// a-c-e
// c---g
//
// The fragment [a,c) is flushed leaving the pending spans as:
//
// c-e
// c---g
// c-e
// c---g
//
// The next span is [c,i):
//
// c-e
// c---g
// c-----i
// c-e
// c---g
// c-----i
//
// No fragments are flushed. The next span is [j,n):
//
// c-e
// c---g
// c-----i
// j---n
// c-e
// c---g
// c-----i
// j---n
//
// The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments
// by their end key, then split the fragments on the end keys:
//
// c-e
// c-e-g
// c-e---i
// c-e
// c-e-g
// c-e---i
//
// The [c,e) fragments all get flushed leaving:
//
// e-g
// e---i
// e-g
// e---i
//
// This process continues until there are no more fragments to flush.
//
@@ -229,16 +229,35 @@ func (f *Fragmenter) Add(s Span) {
f.pending = append(f.pending, s)
}
// Covers returns true if the specified key is covered by one of the pending
// spans. The key must be consistent with the ordering of the
// spans. That is, it is invalid to specify a key here that is out of
// order with the span start keys passed to Add.
func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
// Cover is returned by Framenter.Covers and describes a span's relationship to
// a key at a particular snapshot.
type Cover int8
const (
// NoCover indicates the tested key does not fall within the span's bounds,
// or the span contains no keys with sequence numbers higher than the key's.
NoCover Cover = iota
// CoversInvisibly indicates the tested key does fall within the span's
// bounds and the span contains at least one key with a higher sequence
// number, but none visible at the provided snapshot.
CoversInvisibly
// CoversVisibly indicates the tested key does fall within the span's
// bounds, and the span constains at least one key with a sequence number
// higher than the key's sequence number that is visible at the provided
// snapshot.
CoversVisibly
)
// Covers returns an enum indicating whether the specified key is covered by one
// of the pending keys. The provided key must be consistent with the ordering of
// the spans. That is, it is invalid to specify a key here that is out of order
// with the span start keys passed to Add.
func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) Cover {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if len(f.pending) == 0 {
return false
return NoCover
}
if f.Cmp(f.pending[0].Start, key.UserKey) > 0 {
@@ -246,18 +265,31 @@ func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
f.Format(f.pending[0].Start), key.Pretty(f.Format)))
}
cover := NoCover
seqNum := key.SeqNum()
for _, s := range f.pending {
if f.Cmp(key.UserKey, s.End) < 0 {
// NB: A range deletion tombstone does not delete a point operation
// at the same sequence number, and broadly a span is not considered
// to cover a point operation at the same sequence number.
if s.CoversAt(snapshot, seqNum) {
return true
for i := range s.Keys {
if kseq := s.Keys[i].SeqNum(); kseq > seqNum {
// This key from the span has a higher sequence number than
// `key`. It covers `key`, although the span's key might not
// be visible if its snapshot is too high.
//
// Batch keys are always be visible.
if kseq < snapshot || kseq&base.InternalKeySeqNumBatch != 0 {
return CoversVisibly
}
// s.Keys[i] is not visible.
cover = CoversInvisibly
}
}
}
}
return false
return cover
}
// Empty returns true if all fragments added so far have finished flushing.
@@ -270,20 +302,20 @@ func (f *Fragmenter) Empty() bool {
// emitting of spans which straddle an sstable boundary. Consider
// the scenario:
//
// a---------k#10
// f#8
// f#7
///
// a---------k#10
// f#8
// f#7
//
// Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will
// flush this span:
//
// a-------g#10
// f#8
// f#7
// a-------g#10
// f#8
// f#7
//
// And leave this one in f.pending:
//
// g----k#10
// g----k#10
//
// WARNING: The fragmenter could hold on to the specified end key. Ensure it's
// a safe byte slice that could outlast the current sstable output, and one
@@ -374,8 +406,8 @@ func (f *Fragmenter) truncateAndFlush(key []byte) {
// we want to flush (but not truncate) all range tombstones that start at or
// before the first key in the next sstable. Consider:
//
// a---e#10
// a------h#9
// a---e#10
// a------h#9
//
// If a compaction splits the sstables at key c we want the first sstable to
// contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally

83
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored
View File

@@ -29,9 +29,9 @@ type SpanMask interface {
SpanChanged(*Span)
// SkipPoint is invoked by the interleaving iterator whenever the iterator
// encounters a point key covered by a Span. If SkipPoint returns true, the
// interleaving iterator skips the point key without returning it. This is
// used during range key iteration to skip over point keys 'masked' by range
// keys.
// interleaving iterator skips the point key and all larger keys with the
// same prefix. This is used during range key iteration to skip over point
// keys 'masked' by range keys.
SkipPoint(userKey []byte) bool
}
@@ -79,13 +79,13 @@ type SpanMask interface {
// SeekGE or SeekPrefixGE search key. Consider, for example SetBounds('c', 'e'),
// with an iterator containing the Span [a,z):
//
// First() = `c#72057594037927935,21` Span() = [c,e)
// SeekGE('d') = `d#72057594037927935,21` Span() = [c,e)
// First() = `c#72057594037927935,21` Span() = [c,e)
// SeekGE('d') = `d#72057594037927935,21` Span() = [c,e)
//
// InterleavedIter does not interleave synthetic markers for spans that do not
// contain any keys.
//
// SpanMask
// # SpanMask
//
// InterelavingIter takes a SpanMask parameter that may be used to configure the
// behavior of the iterator. See the documentation on the SpanMask type.
@@ -479,6 +479,30 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) {
return i.interleaveForward(i.lower, nil /* prefix */)
}
// NextPrefix implements (base.InternalIterator).NextPrefix.
func (i *InterleavingIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
if i.dir == -1 {
panic("pebble: cannot switch directions with NextPrefix")
}
// Refresh the point key if the current point key has already been
// interleaved, OR if the next uninterleaved point key has the old suffix
// (it's less than succKey).
if i.pointKeyInterleaved || (i.pointKey != nil && i.cmp(i.pointKey.UserKey, succKey) < 0) {
i.pointKey, i.pointVal = i.pointIter.NextPrefix(succKey)
i.pointKeyInterleaved = false
}
// If we already interleaved the current span start key, and the point key
// is ≥ the span's end key, move to the next span.
if i.keyspanInterleaved && i.pointKey != nil && i.span != nil &&
i.cmp(i.pointKey.UserKey, i.span.End) >= 0 {
i.span = i.keyspanIter.Next()
i.checkForwardBound(nil)
i.savedKeyspan()
}
return i.interleaveForward(i.lower, nil)
}
// Prev implements (base.InternalIterator).Prev.
func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
if i.dir == +1 {
@@ -677,9 +701,29 @@ func (i *InterleavingIter) interleaveForward(
// if we have stepped outside of the span last saved as a mask,
// so that the decision to skip is made with the correct
// knowledge of the covering span.
i.maybeUpdateMask(true /*covered */)
i.maybeUpdateMask(true /* covered */)
if i.mask != nil && i.mask.SkipPoint(i.pointKey.UserKey) {
if i.prefix {
// During prefix-iteration node, once a point is masked,
// all subsequent keys with the same prefix must also be
// masked according to the key ordering. We can stop and
// return nil.
//
// NB: The above is not just an optimization. During
// prefix-iteration mode, the internal iterator contract
// prohibits us from Next-ing beyond the first key
// beyond the iteration prefix. If we didn't already
// stop early, we would need to check if this masked
// point is already beyond the prefix.
return i.yieldNil()
}
// TODO(jackson): If we thread a base.Comparer through to
// InterleavingIter so that we have access to
// ImmediateSuccessor, we could use NextPrefix. We'd need to
// tweak the SpanMask interface slightly, but it's probably
// worthwhile.
i.pointKey, i.pointVal = i.pointIter.Next()
// We may have just invalidated the invariant that
// ensures the span's End is > the point key, so
@@ -780,27 +824,16 @@ func (i *InterleavingIter) interleaveBackward() (*base.InternalKey, base.LazyVal
}
}
// keyspanSeekGE seeks the keyspan iterator to the first span covering k ≥ key.
// Note that this differs from the FragmentIterator.SeekGE semantics, which
// seek to the first span with a start key ≥ key.
func (i *InterleavingIter) keyspanSeekGE(key []byte, prefix []byte) {
// Seek using SeekLT to look for a span that starts before key, with an end
// boundary extending beyond key.
i.span = i.keyspanIter.SeekLT(key)
if i.span == nil || i.cmp(i.span.End, key) <= 0 {
// The iterator is exhausted in the reverse direction, or the span we
// found ends before key. Next to the first key with a start ≥ key.
i.span = i.keyspanIter.Next()
}
// keyspanSeekGE seeks the keyspan iterator to the first span covering a key ≥ k.
func (i *InterleavingIter) keyspanSeekGE(k []byte, prefix []byte) {
i.span = i.keyspanIter.SeekGE(k)
i.checkForwardBound(prefix)
i.savedKeyspan()
}
// keyspanSeekLT seeks the keyspan iterator to the last span covering k < key.
// Note that this differs from the FragmentIterator.SeekLT semantics, which
// seek to the last span with a start key < key.
func (i *InterleavingIter) keyspanSeekLT(key []byte) {
i.span = i.keyspanIter.SeekLT(key)
// keyspanSeekLT seeks the keyspan iterator to the last span covering a key < k.
func (i *InterleavingIter) keyspanSeekLT(k []byte) {
i.span = i.keyspanIter.SeekLT(k)
i.checkBackwardBound()
// The current span's start key is not guaranteed to be less than key,
// because of the bounds enforcement. Consider the following example:
@@ -813,7 +846,7 @@ func (i *InterleavingIter) keyspanSeekLT(key []byte) {
//
// This problem is a consequence of the SeekLT's exclusive search key and
// the fact that we don't perform bounds truncation at every leaf iterator.
if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, key) >= 0 {
if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, k) >= 0 {
i.span = nil
}
i.savedKeyspan()

8
vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go generated vendored
View File

@@ -16,6 +16,7 @@ import "github.com/cockroachdb/pebble/internal/base"
// keyspan.FragmentIterator with point keys.
type InternalIteratorShim struct {
miter MergingIter
mbufs MergingBuffers
span *Span
iterKey base.InternalKey
}
@@ -26,7 +27,7 @@ var _ base.InternalIterator = &InternalIteratorShim{}
// Init initializes the internal iterator shim to merge the provided fragment
// iterators.
func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) {
i.miter.Init(cmp, noopTransform, iters...)
i.miter.Init(cmp, noopTransform, &i.mbufs, iters...)
}
// Span returns the span containing the full set of keys over the key span at
@@ -87,6 +88,11 @@ func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) {
return &i.iterKey, base.MakeInPlaceValue(i.span.End)
}
// NextPrefix implements (base.InternalIterator).NextPrefix.
func (i *InternalIteratorShim) NextPrefix([]byte) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Prev implements (base.InternalIterator).Prev.
func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")

19
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored
View File

@@ -18,12 +18,14 @@ import (
// longer lifetimes but implementations need only guarantee stability until the
// next positioning method.
type FragmentIterator interface {
// SeekGE moves the iterator to the first span whose start key is greater
// than or equal to the given key.
// SeekGE moves the iterator to the first span covering a key greater than
// or equal to the given key. This is equivalent to seeking to the first
// span with an end key greater than the given key.
SeekGE(key []byte) *Span
// SeekLT moves the iterator to the last span whose start key is less than
// the given key.
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
SeekLT(key []byte) *Span
// First moves the iterator to the first span.
@@ -58,7 +60,8 @@ type FragmentIterator interface {
Close() error
}
// TableNewSpanIter creates a new iterator for spans for the given file.
// TableNewSpanIter creates a new iterator for range key spans for the given
// file.
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
@@ -104,6 +107,9 @@ func (i *Iter) Init(cmp base.Compare, spans []Span) {
func (i *Iter) SeekGE(key []byte) *Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(j) = false iff the span i.spans[j] is strictly before `key`
// (equivalently, i.spans[j].End ≤ key.)
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
@@ -111,12 +117,13 @@ func (i *Iter) SeekGE(key []byte) *Span {
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].Start) > 0 {
if i.cmp(key, i.spans[h].End) >= 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
if i.index >= len(i.spans) {

118
vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go generated vendored
View File

@@ -8,25 +8,23 @@ import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
)
// Logger defines an interface for writing log messages.
type Logger interface {
Infof(format string, args ...interface{})
Fatalf(format string, args ...interface{})
}
// LevelIter provides a merged view of spans from sstables in a level.
// It takes advantage of level invariants to only have one sstable span block
// open at one time, opened using the newIter function passed in.
type LevelIter struct {
logger Logger
cmp base.Compare
// Denotes if this level iter should read point key spans (i.e. rangedels,
// or range keys. If key type is Point, no straddle spans are emitted between
// files, and point key bounds are used to find files instead of range key
// bounds.
cmp base.Compare
// Denotes the kind of key the level iterator should read. If the key type
// is KeyTypePoint, the level iterator will read range tombstones (which
// only affect point keys). If the key type is KeyTypeRange, the level
// iterator will read range keys. It is invalid to configure an iterator
// with the KeyTypePointAndRange key type.
//
// If key type is KeyTypePoint, no straddle spans are emitted between files,
// and point key bounds are used to find files instead of range key bounds.
//
// TODO(bilal): Straddle spans can safely be produced in rangedel mode once
// we can guarantee that we will never read sstables in a level that split
@@ -50,14 +48,17 @@ type LevelIter struct {
dir int
straddle Span
straddleDir int
// The iter for the current file. It is nil under any of the following conditions:
// The iter for the current file (iterFile). It is nil under any of the
// following conditions:
// - files.Current() == nil
// - err != nil
// - straddleDir != 0, in which case iterFile is not nil and points to the
// next file (in the straddleDir direction).
// - some other constraint, like the bounds in opts, caused the file at index to not
// be relevant to the iteration.
iter FragmentIterator
iter FragmentIterator
// iterFile holds the current file.
// INVARIANT: iterFile = files.Current()
iterFile *manifest.FileMetadata
newIter TableNewSpanIter
files manifest.LevelIterator
@@ -72,18 +73,17 @@ type LevelIter struct {
// LevelIter implements the keyspan.FragmentIterator interface.
var _ FragmentIterator = (*LevelIter)(nil)
// newLevelIter returns a LevelIter.
func newLevelIter(
// NewLevelIter returns a LevelIter.
func NewLevelIter(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
logger Logger,
keyType manifest.KeyType,
) *LevelIter {
l := &LevelIter{}
l.Init(opts, cmp, newIter, files, level, logger, keyType)
l.Init(opts, cmp, newIter, files, level, keyType)
return l
}
@@ -94,18 +94,19 @@ func (l *LevelIter) Init(
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
logger Logger,
keyType manifest.KeyType,
) {
l.err = nil
l.level = level
l.logger = logger
l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
l.cmp = cmp
l.iterFile = nil
l.newIter = newIter
switch keyType {
case manifest.KeyTypePoint, manifest.KeyTypeRange:
case manifest.KeyTypePoint:
l.keyType = keyType
l.files = files.Filter(keyType)
case manifest.KeyTypeRange:
l.keyType = keyType
l.files = files.Filter(keyType)
default:
@@ -196,7 +197,9 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
f := l.findFileGE(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
// Peek at the previous file.
prevFile := l.files.Prev()
l.files.Next()
if prevFile != nil {
// We could unconditionally return an empty span between the seek key and
// f.SmallestRangeKey, however if this span is to the left of all range
@@ -212,11 +215,10 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
l.files.Next()
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return nil
return l.verify(nil)
}
l.straddleDir = +1
l.straddle = Span{
@@ -224,15 +226,15 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
End: f.SmallestRangeKey.UserKey,
Keys: nil,
}
return &l.straddle
return l.verify(&l.straddle)
}
}
loadFileIndicator := l.loadFile(f, +1)
if loadFileIndicator == noFileLoaded {
return nil
return l.verify(nil)
}
if span := l.iter.SeekGE(key); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileForward()
}
@@ -246,7 +248,9 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
f := l.findFileLT(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
// Peek at the next file.
nextFile := l.files.Next()
l.files.Prev()
if nextFile != nil {
// We could unconditionally return an empty span between f.LargestRangeKey
// and the seek key, however if this span is to the right of all range keys
@@ -262,11 +266,10 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
l.files.Prev()
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return nil
return l.verify(nil)
}
l.straddleDir = -1
l.straddle = Span{
@@ -274,14 +277,14 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
End: nextFile.SmallestRangeKey.UserKey,
Keys: nil,
}
return &l.straddle
return l.verify(&l.straddle)
}
}
if l.loadFile(l.findFileLT(key), -1) == noFileLoaded {
return nil
if l.loadFile(f, -1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.SeekLT(key); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
@@ -294,10 +297,10 @@ func (l *LevelIter) First() *Span {
l.err = nil // clear cached iteration error
if l.loadFile(l.files.First(), +1) == noFileLoaded {
return nil
return l.verify(nil)
}
if span := l.iter.First(); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileForward()
}
@@ -310,10 +313,10 @@ func (l *LevelIter) Last() *Span {
l.err = nil // clear cached iteration error
if l.loadFile(l.files.Last(), -1) == noFileLoaded {
return nil
return l.verify(nil)
}
if span := l.iter.Last(); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
@@ -321,7 +324,7 @@ func (l *LevelIter) Last() *Span {
// Next implements keyspan.FragmentIterator.
func (l *LevelIter) Next() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) {
return nil
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir <= 0
@@ -331,7 +334,7 @@ func (l *LevelIter) Next() *Span {
if l.iter != nil {
if span := l.iter.Next(); span != nil {
return span
return l.verify(span)
}
}
return l.skipEmptyFileForward()
@@ -340,7 +343,7 @@ func (l *LevelIter) Next() *Span {
// Prev implements keyspan.FragmentIterator.
func (l *LevelIter) Prev() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) {
return nil
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir >= 0
@@ -350,7 +353,7 @@ func (l *LevelIter) Prev() *Span {
if l.iter != nil {
if span := l.iter.Prev(); span != nil {
return span
return l.verify(span)
}
}
return l.skipEmptyFileBackward()
@@ -367,7 +370,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
// Straddle spans are not created in rangedel mode.
if err := l.Close(); err != nil {
l.err = err
return nil
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -375,7 +378,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
// which it should be due to the Close() call above.
l.iterFile = l.files.Next()
if l.iterFile == nil {
return nil
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
@@ -386,7 +389,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
End: endKey,
}
l.straddleDir = +1
return &l.straddle
return l.verify(&l.straddle)
}
} else if l.straddleDir < 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
@@ -405,7 +408,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
fileToLoad = l.files.Next()
}
if l.loadFile(fileToLoad, +1) == noFileLoaded {
return nil
return l.verify(nil)
}
span = l.iter.First()
// In rangedel mode, we can expect to get empty files that we'd need to
@@ -414,7 +417,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
break
}
}
return span
return l.verify(span)
}
func (l *LevelIter) skipEmptyFileBackward() *Span {
@@ -428,7 +431,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
l.iterFile != nil && l.iter != nil {
if err := l.Close(); err != nil {
l.err = err
return nil
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -436,7 +439,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
// which it should be due to the Close() call above.
l.iterFile = l.files.Prev()
if l.iterFile == nil {
return nil
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
@@ -447,7 +450,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
End: endKey,
}
l.straddleDir = -1
return &l.straddle
return l.verify(&l.straddle)
}
} else if l.straddleDir > 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
@@ -464,7 +467,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
fileToLoad = l.files.Prev()
}
if l.loadFile(fileToLoad, -1) == noFileLoaded {
return nil
return l.verify(nil)
}
span = l.iter.Last()
// In rangedel mode, we can expect to get empty files that we'd need to
@@ -474,7 +477,22 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
break
}
}
return span
return l.verify(span)
}
// verify is invoked whenever a span is returned from an iterator positioning
// method to a caller. During invariant builds, it asserts invariants to the
// caller.
func (l *LevelIter) verify(s *Span) *Span {
// NB: Do not add any logic outside the invariants.Enabled conditional to
// ensure that verify is always compiled away in production builds.
if invariants.Enabled {
if f := l.files.Current(); f != l.iterFile {
panic(fmt.Sprintf("LevelIter.files.Current (%s) and l.iterFile (%s) diverged",
f, l.iterFile))
}
}
return s
}
// Error implements keyspan.FragmentIterator.

423
vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go generated vendored
View File

@@ -21,6 +21,10 @@ import (
// seeks would require introducing key comparisons to switchTo{Min,Max}Heap
// where there currently are none.
// TODO(jackson): There are several opportunities to use base.Equal in the
// MergingIter implementation, but will require a bit of plumbing to thread the
// Equal function.
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
@@ -45,9 +49,9 @@ var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst
return nil
})
// visibleTransform filters keys that are invisible at the provided snapshot
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func visibleTransform(snapshot uint64) Transformer {
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
@@ -76,34 +80,34 @@ func visibleTransform(snapshot uint64) Transformer {
// trailer descending. If the MergingIter is configured with a Transformer, it's
// permitted to modify the ordering of the spans' keys returned by MergingIter.
//
// Algorithm
// # Algorithm
//
// The merging iterator wraps child iterators, merging and fragmenting spans
// across levels. The high-level algorithm is:
//
// 1. Initialize the heap with bound keys from child iterators' spans.
// 2. Find the next [or previous] two unique user keys' from bounds.
// 3. Consider the span formed between the two unique user keys a candidate
// span.
// 4. Determine if any of the child iterators' spans overlap the candidate
// span.
// 4a. If any of the child iterator's current bounds are end keys
// (during forward iteration) or start keys (during reverse
// iteration), then all the spans with that bound overlap the
// candidate span.
// 4b. Apply the configured transform, which may remove keys.
// 4b. If no spans overlap, forget the smallest (forward iteration)
// or largest (reverse iteration) unique user key and advance
// the iterators to the next unique user key. Start again from 3.
// 1. Initialize the heap with bound keys from child iterators' spans.
// 2. Find the next [or previous] two unique user keys' from bounds.
// 3. Consider the span formed between the two unique user keys a candidate
// span.
// 4. Determine if any of the child iterators' spans overlap the candidate
// span.
// 4a. If any of the child iterator's current bounds are end keys
// (during forward iteration) or start keys (during reverse
// iteration), then all the spans with that bound overlap the
// candidate span.
// 4b. Apply the configured transform, which may remove keys.
// 4c. If no spans overlap, forget the smallest (forward iteration)
// or largest (reverse iteration) unique user key and advance
// the iterators to the next unique user key. Start again from 3.
//
// Detailed algorithm
// # Detailed algorithm
//
// Each level (i0, i1, ...) has a user-provided input FragmentIterator. The
// merging iterator steps through individual boundaries of the underlying
// spans separately. If the underlying FragmentIterator has fragments
// [a,b){#2,#1} [b,c){#1} the mergingIterLevel.{next,prev} step through:
//
// (a, start), (b, end), (b, start), (c, end)
// (a, start), (b, end), (b, start), (c, end)
//
// Note that (a, start) and (b, end) are observed ONCE each, despite two keys
// sharing those bounds. Also note that (b, end) and (b, start) are two distinct
@@ -119,18 +123,18 @@ func visibleTransform(snapshot uint64) Transformer {
// determine which span is next, but it's also responsible for fragmenting
// overlapping spans. Consider the example:
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
//
// fragments: a-b-c-d-e-----h-----k----------p
// fragments: a-b-c-d-e-----h-----k----------p
//
// None of the individual child iterators contain a span with the exact bounds
// [c,d), but the merging iterator must produce a span [c,d). To accomplish
// this, the merging iterator visits every span between unique boundary user
// keys. In the above example, this is:
//
// [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p)
// [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p)
//
// The merging iterator first initializes the heap to prepare for iteration.
// The description below discusses the mechanics of forward iteration after a
@@ -141,9 +145,9 @@ func visibleTransform(snapshot uint64) Transformer {
// mergingIterLevel to the first bound of the first fragment. In the above
// example, this seeks the child iterators to:
//
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (a, boundKindFragmentStart, [ [a,c) ])
// i2: (a, boundKindFragmentStart, [ [a,p) ])
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (a, boundKindFragmentStart, [ [a,c) ])
// i2: (a, boundKindFragmentStart, [ [a,p) ])
//
// After fixing up the heap, the root of the heap is a boundKey with the
// smallest user key ('a' in the example). Once the heap is setup for iteration
@@ -160,9 +164,9 @@ func visibleTransform(snapshot uint64) Transformer {
// In the above example, this results in m.start = 'a', m.end = 'b' and child
// iterators in the following positions:
//
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (c, boundKindFragmentEnd, [ [a,c) ])
// i2: (p, boundKindFragmentEnd, [ [a,p) ])
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (c, boundKindFragmentEnd, [ [a,c) ])
// i2: (p, boundKindFragmentEnd, [ [a,p) ])
//
// With the user key bounds of the next merged span established,
// findNextFragmentSet must determine which, if any, fragments overlap the span.
@@ -191,7 +195,7 @@ func visibleTransform(snapshot uint64) Transformer {
// findNextFragmentSet loops, repeating the above process again until it finds a
// span that does contain keys.
//
// Memory safety
// # Memory safety
//
// The FragmentIterator interface only guarantees stability of a Span and its
// associated slices until the next positioning method is called. Adjacent Spans
@@ -207,24 +211,14 @@ func visibleTransform(snapshot uint64) Transformer {
// accommodate this, find{Next,Prev}FragmentSet copy the initial boundary if the
// subsequent Next/Prev would move to the next span.
type MergingIter struct {
levels []mergingIterLevel
heap mergingIterHeap
*MergingBuffers
// start and end hold the bounds for the span currently under the
// iterator position.
//
// Invariant: None of the levels' iterators contain spans with a bound
// between start and end. For all bounds b, b ≤ start || b ≥ end.
start, end []byte
// buf is a buffer used to save [start, end) boundary keys.
buf []byte
// keys holds all of the keys across all levels that overlap the key span
// [start, end), sorted by Trailer descending. This slice is reconstituted
// in synthesizeKeys from each mergingIterLevel's keys every time the
// [start, end) bounds change.
//
// Each element points into a child iterator's memory, so the keys may not
// be directly modified.
keys keysBySeqNumKind
// transformer defines a transformation to be applied to a span before it's
// yielded to the user. Transforming may filter individual keys contained
// within the span.
@@ -233,9 +227,8 @@ type MergingIter struct {
// destination for transforms. Every tranformed span overwrites the
// previous.
span Span
err error
dir int8
err error
dir int8
// alloc preallocates mergingIterLevel and mergingIterItems for use by the
// merging iterator. As long as the merging iterator is used with
@@ -249,6 +242,35 @@ type MergingIter struct {
}
}
// MergingBuffers holds buffers used while merging keyspans.
type MergingBuffers struct {
// keys holds all of the keys across all levels that overlap the key span
// [start, end), sorted by Trailer descending. This slice is reconstituted
// in synthesizeKeys from each mergingIterLevel's keys every time the
// [start, end) bounds change.
//
// Each element points into a child iterator's memory, so the keys may not
// be directly modified.
keys keysBySeqNumKind
// levels holds levels allocated by MergingIter.init. The MergingIter will
// prefer use of its `manifest.NumLevels+3` array, so this slice will be
// longer if set.
levels []mergingIterLevel
// heap holds a slice for the merging iterator heap allocated by
// MergingIter.init. The MergingIter will prefer use of its
// `manifest.NumLevels+3` items array, so this slice will be longer if set.
heap mergingIterHeap
// buf is a buffer used to save [start, end) boundary keys.
buf []byte
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *MergingBuffers) PrepareForReuse() {
if cap(bufs.buf) > bufferReuseMaxCapacity {
bufs.buf = nil
}
}
// MergingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*MergingIter)(nil)
@@ -300,13 +322,15 @@ func (l *mergingIterLevel) prev() {
}
// Init initializes the merging iterator with the provided fragment iterators.
func (m *MergingIter) Init(cmp base.Compare, transformer Transformer, iters ...FragmentIterator) {
levels, items := m.levels, m.heap.items
func (m *MergingIter) Init(
cmp base.Compare, transformer Transformer, bufs *MergingBuffers, iters ...FragmentIterator,
) {
*m = MergingIter{
heap: mergingIterHeap{cmp: cmp},
transformer: transformer,
MergingBuffers: bufs,
transformer: transformer,
}
m.heap.cmp = cmp
levels, items := m.levels, m.heap.items
// Invariant: cap(levels) >= cap(items)
// Invariant: cap(alloc.levels) == cap(alloc.items)
@@ -334,38 +358,70 @@ func (m *MergingIter) AddLevel(iter FragmentIterator) {
m.levels = append(m.levels, mergingIterLevel{iter: iter})
}
// SeekGE moves the iterator to the first span with a start key greater than or
// equal to key.
// SeekGE moves the iterator to the first span covering a key greater than
// or equal to the given key. This is equivalent to seeking to the first
// span with an end key greater than the given key.
func (m *MergingIter) SeekGE(key []byte) *Span {
m.invalidate() // clear state about current position
// SeekGE(k) seeks to the first span with an end key greater than the given
// key. The merged span M that we're searching for might straddle the seek
// `key`. In this case, the M.Start may be a key ≤ the seek key.
//
// Consider a SeekGE(dog) in the following example.
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
// merged: a-b-c-d-e-----h-----k----------p
//
// The merged span M containing 'dog' is [d,e). The 'd' of the merged span
// comes from i0's [b,d)'s end boundary. The [b,d) span does not cover any
// key >= dog, so we cannot find the span by positioning the child iterators
// using a SeekGE(dog).
//
// Instead, if we take all the child iterators' spans bounds:
// a b c d e h k p
// We want to partition them into keys ≤ `key` and keys > `key`.
// dog
// │
// a b c d│e h k p
// │
// The largest key on the left of the partition forms the merged span's
// start key, and the smallest key on the right of the partition forms the
// merged span's end key. Recharacterized:
//
// M.Start: the largest boundary ≤ k of any child span
// M.End: the smallest boundary > k of any child span
//
// The FragmentIterator interface doesn't implement seeking by all bounds,
// it implements seeking by containment. A SeekGE(k) will ensure we observe
// all start boundaries ≥ k and all end boundaries > k but does not ensure
// we observe end boundaries = k or any boundaries < k. A SeekLT(k) will
// ensure we observe all start boundaries < k and all end boundaries ≤ k but
// does not ensure we observe any start boundaries = k or any boundaries >
// k. This forces us to seek in one direction and step in the other.
//
// In a SeekGE, we want to end up oriented in the forward direction when
// complete, so we begin with searching for M.Start by SeekLT-ing every
// child iterator to `k`. For every child span found, we determine the
// largest bound ≤ `k` and use it to initialize our max heap. The resulting
// root of the max heap is a preliminary value for `M.Start`.
for i := range m.levels {
l := &m.levels[i]
// A SeekGE requires we position each level at the smallest bound ≥ key.
// We must search through both inclusive start and exclusive end bounds.
// Note that this search requirement differs from FragmentIterator's
// .SeekGE'semantics, which returns the span with the smallest start key
// ≥ key. To remedy this difference, we find the last span less than
// key. If its end boundary is greater than or equal to key, we use it.
// Otherwise we use the start boundary of the next span which
// necessarily has a start ≥ key.
s := l.iter.SeekLT(key)
if s != nil && m.cmp(s.End, key) >= 0 {
// s.End ≥ key
// We need to use this span's end bound.
if s == nil {
l.heapKey = boundKey{kind: boundKindInvalid}
} else if m.cmp(s.End, key) <= 0 {
l.heapKey = boundKey{
kind: boundKindFragmentEnd,
key: s.End,
span: s,
}
continue
}
// s.End < key
// The span `s` ends before key. Next to the first span with a Start ≥
// key, and use that.
if s = l.iter.Next(); s == nil {
l.heapKey = boundKey{kind: boundKindInvalid}
} else {
// s.End > key && s.Start < key
// We need to use this span's start bound, since that's the largest
// bound ≤ key.
l.heapKey = boundKey{
kind: boundKindFragmentStart,
key: s.Start,
@@ -373,36 +429,201 @@ func (m *MergingIter) SeekGE(key []byte) *Span {
}
}
}
m.initMinHeap()
return m.findNextFragmentSet()
}
m.initMaxHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
// There are no spans covering any key < `key`. There is no span that
// straddles the seek key. Reorient the heap into a min heap and return
// the first span we find in the forward direction.
m.switchToMinHeap()
return m.findNextFragmentSet()
}
// SeekLT moves the iterator to the last span with a start key less than key.
func (m *MergingIter) SeekLT(key []byte) *Span {
// TODO(jackson): Evaluate whether there's an implementation of SeekLT
// independent of SeekGE that is more efficient. It's tricky, because the
// span we should return might straddle `key` itself.
// The heap root is now the largest boundary key b such that:
// 1. b < k
// 2. b = k, and b is an end boundary
// There's a third case that we will need to consider later, after we've
// switched to a min heap:
// 3. there exists a start boundary key b such that b = k.
// A start boundary key equal to k would not be surfaced when we seeked all
// the levels using SeekLT(k), since no key <k would be covered within a
// span within an inclusive `k` start boundary.
//
// Consider the scenario:
// a----------l #2
// b-----------m #1
//
// The merged, fully-fragmented spans that MergingIter exposes to the caller
// have bounds:
// a-b #2
// b--------l #2
// b--------l #1
// l-m #1
//
// A call SeekLT(c) must return the largest of the above spans with a
// Start user key < key: [b,l)#1. This requires examining bounds both < 'c'
// (the 'b' of [b,m)#1's start key) and bounds ≥ 'c' (the 'l' of ([a,l)#2's
// end key).
if s := m.SeekGE(key); s == nil && m.err != nil {
// Assume that the tightest boundary ≤ k is the current heap root (cases 1 &
// 2). After we switch to a min heap, we'll check for the third case and
// adjust the start boundary if necessary.
m.start = m.heap.items[0].boundKey.key
// Before switching the direction of the heap, save a copy of the start
// boundary if it's the end boundary of some child span. Next-ing the child
// iterator might switch files and invalidate the memory of the bound.
if m.heap.items[0].boundKey.kind == boundKindFragmentEnd {
m.buf = append(m.buf[:0], m.start...)
m.start = m.buf
}
// Switch to a min heap. This will move each level to the next bound in
// every level, and then establish a min heap. This allows us to obtain the
// smallest boundary key > `key`, which will serve as our candidate end
// bound.
m.switchToMinHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
return nil
}
// Prev to the previous span.
return m.Prev()
// Check for the case 3 described above. It's possible that when we switch
// heap directions, we discover a start boundary of some child span that is
// equal to the seek key `key`. In this case, we want this key to be our
// start boundary.
if m.heap.items[0].boundKey.kind == boundKindFragmentStart &&
m.cmp(m.heap.items[0].boundKey.key, key) == 0 {
// Call findNextFragmentSet, which will set m.start to the heap root and
// proceed forward.
return m.findNextFragmentSet()
}
m.end = m.heap.items[0].boundKey.key
if found, s := m.synthesizeKeys(+1); found && s != nil {
return s
}
return m.findNextFragmentSet()
}
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
func (m *MergingIter) SeekLT(key []byte) *Span {
m.invalidate() // clear state about current position
// SeekLT(k) seeks to the last span with a start key less than the given
// key. The merged span M that we're searching for might straddle the seek
// `key`. In this case, the M.End may be a key ≥ the seek key.
//
// Consider a SeekLT(dog) in the following example.
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
// merged: a-b-c-d-e-----h-----k----------p
//
// The merged span M containing the largest key <'dog' is [d,e). The 'e' of
// the merged span comes from i0's [e,h)'s start boundary. The [e,h) span
// does not cover any key < dog, so we cannot find the span by positioning
// the child iterators using a SeekLT(dog).
//
// Instead, if we take all the child iterators' spans bounds:
// a b c d e h k p
// We want to partition them into keys < `key` and keys ≥ `key`.
// dog
// │
// a b c d│e h k p
// │
// The largest key on the left of the partition forms the merged span's
// start key, and the smallest key on the right of the partition forms the
// merged span's end key. Recharacterized:
//
// M.Start: the largest boundary < k of any child span
// M.End: the smallest boundary ≥ k of any child span
//
// The FragmentIterator interface doesn't implement seeking by all bounds,
// it implements seeking by containment. A SeekGE(k) will ensure we observe
// all start boundaries ≥ k and all end boundaries > k but does not ensure
// we observe end boundaries = k or any boundaries < k. A SeekLT(k) will
// ensure we observe all start boundaries < k and all end boundaries ≤ k but
// does not ensure we observe any start boundaries = k or any boundaries >
// k. This forces us to seek in one direction and step in the other.
//
// In a SeekLT, we want to end up oriented in the backward direction when
// complete, so we begin with searching for M.End by SeekGE-ing every
// child iterator to `k`. For every child span found, we determine the
// smallest bound ≥ `k` and use it to initialize our min heap. The resulting
// root of the min heap is a preliminary value for `M.End`.
for i := range m.levels {
l := &m.levels[i]
s := l.iter.SeekGE(key)
if s == nil {
l.heapKey = boundKey{kind: boundKindInvalid}
} else if m.cmp(s.Start, key) >= 0 {
l.heapKey = boundKey{
kind: boundKindFragmentStart,
key: s.Start,
span: s,
}
} else {
// s.Start < key
// We need to use this span's end bound, since that's the smallest
// bound > key.
l.heapKey = boundKey{
kind: boundKindFragmentEnd,
key: s.End,
span: s,
}
}
}
m.initMinHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
// There are no spans covering any key ≥ `key`. There is no span that
// straddles the seek key. Reorient the heap into a max heap and return
// the first span we find in the reverse direction.
m.switchToMaxHeap()
return m.findPrevFragmentSet()
}
// The heap root is now the smallest boundary key b such that:
// 1. b > k
// 2. b = k, and b is a start boundary
// There's a third case that we will need to consider later, after we've
// switched to a max heap:
// 3. there exists an end boundary key b such that b = k.
// An end boundary key equal to k would not be surfaced when we seeked all
// the levels using SeekGE(k), since k would not be contained within the
// exclusive end boundary.
//
// Assume that the tightest boundary ≥ k is the current heap root (cases 1 &
// 2). After we switch to a max heap, we'll check for the third case and
// adjust the end boundary if necessary.
m.end = m.heap.items[0].boundKey.key
// Before switching the direction of the heap, save a copy of the end
// boundary if it's the start boundary of some child span. Prev-ing the
// child iterator might switch files and invalidate the memory of the bound.
if m.heap.items[0].boundKey.kind == boundKindFragmentStart {
m.buf = append(m.buf[:0], m.end...)
m.end = m.buf
}
// Switch to a max heap. This will move each level to the previous bound in
// every level, and then establish a max heap. This allows us to obtain the
// largest boundary key < `key`, which will serve as our candidate start
// bound.
m.switchToMaxHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
return nil
}
// Check for the case 3 described above. It's possible that when we switch
// heap directions, we discover an end boundary of some child span that is
// equal to the seek key `key`. In this case, we want this key to be our end
// boundary.
if m.heap.items[0].boundKey.kind == boundKindFragmentEnd &&
m.cmp(m.heap.items[0].boundKey.key, key) == 0 {
// Call findPrevFragmentSet, which will set m.end to the heap root and
// proceed backwards.
return m.findPrevFragmentSet()
}
m.start = m.heap.items[0].boundKey.key
if found, s := m.synthesizeKeys(-1); found && s != nil {
return s
}
return m.findPrevFragmentSet()
}
// First seeks the iterator to the first span.
@@ -823,8 +1044,8 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) {
// sorted by trailer descending for the range key iteration use case.
sort.Sort(&m.keys)
// Apply the configured transform. See visibleTransform.
s := Span{
// Apply the configured transform. See VisibleTransform.
m.span = Span{
Start: m.start,
End: m.end,
Keys: m.keys,
@@ -832,7 +1053,7 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) {
}
// NB: m.heap.cmp is a base.Compare, whereas m.cmp is a method on
// MergingIter.
if err := m.transformer.Transform(m.heap.cmp, s, &m.span); err != nil {
if err := m.transformer.Transform(m.heap.cmp, m.span, &m.span); err != nil {
m.err = err
return false, nil
}

28
vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go generated vendored
View File

@@ -6,34 +6,6 @@ package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// SeekGE seeks to the span that contains the target key or the first span past
// the target key.
func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were to
// use SeekGE then the search key `c` would land on the span [e,i) and
// we'd have to backtrack. The one complexity here is what happens for the
// search key `e`. In that case SeekLT will land us on the span [a,e)
// and we'll have to move forward.
iterSpan := iter.SeekLT(key)
// Invariant: key > iterSpan.Start
if iterSpan == nil || cmp(key, iterSpan.End) >= 0 {
// The current span lies entirely before the search key, or the iterator
// is exhausted. Advance the iterator to the next span which is
// guaranteed to lie at or past the search key.
iterSpan = iter.Next()
}
return iterSpan
}
// SeekLE seeks to the span that contains or is before the target key.
func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search

23
vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go generated vendored
View File

@@ -24,7 +24,8 @@ import (
// Note that the start user key is inclusive and the end user key is exclusive.
//
// Currently the only supported key kinds are:
// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
//
// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
type Span struct {
// Start and End encode the user key range of all the contained items, with
// an inclusive start key and exclusive end key. Both Start and End must be
@@ -75,6 +76,15 @@ func (k Key) SeqNum() uint64 {
return k.Trailer >> 8
}
// VisibleAt returns true if the provided key is visible at the provided
// snapshot sequence number. It interprets batch sequence numbers as always
// visible, because non-visible batch span keys are filtered when they're
// fragmented.
func (k Key) VisibleAt(snapshot uint64) bool {
seq := k.SeqNum()
return seq < snapshot || seq&base.InternalKeySeqNumBatch != 0
}
// Kind returns the kind component of the key.
func (k Key) Kind() base.InternalKeyKind {
return base.InternalKeyKind(k.Trailer & 0xff)
@@ -396,6 +406,17 @@ func SortKeysByTrailer(keys *[]Key) {
sort.Sort(sorted)
}
// KeysBySuffix implements sort.Interface, sorting its member Keys slice to by
// Suffix in the order dictated by Cmp.
type KeysBySuffix struct {
Cmp base.Compare
Keys []Key
}
func (s *KeysBySuffix) Len() int { return len(s.Keys) }
func (s *KeysBySuffix) Less(i, j int) bool { return s.Cmp(s.Keys[i].Suffix, s.Keys[j].Suffix) < 0 }
func (s *KeysBySuffix) Swap(i, j int) { s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] }
// ParseSpan parses the string representation of a Span. It's intended for
// tests. ParseSpan panics if passed a malformed span representation.
func ParseSpan(input string) Span {

18
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored
View File

@@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base"
// truncated to be contained within the range [lower, upper). If start and end
// are specified, filter out any spans that are completely outside those bounds.
func Truncate(
cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
cmp base.Compare,
iter FragmentIterator,
lower, upper []byte,
start, end *base.InternalKey,
panicOnPartialOverlap bool,
) FragmentIterator {
return Filter(iter, func(in *Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
// Ignore this span if it lies completely outside [start, end].
// Ignore this span if it lies completely outside start, end. Note that
// end endInclusive indicated whether end is inclusive.
//
// The comparison between s.End and start is by user key only, as
// the span is exclusive at s.End, so comparing by user keys
@@ -48,13 +53,22 @@ func Truncate(
// Wholly within the end bound. Keep it.
}
}
var truncated bool
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
truncated = true
out.Start = lower
}
if cmp(in.End, upper) > 0 {
truncated = true
out.End = upper
}
if panicOnPartialOverlap && truncated {
panic("pebble: bounds should not be truncated")
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}