Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go

@@ -34,17 +34,21 @@ func (it *flushIterator) String() string {
 	return "memtable"
 }
 
-func (it *flushIterator) SeekGE(key []byte, tryNextUsingSeek bool) (*base.InternalKey, []byte) {
+func (it *flushIterator) SeekGE(
+	key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
 	panic("pebble: SeekGE unimplemented")
 }
 
 func (it *flushIterator) SeekPrefixGE(
-	prefix, key []byte, trySeekUsingNext bool,
-) (*base.InternalKey, []byte) {
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
 	panic("pebble: SeekPrefixGE unimplemented")
 }
 
-func (it *flushIterator) SeekLT(key []byte) (*base.InternalKey, []byte) {
+func (it *flushIterator) SeekLT(
+	key []byte, flags base.SeekLTFlags,
+) (*base.InternalKey, base.LazyValue) {
 	panic("pebble: SeekLT unimplemented")
 }
 
@@ -52,10 +56,10 @@ func (it *flushIterator) SeekLT(key []byte) (*base.InternalKey, []byte) {
 // if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
 // that First only checks the upper bound. It is up to the caller to ensure
 // that key is greater than or equal to the lower bound.
-func (it *flushIterator) First() (*base.InternalKey, []byte) {
+func (it *flushIterator) First() (*base.InternalKey, base.LazyValue) {
 	key, val := it.Iterator.First()
 	if key == nil {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	*it.bytesIterated += uint64(it.nd.allocSize)
 	return key, val
@@ -65,16 +69,16 @@ func (it *flushIterator) First() (*base.InternalKey, []byte) {
 // iterator is pointing at a valid entry, and (nil, nil) otherwise.
 // Note: flushIterator.Next mirrors the implementation of Iterator.Next
 // due to performance. Keep the two in sync.
-func (it *flushIterator) Next() (*base.InternalKey, []byte) {
+func (it *flushIterator) Next() (*base.InternalKey, base.LazyValue) {
 	it.nd = it.list.getNext(it.nd, 0)
 	if it.nd == it.list.tail {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	it.decodeKey()
 	*it.bytesIterated += uint64(it.nd.allocSize)
-	return &it.key, it.value()
+	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
-func (it *flushIterator) Prev() (*base.InternalKey, []byte) {
+func (it *flushIterator) Prev() (*base.InternalKey, base.LazyValue) {
 	panic("pebble: Prev unimplemented")
 }

vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go

@@ -78,11 +78,11 @@ func (it *Iterator) Error() error {
 // pointing at a valid entry, and (nil, nil) otherwise. Note that SeekGE only
 // checks the upper bound. It is up to the caller to ensure that key is greater
 // than or equal to the lower bound.
-func (it *Iterator) SeekGE(key []byte, trySeekUsingNext bool) (*base.InternalKey, []byte) {
-	if trySeekUsingNext {
+func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	if flags.TrySeekUsingNext() {
 		if it.nd == it.list.tail {
 			// Iterator is done.
-			return nil, nil
+			return nil, base.LazyValue{}
 		}
 		less := it.list.cmp(it.key.UserKey, key) < 0
 		// Arbitrary constant. By measuring the seek cost as a function of the
@@ -94,24 +94,24 @@ func (it *Iterator) SeekGE(key []byte, trySeekUsingNext bool) (*base.InternalKey
 			k, _ := it.Next()
 			if k == nil {
 				// Iterator is done.
-				return nil, nil
+				return nil, base.LazyValue{}
 			}
 			less = it.list.cmp(it.key.UserKey, key) < 0
 		}
 		if !less {
-			return &it.key, it.value()
+			return &it.key, base.MakeInPlaceValue(it.value())
 		}
 	}
 	_, it.nd, _ = it.seekForBaseSplice(key)
 	if it.nd == it.list.tail {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	it.decodeKey()
 	if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
 		it.nd = it.list.tail
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
-	return &it.key, it.value()
+	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
 // SeekPrefixGE moves the iterator to the first entry whose key is greater than
@@ -119,94 +119,94 @@ func (it *Iterator) SeekGE(key []byte, trySeekUsingNext bool) (*base.InternalKey
 // provided so that an arenaskl.Iterator implements the
 // internal/base.InternalIterator interface.
 func (it *Iterator) SeekPrefixGE(
-	prefix, key []byte, trySeekUsingNext bool,
-) (*base.InternalKey, []byte) {
-	return it.SeekGE(key, trySeekUsingNext)
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return it.SeekGE(key, flags)
 }
 
 // SeekLT moves the iterator to the last entry whose key is less than the given
 // key. Returns the key and value if the iterator is pointing at a valid entry,
 // and (nil, nil) otherwise. Note that SeekLT only checks the lower bound. It
 // is up to the caller to ensure that key is less than the upper bound.
-func (it *Iterator) SeekLT(key []byte) (*base.InternalKey, []byte) {
+func (it *Iterator) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
 	// NB: the top-level Iterator has already adjusted key based on
 	// the upper-bound.
 	it.nd, _, _ = it.seekForBaseSplice(key)
 	if it.nd == it.list.head {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	it.decodeKey()
 	if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
 		it.nd = it.list.head
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
-	return &it.key, it.value()
+	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
 // First seeks position at the first entry in list. Returns the key and value
 // if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
 // that First only checks the upper bound. It is up to the caller to ensure
 // that key is greater than or equal to the lower bound (e.g. via a call to SeekGE(lower)).
-func (it *Iterator) First() (*base.InternalKey, []byte) {
+func (it *Iterator) First() (*base.InternalKey, base.LazyValue) {
 	it.nd = it.list.getNext(it.list.head, 0)
 	if it.nd == it.list.tail {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	it.decodeKey()
 	if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
 		it.nd = it.list.tail
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
-	return &it.key, it.value()
+	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
 // Last seeks position at the last entry in list. Returns the key and value if
 // the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
 // that Last only checks the lower bound. It is up to the caller to ensure that
 // key is less than the upper bound (e.g. via a call to SeekLT(upper)).
-func (it *Iterator) Last() (*base.InternalKey, []byte) {
+func (it *Iterator) Last() (*base.InternalKey, base.LazyValue) {
 	it.nd = it.list.getPrev(it.list.tail, 0)
 	if it.nd == it.list.head {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	it.decodeKey()
 	if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
 		it.nd = it.list.head
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
-	return &it.key, it.value()
+	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
 // Next advances to the next position. Returns the key and value if the
 // iterator is pointing at a valid entry, and (nil, nil) otherwise.
 // Note: flushIterator.Next mirrors the implementation of Iterator.Next
 // due to performance. Keep the two in sync.
-func (it *Iterator) Next() (*base.InternalKey, []byte) {
+func (it *Iterator) Next() (*base.InternalKey, base.LazyValue) {
 	it.nd = it.list.getNext(it.nd, 0)
 	if it.nd == it.list.tail {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	it.decodeKey()
 	if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
 		it.nd = it.list.tail
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
-	return &it.key, it.value()
+	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
 // Prev moves to the previous position. Returns the key and value if the
 // iterator is pointing at a valid entry, and (nil, nil) otherwise.
-func (it *Iterator) Prev() (*base.InternalKey, []byte) {
+func (it *Iterator) Prev() (*base.InternalKey, base.LazyValue) {
 	it.nd = it.list.getPrev(it.nd, 0)
 	if it.nd == it.list.head {
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
 	it.decodeKey()
 	if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
 		it.nd = it.list.head
-		return nil, nil
+		return nil, base.LazyValue{}
 	}
-	return &it.key, it.value()
+	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
 // value returns the value at the current position.

vendor/github.com/cockroachdb/pebble/internal/base/comparer.go

@@ -66,6 +66,23 @@ type Separator func(dst, a, b []byte) []byte
 // key must be valid to pass to Compare.
 type Successor func(dst, a []byte) []byte
 
+// ImmediateSuccessor is invoked with a prefix key ([Split(a) == len(a)]) and
+// returns the smallest key that is larger than the given prefix a.
+// ImmediateSuccessor must return a prefix key k such that:
+//
+//   Split(k) == len(k) and Compare(k, a) > 0
+//
+// and there exists no representable k2 such that:
+//
+//   Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
+//
+// As an example, an implementation built on the natural byte ordering using
+// bytes.Compare could append a `\0` to `a`.
+//
+// The dst parameter may be used to store the returned key, though it is valid
+// to pass nil. The returned key must be valid to pass to Compare.
+type ImmediateSuccessor func(dst, a []byte) []byte
+
 // Split returns the length of the prefix of the user key that corresponds to
 // the key portion of an MVCC encoding scheme to enable the use of prefix bloom
 // filters.
@@ -104,14 +121,15 @@ type Split func(a []byte) int
 // Comparer defines a total ordering over the space of []byte keys: a 'less
 // than' relationship.
 type Comparer struct {
-	Compare        Compare
-	Equal          Equal
-	AbbreviatedKey AbbreviatedKey
-	FormatKey      FormatKey
-	FormatValue    FormatValue
-	Separator      Separator
-	Split          Split
-	Successor      Successor
+	Compare            Compare
+	Equal              Equal
+	AbbreviatedKey     AbbreviatedKey
+	FormatKey          FormatKey
+	FormatValue        FormatValue
+	Separator          Separator
+	Split              Split
+	Successor          Successor
+	ImmediateSuccessor ImmediateSuccessor
 
 	// Name is the name of the comparer.
 	//
@@ -193,6 +211,10 @@ var DefaultComparer = &Comparer{
 		return append(dst, a...)
 	},
 
+	ImmediateSuccessor: func(dst, a []byte) (ret []byte) {
+		return append(append(dst, a...), 0x00)
+	},
+
 	// This name is part of the C++ Level-DB implementation's default file
 	// format, and should not be changed.
 	Name: "leveldb.BytewiseComparator",

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -71,6 +71,10 @@ const (
 	// seqNum.
 	InternalKeyKindMax InternalKeyKind = 21
 
+	// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
+	// zero sequence number.
+	InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid)
+
 	// A marker for an invalid key.
 	InternalKeyKindInvalid InternalKeyKind = 255
 
@@ -329,14 +333,28 @@ func (k InternalKey) SeqNum() uint64 {
 
 // Visible returns true if the key is visible at the specified snapshot
 // sequence number.
-func (k InternalKey) Visible(snapshot uint64) bool {
-	return Visible(k.SeqNum(), snapshot)
+func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
+	return Visible(k.SeqNum(), snapshot, batchSnapshot)
 }
 
 // Visible returns true if a key with the provided sequence number is visible at
-// the specified snapshot sequence number.
-func Visible(seqNum uint64, snapshot uint64) bool {
-	return seqNum < snapshot || (seqNum&InternalKeySeqNumBatch) != 0
+// the specified snapshot sequence numbers.
+func Visible(seqNum uint64, snapshot, batchSnapshot uint64) bool {
+	// There are two snapshot sequence numbers, one for committed keys and one
+	// for batch keys. If a seqNum is less than `snapshot`, then seqNum
+	// corresponds to a committed key that is visible. If seqNum has its batch
+	// bit set, then seqNum corresponds to an uncommitted batch key. Its
+	// visible if its snapshot is less than batchSnapshot.
+	//
+	// There's one complication. The maximal sequence number
+	// (`InternalKeySeqNumMax`) is used across Pebble for exclusive sentinel
+	// keys and other purposes. The maximal sequence number has its batch bit
+	// set, but it can never be < `batchSnapshot`, since there is no expressible
+	// larger snapshot. We dictate that the maximal sequence number is always
+	// visible.
+	return seqNum < snapshot ||
+		((seqNum&InternalKeySeqNumBatch) != 0 && seqNum < batchSnapshot) ||
+		seqNum == InternalKeySeqNumMax
 }
 
 // SetKind sets the kind component of the key.

vendor/github.com/cockroachdb/pebble/internal/base/iterator.go

@@ -85,33 +85,19 @@ import "fmt"
 // them through the Error method. All of the absolute positioning methods
 // reset any accumulated error before positioning. Relative positioning
 // methods return without advancing if the iterator has accumulated an error.
+//
+// nilv == shorthand for LazyValue{}, which represents a nil value.
 type InternalIterator interface {
 	// SeekGE moves the iterator to the first key/value pair whose key is greater
 	// than or equal to the given key. Returns the key and value if the iterator
-	// is pointing at a valid entry, and (nil, nil) otherwise. Note that SeekGE
+	// is pointing at a valid entry, and (nil, nilv) otherwise. Note that SeekGE
 	// only checks the upper bound. It is up to the caller to ensure that key
 	// is greater than or equal to the lower bound.
-	//
-	// trySeekUsingNext is a performance optimization that callers can use to
-	// indicate that they have not done any action to move this iterator beyond
-	// the first key that would be found if this iterator were to honestly do
-	// the intended seek. For example, say the caller did a
-	// SeekGE(k1...), followed by SeekGE(k2...) where k1 <= k2, without any
-	// intermediate positioning calls. The caller can safely specify true for this
-	// parameter in the second call. As another example, say the caller did do one
-	// call to Next between the two Seek calls, and k1 < k2. Again, the caller can
-	// safely specify a true value for this parameter. Note that a false value is
-	// always safe. The callee is free to ignore the true value if its
-	// implementation does not permit this optimization.
-	// - We make the caller do this determination since a string comparison of
-	//   k1, k2 is not necessarily cheap, and there may be many iterators in
-	//   the iterator stack. Doing it once at the root of the iterator stack
-	//   is cheaper.
-	SeekGE(key []byte, trySeekUsingNext bool) (*InternalKey, []byte)
+	SeekGE(key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
 
 	// SeekPrefixGE moves the iterator to the first key/value pair whose key is
 	// greater than or equal to the given key. Returns the key and value if the
-	// iterator is pointing at a valid entry, and (nil, nil) otherwise. Note that
+	// iterator is pointing at a valid entry, and (nil, nilv) otherwise. Note that
 	// SeekPrefixGE only checks the upper bound. It is up to the caller to ensure
 	// that key is greater than or equal to the lower bound.
 	//
@@ -120,7 +106,7 @@ type InternalIterator interface {
 	// function must be supplied to the Comparer for the DB. The supplied prefix
 	// will be the prefix of the given key returned by that Split function. If
 	// the iterator is able to determine that no key with the prefix exists, it
-	// can return (nil,nil). Unlike SeekGE, this is not an indication that
+	// can return (nil,nilv). Unlike SeekGE, this is not an indication that
 	// iteration is exhausted.
 	//
 	// Note that the iterator may return keys not matching the prefix. It is up
@@ -132,78 +118,53 @@ type InternalIterator interface {
 	// not supporting reverse iteration in prefix iteration mode until a
 	// different positioning routine (SeekGE, SeekLT, First or Last) switches the
 	// iterator out of prefix iteration.
-	//
-	// trySeekUsingNext is a performance optimization that callers can use to
-	// indicate that they have not done any action to move this iterator
-	// beyond the first key that would be found if this iterator were to
-	// honestly do the indicated seek. For example, say the caller did a
-	// SeekPrefixGE(k1...), followed by SeekPrefixGE(k2...) where k1 <= k2,
-	// without any intermediate positioning calls. The caller can safely
-	// specify true for this parameter in the second call. As another example,
-	// say the caller did do one call to Next between the two SeekPrefixGE
-	// calls, and k1 < k2. Again, the caller can safely specify a true value
-	// for this parameter. Note that a false value is always safe. The callee
-	// is free to ignore the true value if its implementation does not permit
-	// this optimization.
-	// - We make the caller do this determination since a string comparison of
-	//   k1, k2 is not necessarily cheap, and there may be many iterators in
-	//   the iterator stack. Doing it once at the root of the iterator stack
-	//   is cheaper.
-	// - This optimization could also be applied to SeekLT (where it would be
-	//   trySeekUsingPrev). We currently only do it for SeekPrefixGE and SeekGE
-	//   because this is where this optimization helps the performance of
-	//   CockroachDB. The SeekLT cases in CockroachDB are typically accompanied
-	//   with bounds that change between seek calls, and is optimized inside
-	//   certain iterator implementations, like singleLevelIterator, without any
-	//   extra parameter passing (though the same amortization of string
-	//   comparisons could be done to improve that optimization, by making the
-	//   root of the iterator stack do it).
-	SeekPrefixGE(prefix, key []byte, trySeekUsingNext bool) (*InternalKey, []byte)
+	SeekPrefixGE(prefix, key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
 
 	// SeekLT moves the iterator to the last key/value pair whose key is less
 	// than the given key. Returns the key and value if the iterator is pointing
-	// at a valid entry, and (nil, nil) otherwise. Note that SeekLT only checks
+	// at a valid entry, and (nil, nilv) otherwise. Note that SeekLT only checks
 	// the lower bound. It is up to the caller to ensure that key is less than
 	// the upper bound.
-	SeekLT(key []byte) (*InternalKey, []byte)
+	SeekLT(key []byte, flags SeekLTFlags) (*InternalKey, LazyValue)
 
 	// First moves the iterator the the first key/value pair. Returns the key and
-	// value if the iterator is pointing at a valid entry, and (nil, nil)
+	// value if the iterator is pointing at a valid entry, and (nil, nilv)
 	// otherwise. Note that First only checks the upper bound. It is up to the
 	// caller to ensure that First() is not called when there is a lower bound,
 	// and instead call SeekGE(lower).
-	First() (*InternalKey, []byte)
+	First() (*InternalKey, LazyValue)
 
 	// Last moves the iterator the the last key/value pair. Returns the key and
-	// value if the iterator is pointing at a valid entry, and (nil, nil)
+	// value if the iterator is pointing at a valid entry, and (nil, nilv)
 	// otherwise. Note that Last only checks the lower bound. It is up to the
 	// caller to ensure that Last() is not called when there is an upper bound,
 	// and instead call SeekLT(upper).
-	Last() (*InternalKey, []byte)
+	Last() (*InternalKey, LazyValue)
 
 	// Next moves the iterator to the next key/value pair. Returns the key and
-	// value if the iterator is pointing at a valid entry, and (nil, nil)
+	// value if the iterator is pointing at a valid entry, and (nil, nilv)
 	// otherwise. Note that Next only checks the upper bound. It is up to the
 	// caller to ensure that key is greater than or equal to the lower bound.
 	//
 	// It is valid to call Next when the iterator is positioned before the first
 	// key/value pair due to either a prior call to SeekLT or Prev which returned
-	// (nil, nil). It is not allowed to call Next when the previous call to SeekGE,
-	// SeekPrefixGE or Next returned (nil, nil).
-	Next() (*InternalKey, []byte)
+	// (nil, nilv). It is not allowed to call Next when the previous call to SeekGE,
+	// SeekPrefixGE or Next returned (nil, nilv).
+	Next() (*InternalKey, LazyValue)
 
 	// Prev moves the iterator to the previous key/value pair. Returns the key
-	// and value if the iterator is pointing at a valid entry, and (nil, nil)
+	// and value if the iterator is pointing at a valid entry, and (nil, nilv)
 	// otherwise. Note that Prev only checks the lower bound. It is up to the
 	// caller to ensure that key is less than the upper bound.
 	//
 	// It is valid to call Prev when the iterator is positioned after the last
 	// key/value pair due to either a prior call to SeekGE or Next which returned
-	// (nil, nil). It is not allowed to call Prev when the previous call to SeekLT
-	// or Prev returned (nil, nil).
-	Prev() (*InternalKey, []byte)
+	// (nil, nilv). It is not allowed to call Prev when the previous call to SeekLT
+	// or Prev returned (nil, nilv).
+	Prev() (*InternalKey, LazyValue)
 
-	// Error returns any accumulated error.
+	// Error returns any accumulated error. It may not include errors returned
+	// to the client when calling LazyValue.Value().
 	Error() error
 
 	// Close closes the iterator and returns any accumulated error. Exhausting
@@ -225,11 +186,137 @@ type InternalIterator interface {
 	fmt.Stringer
 }
 
-// InternalIteratorWithStats extends InternalIterator to expose stats.
-type InternalIteratorWithStats interface {
-	InternalIterator
-	Stats() InternalIteratorStats
-	ResetStats()
+// SeekGEFlags holds flags that may configure the behavior of a forward seek.
+// Not all flags are relevant to all iterators.
+type SeekGEFlags uint8
+
+const (
+	seekGEFlagTrySeekUsingNext uint8 = iota
+	seekGEFlagRelativeSeek
+	seekGEFlagBatchJustRefreshed
+)
+
+// SeekGEFlagsNone is the default value of SeekGEFlags, with all flags disabled.
+const SeekGEFlagsNone = SeekGEFlags(0)
+
+// TrySeekUsingNext indicates whether a performance optimization was enabled
+// by a caller, indicating the caller has not done any action to move this
+// iterator beyond the first key that would be found if this iterator were to
+// honestly do the intended seek. For example, say the caller did a
+// SeekGE(k1...), followed by SeekGE(k2...) where k1 <= k2, without any
+// intermediate positioning calls. The caller can safely specify true for this
+// parameter in the second call. As another example, say the caller did do one
+// call to Next between the two Seek calls, and k1 < k2. Again, the caller can
+// safely specify a true value for this parameter. Note that a false value is
+// always safe. The callee is free to ignore the true value if its
+// implementation does not permit this optimization.
+//
+// We make the caller do this determination since a string comparison of k1, k2
+// is not necessarily cheap, and there may be many iterators in the iterator
+// stack. Doing it once at the root of the iterator stack is cheaper.
+//
+// This optimization could also be applied to SeekLT (where it would be
+// trySeekUsingPrev). We currently only do it for SeekPrefixGE and SeekGE
+// because this is where this optimization helps the performance of CockroachDB.
+// The SeekLT cases in CockroachDB are typically accompanied with bounds that
+// change between seek calls, and is optimized inside certain iterator
+// implementations, like singleLevelIterator, without any extra parameter
+// passing (though the same amortization of string comparisons could be done to
+// improve that optimization, by making the root of the iterator stack do it).
+func (s SeekGEFlags) TrySeekUsingNext() bool { return (s & (1 << seekGEFlagTrySeekUsingNext)) != 0 }
+
+// RelativeSeek is set when in the course of a forward positioning operation, a
+// higher-level iterator seeks a lower-level iterator to a larger key than the
+// one at the current iterator position.
+//
+// Concretely, this occurs when the merging iterator observes a range deletion
+// covering the key at a level's current position, and the merging iterator
+// seeks the level to the range deletion's end key. During lazy-combined
+// iteration, this flag signals to the level iterator that the seek is NOT an
+// absolute-positioning operation from the perspective of the pebble.Iterator,
+// and the level iterator must look for range keys in tables between the current
+// iterator position and the new seeked position.
+func (s SeekGEFlags) RelativeSeek() bool { return (s & (1 << seekGEFlagRelativeSeek)) != 0 }
+
+// BatchJustRefreshed is set by Seek[Prefix]GE when an iterator's view of an
+// indexed batch was just refreshed. It serves as a signal to the batch iterator
+// to ignore the TrySeekUsingNext optimization, because the external knowledge
+// imparted by the TrySeekUsingNext flag does not apply to the batch iterator's
+// position. See (pebble.Iterator).batchJustRefreshed.
+func (s SeekGEFlags) BatchJustRefreshed() bool { return (s & (1 << seekGEFlagBatchJustRefreshed)) != 0 }
+
+// EnableTrySeekUsingNext returns the provided flags with the
+// try-seek-using-next optimization enabled. See TrySeekUsingNext for an
+// explanation of this optimization.
+func (s SeekGEFlags) EnableTrySeekUsingNext() SeekGEFlags {
+	return s | (1 << seekGEFlagTrySeekUsingNext)
+}
+
+// DisableTrySeekUsingNext returns the provided flags with the
+// try-seek-using-next optimization disabled.
+func (s SeekGEFlags) DisableTrySeekUsingNext() SeekGEFlags {
+	return s &^ (1 << seekGEFlagTrySeekUsingNext)
+}
+
+// EnableRelativeSeek returns the provided flags with the relative-seek flag
+// enabled. See RelativeSeek for an explanation of this flag's use.
+func (s SeekGEFlags) EnableRelativeSeek() SeekGEFlags {
+	return s | (1 << seekGEFlagRelativeSeek)
+}
+
+// DisableRelativeSeek returns the provided flags with the relative-seek flag
+// disabled.
+func (s SeekGEFlags) DisableRelativeSeek() SeekGEFlags {
+	return s &^ (1 << seekGEFlagRelativeSeek)
+}
+
+// EnableBatchJustRefreshed returns the provided flags with the
+// batch-just-refreshed bit set. See BatchJustRefreshed for an explanation of
+// this flag.
+func (s SeekGEFlags) EnableBatchJustRefreshed() SeekGEFlags {
+	return s | (1 << seekGEFlagBatchJustRefreshed)
+}
+
+// DisableBatchJustRefreshed returns the provided flags with the
+// batch-just-refreshed bit unset.
+func (s SeekGEFlags) DisableBatchJustRefreshed() SeekGEFlags {
+	return s &^ (1 << seekGEFlagBatchJustRefreshed)
+}
+
+// SeekLTFlags holds flags that may configure the behavior of a reverse seek.
+// Not all flags are relevant to all iterators.
+type SeekLTFlags uint8
+
+const (
+	seekLTFlagRelativeSeek uint8 = iota
+)
+
+// SeekLTFlagsNone is the default value of SeekLTFlags, with all flags disabled.
+const SeekLTFlagsNone = SeekLTFlags(0)
+
+// RelativeSeek is set when in the course of a reverse positioning operation, a
+// higher-level iterator seeks a lower-level iterator to a smaller key than the
+// one at the current iterator position.
+//
+// Concretely, this occurs when the merging iterator observes a range deletion
+// covering the key at a level's current position, and the merging iterator
+// seeks the level to the range deletion's start key. During lazy-combined
+// iteration, this flag signals to the level iterator that the seek is NOT an
+// absolute-positioning operation from the perspective of the pebble.Iterator,
+// and the level iterator must look for range keys in tables between the current
+// iterator position and the new seeked position.
+func (s SeekLTFlags) RelativeSeek() bool { return s&(1<<seekLTFlagRelativeSeek) != 0 }
+
+// EnableRelativeSeek returns the provided flags with the relative-seek flag
+// enabled. See RelativeSeek for an explanation of this flag's use.
+func (s SeekLTFlags) EnableRelativeSeek() SeekLTFlags {
+	return s | (1 << seekLTFlagRelativeSeek)
+}
+
+// DisableRelativeSeek returns the provided flags with the relative-seek flag
+// disabled.
+func (s SeekLTFlags) DisableRelativeSeek() SeekLTFlags {
+	return s &^ (1 << seekLTFlagRelativeSeek)
 }
 
 // InternalIteratorStats contains miscellaneous stats produced by
@@ -238,8 +325,8 @@ type InternalIteratorWithStats interface {
 // are aggregated as one goes up the InternalIterator tree.
 type InternalIteratorStats struct {
 	// Bytes in the loaded blocks. If the block was compressed, this is the
-	// compressed bytes. Currently, only the second-level index and data blocks
-	// containing points are included.
+	// compressed bytes. Currently, only the index blocks, data blocks
+	// containing points, and filter blocks are included.
 	BlockBytes uint64
 	// Subset of BlockBytes that were in the block cache.
 	BlockBytesInCache uint64
@@ -271,30 +358,3 @@ func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
 	s.PointCount += from.PointCount
 	s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones
 }
-
-type internalIteratorWithEmptyStats struct {
-	InternalIterator
-}
-
-var _ InternalIteratorWithStats = internalIteratorWithEmptyStats{}
-
-// Stats implements InternalIteratorWithStats.
-func (i internalIteratorWithEmptyStats) Stats() InternalIteratorStats {
-	return InternalIteratorStats{}
-}
-
-// ResetStats implements InternalIteratorWithStats.
-func (i internalIteratorWithEmptyStats) ResetStats() {}
-
-// WrapIterWithStats ensures that either iter implements the stats methods or
-// wraps it, such that the return value implements InternalIteratorWithStats.
-func WrapIterWithStats(iter InternalIterator) InternalIteratorWithStats {
-	if iter == nil {
-		return nil
-	}
-	i, ok := iter.(InternalIteratorWithStats)
-	if ok {
-		return i
-	}
-	return internalIteratorWithEmptyStats{InternalIterator: iter}
-}

vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go

@@ -0,0 +1,256 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import "github.com/cockroachdb/pebble/internal/invariants"
+
+// A value can have user-defined attributes that are a function of the value
+// byte slice. For now, we only support "short attributes", which can be
+// encoded in 3 bits. We will likely extend this to "long attributes" later
+// for values that are even more expensive to access than those in value
+// blocks in the same sstable.
+//
+// When a sstable writer chooses not to store a value together with the key,
+// it can call the ShortAttributeExtractor to extract the attribute and store
+// it together with the key. This allows for cheap retrieval of
+// AttributeAndLen on the read-path, without doing a more expensive retrieval
+// of the value.
+//
+// In general, the extraction code may want to also look at the key to decide
+// how to treat the value. Our current needs can be satisfied by configuring a
+// keyspan for which the ShortAttributeExtractor should be used, so we do not
+// expose the key to the extractor.
+//
+// Write path performance: The ShortAttributeExtractor func cannot be inlined,
+// so we will pay the cost of this function call. However, we will only pay
+// this when (a) the value is not being stored together with the key, and (b)
+// the key-value pair is being initially written to the DB, or a compaction is
+// transitioning the key-value pair from being stored together to being stored
+// separately.
+
+// ShortAttribute encodes a user-specified attribute of the value.
+type ShortAttribute uint8
+
+// MaxShortAttribute is the maximum value of the short attribute (3 bits).
+const MaxShortAttribute = 7
+
+// ShortAttributeExtractor is an extractor that given the value, will return
+// the ShortAttribute.
+type ShortAttributeExtractor func(value []byte) (ShortAttribute, error)
+
+// AttributeAndLen represents the pair of value length and the short
+// attribute.
+type AttributeAndLen struct {
+	ValueLen       int32
+	ShortAttribute ShortAttribute
+}
+
+// LazyValue represents a value that may not already have been extracted.
+// Currently, it can represent either an in-place value (stored with the key)
+// or a value stored in the value section. However, the interface is general
+// enough to support values that are stored in separate files.
+//
+// LazyValue is used in the InternalIterator interface, such that all
+// positioning calls return (*InternalKey, LazyValue). It is also exposed via
+// the public Iterator for callers that need to remember a recent but not
+// necessarily latest LazyValue, in case they need the actual value in the
+// future. An example is a caller that is iterating in reverse and looking for
+// the latest MVCC version for a key -- it cannot identify the latest MVCC
+// version without stepping to the previous key-value pair e.g.
+// storage.pebbleMVCCScanner in CockroachDB.
+//
+// Performance note: It is important for this struct to not exceed a sizeof 32
+// bytes, for optimizing the common case of the in-place value. Prior to
+// introducing LazyValue, we were passing around a []byte which is 24 bytes.
+// Passing a 40 byte or larger struct causes performance to drop by 75% on
+// some benchmarks that do tight iteration loops.
+//
+// Memory management:
+// This is subtle, but important for performance.
+//
+// - A LazyValue returned by an InternalIterator or Iterator is unstable in
+//   that repositioning the iterator will invalidate the memory inside it. A
+//   caller wishing to maintain that LazyValue needs to call
+//   LazyValue.Clone(). Note that this does not fetch the value if it is not
+//   in-place. Additionally, Clone() *must* only be called if
+//   LazyValue.Value() has *not* been called, since (a) it only makes a
+//   promise about stability of LazyValue and not the underlying value, (b) if
+//   LazyValue.Value() has been called already, the caller has the value
+//   []byte slice so it is unnecessary to fiddle around with LazyValue.
+//
+// - A user of an iterator that calls LazyValue.Value() wants as much as
+//   possible for the returned value []byte to point to iterator owned memory.
+//   - [P1] The underlying iterator that owns that memory also needs a promise
+//     from that user that at any time there is at most one value []byte slice
+//     that the caller is expecting it to maintain. Otherwise, the underlying
+//     iterator has to maintain multiple such []byte slices which results in
+//     more complicated and inefficient code.
+//   - [P2] The underlying iterator, in order to make the promise that it is
+//     maintaining the one value []byte slice, also needs a way to know when
+//     it is relieved of that promise. One way it is relieved of that promise
+//     is by being told that it is being repositioned. Typically, the owner of
+//     the value []byte slice is a sstable iterator, and it will know that it
+//     is relieved of the promise when it is repositioned. However, consider
+//     the case where the caller has used LazyValue.Clone() and repositioned
+//     the iterator (which is actually a tree of iterators). In this case the
+//     underlying sstable iterator may not even be open. LazyValue.Value()
+//     will still work (at a higher cost), but since the sstable iterator is
+//     not open, it does not have a mechanism to know when the retrieved value
+//     is no longer in use. We refer to this situation as "not satisfying P2".
+//     To handle this situation, the LazyValue.Value() method accepts a caller
+//     owned buffer, that the callee will use if needed. The callee explicitly
+//     tells the caller whether the []byte slice for the value is now owned by
+//     the caller. This will be true if the callee attempted to use buf and
+//     either successfully used it or allocated a new []byte slice.
+//
+//  To ground the above in reality, we consider three examples of callers of
+//  LazyValue.Value():
+//  - Iterator: it calls LazyValue.Value for its own use when merging values.
+//    When merging during reverse iteration, it may have cloned the LazyValue.
+//    In this case it calls LazyValue.Value() on the cloned value, merges it,
+//    and then calls LazyValue.Value() on the current iterator position and
+//    merges it. So it is honoring P1.
+//  - Iterator on behalf of Iterator clients: The Iterator.Value() method
+//    needs to call LazyValue.Value(). The client of Iterator is satisfying P1
+//    because of the inherent Iterator interface constraint, i.e., it is calling
+//    Iterator.Value() on the current Iterator position. It is possible that
+//    the Iterator has cloned this LazyValue (for the reverse iteration case),
+//    which the client is unaware of, so the underlying sstable iterator may
+//    not be able to satisfy P2. This is ok because Iterator will call
+//    LazyValue.Value with its (reusable) owned buffer.
+//  - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
+//    since during reverse iteration in order to find the highest version that
+//    satisfies a read it needs to clone the LazyValue, step back the iterator
+//    and then decide whether it needs the value from the previously cloned
+//    LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
+//    similar to the previous case in that it will call LazyValue.Value with
+//    its (reusable) owned buffer.
+//
+// Corollary: callers that directly use InternalIterator can know that they
+// have done nothing to interfere with promise P2 can pass in a nil buf and be
+// sure that it will not trigger an allocation.
+//
+// Repeated calling of LazyValue.Value:
+// This is ok as long as the caller continues to satisfy P1. The previously
+// fetched value will be remembered inside LazyValue to avoid fetching again.
+// So if the caller's buffer is used the first time the value was fetched, it
+// is still in use.
+//
+// LazyValue fields are visible outside the package for use in
+// InternalIterator implementations and in Iterator, but not meant for direct
+// use by users of Pebble.
+type LazyValue struct {
+	// ValueOrHandle represents a value, or a handle to be passed to ValueFetcher.
+	// - Fetcher == nil: ValueOrHandle is a value.
+	// - Fetcher != nil: ValueOrHandle is a handle and Fetcher.Attribute is
+	//   initialized.
+	// The ValueOrHandle exposed by InternalIterator or Iterator may not be stable
+	// if the iterator is stepped. To make it stable, make a copy using Clone.
+	ValueOrHandle []byte
+	// Fetcher provides support for fetching an actually lazy value.
+	Fetcher *LazyFetcher
+}
+
+// LazyFetcher supports fetching a lazy value.
+type LazyFetcher struct {
+	// ValueFetcher, given a handle, returns the value. It is acceptable to call
+	// the ValueFetcher as long as the DB is open. However, one should assume
+	// there is a fast-path when the iterator tree has not moved off the sstable
+	// iterator that initially provided this LazyValue. Hence, to utilize this
+	// fast-path the caller should try to decide whether it needs the value or
+	// not as soon as possible, with minimal possible stepping of the iterator.
+	//
+	// buf will be used if the fetcher cannot satisfy P2 (see earlier comment).
+	// If the fetcher attempted to use buf *and* len(buf) was insufficient, it
+	// will allocate a new slice for the value. In either case it will set
+	// callerOwned to true.
+	ValueFetcher func(handle []byte, buf []byte) (val []byte, callerOwned bool, err error)
+	// Attribute includes the short attribute and value length.
+	Attribute   AttributeAndLen
+	fetched     bool
+	value       []byte
+	callerOwned bool
+	err         error
+}
+
+// Value returns the underlying value.
+func (lv *LazyValue) Value(buf []byte) (val []byte, callerOwned bool, err error) {
+	if lv.Fetcher == nil {
+		return lv.ValueOrHandle, false, nil
+	}
+	// Do the rest of the work in a separate method to attempt mid-stack
+	// inlining of Value(). Unfortunately, this still does not inline since the
+	// cost of 85 exceeds the budget of 80.
+	//
+	// TODO(sumeer): Packing the return values into a struct{[]byte error bool}
+	// causes it to be below the budget. Consider this if we need to recover
+	// more performance. I suspect that inlining this only matters in
+	// micro-benchmarks, and in actual use cases in CockroachDB it will not
+	// matter because there is substantial work done with a fetched value.
+	return lv.fetchValue(buf)
+}
+
+// INVARIANT: lv.Fetcher != nil
+func (lv *LazyValue) fetchValue(buf []byte) (val []byte, callerOwned bool, err error) {
+	f := lv.Fetcher
+	if !f.fetched {
+		f.fetched = true
+		f.value, f.callerOwned, f.err = lv.Fetcher.ValueFetcher(lv.ValueOrHandle, buf)
+	}
+	return f.value, f.callerOwned, f.err
+}
+
+// InPlaceValue returns the value under the assumption that it is in-place.
+// This is for Pebble-internal code.
+func (lv *LazyValue) InPlaceValue() []byte {
+	if invariants.Enabled && lv.Fetcher != nil {
+		panic("value must be in-place")
+	}
+	return lv.ValueOrHandle
+}
+
+// Len returns the length of the value.
+func (lv *LazyValue) Len() int {
+	if lv.Fetcher == nil {
+		return len(lv.ValueOrHandle)
+	}
+	return int(lv.Fetcher.Attribute.ValueLen)
+}
+
+// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
+// REQUIRES: LazyValue.Value() has not been called.
+// Ensure that this is only called before LazyValue.Value is called. We don't
+// actually care if it was in-place but we don't want confusing things about whether
+// we need to clone the value inside the fetcher.
+func (lv *LazyValue) Clone(buf []byte) (LazyValue, []byte) {
+	if invariants.Enabled && lv.Fetcher != nil && lv.Fetcher.fetched {
+		panic("value has already been fetched")
+	}
+	// INVARIANT: LazyFetcher.value is nil, so there is nothing to copy there.
+	vLen := len(lv.ValueOrHandle)
+	lvCopy := LazyValue{Fetcher: lv.Fetcher}
+	if vLen == 0 {
+		return lvCopy, buf
+	}
+	bufLen := len(buf)
+	buf = append(buf, lv.ValueOrHandle...)
+	lvCopy.ValueOrHandle = buf[bufLen : bufLen+vLen]
+	return lvCopy, buf
+}
+
+// MakeInPlaceValue constructs an in-place value.
+func MakeInPlaceValue(val []byte) LazyValue {
+	return LazyValue{ValueOrHandle: val}
+}
+
+// TODO(sumeer): test that callers are adhering to promise P1. In the sstable
+// iterators, when invariants are enabled, create a testingFetcher struct that
+// contains the sstable name and the []byte backing for the last fetched
+// value. The sstable iterator, instead of returning an in-place value, will
+// return a copy of the key in ValueOrHandle, and a ValueFetcher implemented
+// by testingFetcher.valueFetcher. Each time this method is called, it will
+// mangle the last fetched value []byte slice, allocate a new one, open the
+// sstable, seek to the key, and copy the value into this new byte slice and
+// return.

vendor/github.com/cockroachdb/pebble/internal/base/metrics.go

@@ -29,6 +29,13 @@ func (tm *ThroughputMetric) Merge(x ThroughputMetric) {
 	tm.IdleDuration += x.IdleDuration
 }
 
+// Subtract subtracts the information from another ThroughputMetric
+func (tm *ThroughputMetric) Subtract(x ThroughputMetric) {
+	tm.Bytes -= x.Bytes
+	tm.WorkDuration -= x.WorkDuration
+	tm.IdleDuration -= x.IdleDuration
+}
+
 // PeakRate returns the approximate peak rate if there was no idling.
 func (tm *ThroughputMetric) PeakRate() int64 {
 	if tm.Bytes == 0 {
@@ -67,6 +74,12 @@ func (gsm *GaugeSampleMetric) Merge(x GaugeSampleMetric) {
 	gsm.count += x.count
 }
 
+// Subtract subtracts the information from another gauge metric.
+func (gsm *GaugeSampleMetric) Subtract(x GaugeSampleMetric) {
+	gsm.sampleSum -= x.sampleSum
+	gsm.count -= x.count
+}
+
 // Mean returns the mean value.
 func (gsm *GaugeSampleMetric) Mean() float64 {
 	if gsm.count == 0 {

vendor/github.com/cockroachdb/pebble/internal/batchskl/iterator.go

@@ -47,7 +47,31 @@ func (it *Iterator) Close() error {
 // entry and false otherwise. Note that SeekGE only checks the upper bound. It
 // is up to the caller to ensure that key is greater than or equal to the lower
 // bound.
-func (it *Iterator) SeekGE(key []byte) *base.InternalKey {
+func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) *base.InternalKey {
+	if flags.TrySeekUsingNext() {
+		if it.nd == it.list.tail {
+			// Iterator is done.
+			return nil
+		}
+		less := it.list.cmp(it.key.UserKey, key) < 0
+		// Arbitrary constant. By measuring the seek cost as a function of the
+		// number of elements in the skip list, and fitting to a model, we
+		// could adjust the number of nexts based on the current size of the
+		// skip list.
+		const numNexts = 5
+		for i := 0; less && i < numNexts; i++ {
+			k := it.Next()
+			if k == nil {
+				// Iterator is done.
+				return nil
+			}
+			less = it.list.cmp(k.UserKey, key) < 0
+		}
+		if !less {
+			return &it.key
+		}
+	}
+
 	_, it.nd = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
 	if it.nd == it.list.tail {
 		return nil

vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go

@@ -0,0 +1,265 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// TODO(jackson): Consider removing this type and adding bounds enforcement
+// directly to the MergingIter. This type is probably too lightweight to warrant
+// its own type, but for now we implement it separately for expediency.
+
+// boundedIterPos records the position of the BoundedIter relative to the
+// underlying iterator's position. It's used to avoid Next/Prev-ing the iterator
+// if there can't possibly be another span within bounds, because the current
+// span overlaps the bound.
+//
+// Imagine bounds [a,c) and an iterator that seeks to a span [b,d). The span
+// [b,d) overlaps some portion of the iterator bounds, so the iterator must
+// return it. If the iterator is subsequently Nexted, Next can tell that the
+// iterator is exhausted without advancing the underlying iterator because the
+// current span's end bound of d is ≥ the upper bound of c. In this case, the
+// bounded iterator returns nil and records i.pos as posAtUpperLimit to remember
+// that the underlying iterator position does not match the current BoundedIter
+// position.
+type boundedIterPos int8
+
+const (
+	posAtLowerLimit boundedIterPos = -1
+	posAtIterSpan   boundedIterPos = 0
+	posAtUpperLimit boundedIterPos = +1
+)
+
+// BoundedIter implements FragmentIterator and enforces bounds.
+//
+// Like the point InternalIterator interface, the bounded iterator's forward
+// positioning routines (SeekGE, First, and Next) only check the upper bound.
+// The reverse positioning routines (SeekLT, Last, and Prev) only check the
+// lower bound. It is up to the caller to ensure that the forward positioning
+// routines respect the lower bound and the reverse positioning routines respect
+// the upper bound (i.e. calling SeekGE instead of First if there is a lower
+// bound, and SeekLT instead of Last if there is an upper bound).
+//
+// When the hasPrefix parameter indicates that the iterator is in prefix
+// iteration mode, BoundedIter elides any spans that do not overlap with the
+// prefix's keyspace. In prefix iteration mode, reverse iteration is disallowed,
+// except for an initial SeekLT with a seek key greater than or equal to the
+// prefix. In prefix iteration mode, the first seek must position the iterator
+// at or immediately before the first fragment covering a key greater than or
+// equal to the prefix.
+type BoundedIter struct {
+	iter      FragmentIterator
+	iterSpan  *Span
+	cmp       base.Compare
+	split     base.Split
+	lower     []byte
+	upper     []byte
+	hasPrefix *bool
+	prefix    *[]byte
+	pos       boundedIterPos
+}
+
+// Init initializes the bounded iterator.
+//
+// In addition to the iterator bounds, Init takes pointers to a boolean
+// indicating whether the iterator is in prefix iteration mode and the prefix
+// key if it is. This is used to exclude spans that are outside the iteration
+// prefix.
+func (i *BoundedIter) Init(
+	cmp base.Compare,
+	split base.Split,
+	iter FragmentIterator,
+	lower, upper []byte,
+	hasPrefix *bool,
+	prefix *[]byte,
+) {
+	*i = BoundedIter{
+		iter:      iter,
+		cmp:       cmp,
+		split:     split,
+		lower:     lower,
+		upper:     upper,
+		hasPrefix: hasPrefix,
+		prefix:    prefix,
+	}
+}
+
+var _ FragmentIterator = (*BoundedIter)(nil)
+
+// Seek calls.
+//
+// Seek calls check iterator bounds in the direction of the seek. Additionally,
+// if the iterator is in prefix iteration mode, seek calls check both start and
+// end bounds against the prefix's bounds. We check both bounds for defense in
+// depth. This optimization has been a source of various bugs due to various
+// other prefix iteration optimizations that can result in seek keys that don't
+// respect the prefix bounds.
+
+// SeekGE implements FragmentIterator.
+func (i *BoundedIter) SeekGE(key []byte) *Span {
+	s := i.iter.SeekGE(key)
+	s = i.checkPrefixSpanStart(s)
+	s = i.checkPrefixSpanEnd(s)
+	return i.checkForwardBound(s)
+}
+
+// SeekLT implements FragmentIterator.
+func (i *BoundedIter) SeekLT(key []byte) *Span {
+	s := i.iter.SeekLT(key)
+	s = i.checkPrefixSpanStart(s)
+	s = i.checkPrefixSpanEnd(s)
+	return i.checkBackwardBound(s)
+}
+
+// First implements FragmentIterator.
+func (i *BoundedIter) First() *Span {
+	s := i.iter.First()
+	s = i.checkPrefixSpanStart(s)
+	return i.checkForwardBound(s)
+}
+
+// Last implements FragmentIterator.
+func (i *BoundedIter) Last() *Span {
+	s := i.iter.Last()
+	s = i.checkPrefixSpanEnd(s)
+	return i.checkBackwardBound(s)
+}
+
+// Next implements FragmentIterator.
+func (i *BoundedIter) Next() *Span {
+	switch i.pos {
+	case posAtLowerLimit:
+		// The BoundedIter had previously returned nil, because it knew from
+		// i.iterSpan's bounds that there was no previous span. To Next, we only
+		// need to return the current iter span and reset i.pos to reflect that
+		// we're no longer positioned at the limit.
+		i.pos = posAtIterSpan
+		return i.iterSpan
+	case posAtIterSpan:
+		// If the span at the underlying iterator position extends to or beyond the
+		// upper bound, we can avoid advancing because the next span is necessarily
+		// out of bounds.
+		if i.iterSpan != nil && i.upper != nil && i.cmp(i.iterSpan.End, i.upper) >= 0 {
+			i.pos = posAtUpperLimit
+			return nil
+		}
+		// Similarly, if the span extends to the next prefix and we're in prefix
+		// iteration mode, we can avoid advancing.
+		if i.iterSpan != nil && *i.hasPrefix {
+			ei := i.split(i.iterSpan.End)
+			if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 {
+				i.pos = posAtUpperLimit
+				return nil
+			}
+		}
+		return i.checkForwardBound(i.checkPrefixSpanStart(i.iter.Next()))
+	case posAtUpperLimit:
+		// Already exhausted.
+		return nil
+	default:
+		panic("unreachable")
+	}
+}
+
+// Prev implements FragmentIterator.
+func (i *BoundedIter) Prev() *Span {
+	switch i.pos {
+	case posAtLowerLimit:
+		// Already exhausted.
+		return nil
+	case posAtIterSpan:
+		// If the span at the underlying iterator position extends to or beyond
+		// the lower bound, we can avoid advancing because the previous span is
+		// necessarily out of bounds.
+		if i.iterSpan != nil && i.lower != nil && i.cmp(i.iterSpan.Start, i.lower) <= 0 {
+			i.pos = posAtLowerLimit
+			return nil
+		}
+		// Similarly, if the span extends to or beyond the current prefix and
+		// we're in prefix iteration mode, we can avoid advancing.
+		if i.iterSpan != nil && *i.hasPrefix {
+			si := i.split(i.iterSpan.Start)
+			if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 {
+				i.pos = posAtLowerLimit
+				return nil
+			}
+		}
+		return i.checkBackwardBound(i.checkPrefixSpanEnd(i.iter.Prev()))
+	case posAtUpperLimit:
+		// The BoundedIter had previously returned nil, because it knew from
+		// i.iterSpan's bounds that there was no next span. To Prev, we only
+		// need to return the current iter span and reset i.pos to reflect that
+		// we're no longer positioned at the limit.
+		i.pos = posAtIterSpan
+		return i.iterSpan
+	default:
+		panic("unreachable")
+	}
+}
+
+// Error implements FragmentIterator.
+func (i *BoundedIter) Error() error {
+	return i.iter.Error()
+}
+
+// Close implements FragmentIterator.
+func (i *BoundedIter) Close() error {
+	return i.iter.Close()
+}
+
+// SetBounds modifies the FragmentIterator's bounds.
+func (i *BoundedIter) SetBounds(lower, upper []byte) {
+	i.lower, i.upper = lower, upper
+}
+
+func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span {
+	// Compare to the prefix's bounds, if in prefix iteration mode.
+	if span != nil && *i.hasPrefix {
+		si := i.split(span.Start)
+		if i.cmp(span.Start[:si], *i.prefix) > 0 {
+			// This span starts at a prefix that sorts after our current prefix.
+			span = nil
+		}
+	}
+	return span
+}
+
+// checkForwardBound enforces the upper bound, returning nil if the provided
+// span is wholly outside the upper bound. It also updates i.pos and i.iterSpan
+// to reflect the new iterator position.
+func (i *BoundedIter) checkForwardBound(span *Span) *Span {
+	// Compare to the upper bound.
+	if span != nil && i.upper != nil && i.cmp(span.Start, i.upper) >= 0 {
+		span = nil
+	}
+	i.iterSpan = span
+	if i.pos != posAtIterSpan {
+		i.pos = posAtIterSpan
+	}
+	return span
+}
+
+func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span {
+	// Compare to the prefix's bounds, if in prefix iteration mode.
+	if span != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
+		// This span ends before the current prefix.
+		span = nil
+	}
+	return span
+}
+
+// checkBackward enforces the lower bound, returning nil if the provided span is
+// wholly outside the lower bound.  It also updates i.pos and i.iterSpan to
+// reflect the new iterator position.
+func (i *BoundedIter) checkBackwardBound(span *Span) *Span {
+	// Compare to the lower bound.
+	if span != nil && i.lower != nil && i.cmp(span.End, i.lower) <= 0 {
+		span = nil
+	}
+	i.iterSpan = span
+	if i.pos != posAtIterSpan {
+		i.pos = posAtIterSpan
+	}
+	return span
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -18,15 +18,33 @@ const bufferReuseMaxCapacity = 10 << 10 // 10 KB
 
 // DefragmentMethod configures the defragmentation performed by the
 // DefragmentingIter.
-type DefragmentMethod func(base.Compare, Span, Span) bool
+type DefragmentMethod interface {
+	// ShouldDefragment takes two abutting spans and returns whether the two
+	// spans should be combined into a single, defragmented Span.
+	ShouldDefragment(equal base.Equal, left, right *Span) bool
+}
+
+// The DefragmentMethodFunc type is an adapter to allow the use of ordinary
+// functions as DefragmentMethods. If f is a function with the appropriate
+// signature, DefragmentMethodFunc(f) is a DefragmentMethod that calls f.
+type DefragmentMethodFunc func(equal base.Equal, left, right *Span) bool
+
+// ShouldDefragment calls f(equal, left, right).
+func (f DefragmentMethodFunc) ShouldDefragment(equal base.Equal, left, right *Span) bool {
+	return f(equal, left, right)
+}
 
 // DefragmentInternal configures a DefragmentingIter to defragment spans
-// only if they have identical keys.
+// only if they have identical keys. It requires spans' keys to be sorted in
+// trailer descending order.
 //
 // This defragmenting method is intended for use in compactions that may see
 // internal range keys fragments that may now be joined, because the state that
 // required their fragmentation has been dropped.
-var DefragmentInternal DefragmentMethod = func(cmp base.Compare, a, b Span) bool {
+var DefragmentInternal DefragmentMethod = DefragmentMethodFunc(func(equal base.Equal, a, b *Span) bool {
+	if a.KeysOrder != ByTrailerDesc || b.KeysOrder != ByTrailerDesc {
+		panic("pebble: span keys unexpectedly not in trailer descending order")
+	}
 	if len(a.Keys) != len(b.Keys) {
 		return false
 	}
@@ -34,7 +52,7 @@ var DefragmentInternal DefragmentMethod = func(cmp base.Compare, a, b Span) bool
 		if a.Keys[i].Trailer != b.Keys[i].Trailer {
 			return false
 		}
-		if cmp(a.Keys[i].Suffix, b.Keys[i].Suffix) != 0 {
+		if !equal(a.Keys[i].Suffix, b.Keys[i].Suffix) {
 			return false
 		}
 		if !bytes.Equal(a.Keys[i].Value, b.Keys[i].Value) {
@@ -42,7 +60,7 @@ var DefragmentInternal DefragmentMethod = func(cmp base.Compare, a, b Span) bool
 		}
 	}
 	return true
-}
+})
 
 // DefragmentReducer merges the current and next Key slices, returning a new Key
 // slice.
@@ -100,9 +118,10 @@ const (
 // defragments in the iteration direction, ensuring it's found a whole
 // defragmented span.
 type DefragmentingIter struct {
-	cmp      base.Compare
+	comparer *base.Comparer
+	equal    base.Equal
 	iter     FragmentIterator
-	iterSpan Span
+	iterSpan *Span
 	iterPos  iterPos
 
 	// curr holds the span at the current iterator position. currBuf is a buffer
@@ -119,10 +138,10 @@ type DefragmentingIter struct {
 	keysBuf []Key
 	keyBuf  []byte
 
-	// equal is a comparison function for two spans. equal is called when two
+	// method is a comparison function for two spans. method is called when two
 	// spans are abutting to determine whether they may be defragmented.
-	// equal does not itself check for adjacency for the two spans.
-	equal DefragmentMethod
+	// method does not itself check for adjacency for the two spans.
+	method DefragmentMethod
 
 	// reduce is the reducer function used to collect Keys across all spans that
 	// constitute a defragmented span.
@@ -135,13 +154,14 @@ var _ FragmentIterator = (*DefragmentingIter)(nil)
 // Init initializes the defragmenting iter using the provided defragment
 // method.
 func (i *DefragmentingIter) Init(
-	cmp base.Compare, iter FragmentIterator, equal DefragmentMethod, reducer DefragmentReducer,
+	comparer *base.Comparer, iter FragmentIterator, equal DefragmentMethod, reducer DefragmentReducer,
 ) {
 	*i = DefragmentingIter{
-		cmp:    cmp,
-		iter:   iter,
-		equal:  equal,
-		reduce: reducer,
+		comparer: comparer,
+		equal:    comparer.Equal,
+		iter:     iter,
+		method:   equal,
+		reduce:   reducer,
 	}
 }
 
@@ -157,16 +177,19 @@ func (i *DefragmentingIter) Close() error {
 
 // SeekGE seeks the iterator to the first span with a start key greater than or
 // equal to key and returns it.
-func (i *DefragmentingIter) SeekGE(key []byte) Span {
+func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 	i.iterSpan = i.iter.SeekGE(key)
-	if !i.iterSpan.Valid() {
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	} else if i.iterSpan.Empty() {
 		i.iterPos = iterPosCurr
 		return i.iterSpan
 	}
 	// Save the current span and peek backwards.
-	i.saveCurrent(i.iterSpan)
+	i.saveCurrent()
 	i.iterSpan = i.iter.Prev()
-	if i.cmp(i.curr.Start, i.iterSpan.End) == 0 && i.checkEqual(i.iterSpan, i.curr) {
+	if i.iterSpan != nil && i.equal(i.curr.Start, i.iterSpan.End) && i.checkEqual(i.iterSpan, &i.curr) {
 		// A continuation. The span we originally landed on and defragmented
 		// backwards has a true Start key < key. To obey the FragmentIterator
 		// contract, we must not return this defragmented span. Defragment
@@ -187,8 +210,15 @@ func (i *DefragmentingIter) SeekGE(key []byte) Span {
 
 // SeekLT seeks the iterator to the last span with a start key less than
 // key and returns it.
-func (i *DefragmentingIter) SeekLT(key []byte) Span {
+func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 	i.iterSpan = i.iter.SeekLT(key)
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	} else if i.iterSpan.Empty() {
+		i.iterPos = iterPosCurr
+		return i.iterSpan
+	}
 	// Defragment forward to find the end of the defragmented span.
 	i.defragmentForward()
 	if i.iterPos == iterPosNext {
@@ -200,19 +230,27 @@ func (i *DefragmentingIter) SeekLT(key []byte) Span {
 }
 
 // First seeks the iterator to the first span and returns it.
-func (i *DefragmentingIter) First() Span {
+func (i *DefragmentingIter) First() *Span {
 	i.iterSpan = i.iter.First()
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	}
 	return i.defragmentForward()
 }
 
 // Last seeks the iterator to the last span and returns it.
-func (i *DefragmentingIter) Last() Span {
+func (i *DefragmentingIter) Last() *Span {
 	i.iterSpan = i.iter.Last()
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	}
 	return i.defragmentBackward()
 }
 
 // Next advances to the next span and returns it.
-func (i *DefragmentingIter) Next() Span {
+func (i *DefragmentingIter) Next() *Span {
 	switch i.iterPos {
 	case iterPosPrev:
 		// Switching directions; The iterator is currently positioned over the
@@ -228,7 +266,7 @@ func (i *DefragmentingIter) Next() Span {
 		// Next once to move onto y, defragment forward to land on the first z
 		// position.
 		i.iterSpan = i.iter.Next()
-		if !i.iterSpan.Valid() {
+		if invariants.Enabled && i.iterSpan == nil {
 			panic("pebble: invariant violation: no next span while switching directions")
 		}
 		// We're now positioned on the first span that was defragmented into the
@@ -236,6 +274,10 @@ func (i *DefragmentingIter) Next() Span {
 		// position's constitutent fragments. In the above example, this would
 		// land on the first 'z'.
 		i.defragmentForward()
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
 
 		// Now that we're positioned over the first of the next set of
 		// fragments, defragment forward.
@@ -243,14 +285,21 @@ func (i *DefragmentingIter) Next() Span {
 	case iterPosCurr:
 		// iterPosCurr is only used when the iter is exhausted or when the iterator
 		// is at an empty span.
-		if invariants.Enabled && i.iterSpan.Valid() && !i.iterSpan.Empty() {
+		if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
 			panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
 		}
 
 		i.iterSpan = i.iter.Next()
+		if i.iterSpan == nil {
+			return nil
+		}
 		return i.defragmentForward()
 	case iterPosNext:
 		// Already at the next span.
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
 		return i.defragmentForward()
 	default:
 		panic("unreachable")
@@ -258,19 +307,26 @@ func (i *DefragmentingIter) Next() Span {
 }
 
 // Prev steps back to the previous span and returns it.
-func (i *DefragmentingIter) Prev() Span {
+func (i *DefragmentingIter) Prev() *Span {
 	switch i.iterPos {
 	case iterPosPrev:
 		// Already at the previous span.
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
 		return i.defragmentBackward()
 	case iterPosCurr:
 		// iterPosCurr is only used when the iter is exhausted or when the iterator
 		// is at an empty span.
-		if invariants.Enabled && i.iterSpan.Valid() && !i.iterSpan.Empty() {
+		if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
 			panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
 		}
 
 		i.iterSpan = i.iter.Prev()
+		if i.iterSpan == nil {
+			return nil
+		}
 		return i.defragmentBackward()
 	case iterPosNext:
 		// Switching directions; The iterator is currently positioned over the
@@ -286,7 +342,7 @@ func (i *DefragmentingIter) Prev() Span {
 		// Prev once to move onto y, defragment backward to land on the last x
 		// position.
 		i.iterSpan = i.iter.Prev()
-		if !i.iterSpan.Valid() {
+		if invariants.Enabled && i.iterSpan == nil {
 			panic("pebble: invariant violation: no previous span while switching directions")
 		}
 		// We're now positioned on the last span that was defragmented into the
@@ -297,6 +353,10 @@ func (i *DefragmentingIter) Prev() Span {
 
 		// Now that we're positioned over the last of the prev set of
 		// fragments, defragment backward.
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
 		return i.defragmentBackward()
 	default:
 		panic("unreachable")
@@ -306,17 +366,14 @@ func (i *DefragmentingIter) Prev() Span {
 // checkEqual checks the two spans for logical equivalence. It uses the passed-in
 // DefragmentMethod and ensures both spans are NOT empty; not defragmenting empty
 // spans is an optimization that lets us load fewer sstable blocks.
-func (i *DefragmentingIter) checkEqual(left, right Span) bool {
-	return i.equal(i.cmp, i.iterSpan, i.curr) && !(left.Empty() && right.Empty())
+func (i *DefragmentingIter) checkEqual(left, right *Span) bool {
+	return (!left.Empty() && !right.Empty()) && i.method.ShouldDefragment(i.equal, i.iterSpan, &i.curr)
 }
 
 // defragmentForward defragments spans in the forward direction, starting from
-// i.iter's current position.
-func (i *DefragmentingIter) defragmentForward() Span {
-	if !i.iterSpan.Valid() {
-		i.iterPos = iterPosCurr
-		return i.iterSpan
-	}
+// i.iter's current position. The span at the current position must be non-nil,
+// but may be Empty().
+func (i *DefragmentingIter) defragmentForward() *Span {
 	if i.iterSpan.Empty() {
 		// An empty span will never be equal to another span; see checkEqual for
 		// why. To avoid loading non-empty range keys further ahead by calling Next,
@@ -324,16 +381,16 @@ func (i *DefragmentingIter) defragmentForward() Span {
 		i.iterPos = iterPosCurr
 		return i.iterSpan
 	}
-	i.saveCurrent(i.iterSpan)
+	i.saveCurrent()
 
 	i.iterPos = iterPosNext
 	i.iterSpan = i.iter.Next()
-	for i.iterSpan.Valid() {
-		if i.cmp(i.curr.End, i.iterSpan.Start) != 0 {
+	for i.iterSpan != nil {
+		if !i.equal(i.curr.End, i.iterSpan.Start) {
 			// Not a continuation.
 			break
 		}
-		if !i.checkEqual(i.iterSpan, i.curr) {
+		if !i.checkEqual(i.iterSpan, &i.curr) {
 			// Not a continuation.
 			break
 		}
@@ -343,16 +400,13 @@ func (i *DefragmentingIter) defragmentForward() Span {
 		i.iterSpan = i.iter.Next()
 	}
 	i.curr.Keys = i.keysBuf
-	return i.curr
+	return &i.curr
 }
 
 // defragmentBackward defragments spans in the backward direction, starting from
-// i.iter's current position.
-func (i *DefragmentingIter) defragmentBackward() Span {
-	if !i.iterSpan.Valid() {
-		i.iterPos = iterPosCurr
-		return i.iterSpan
-	}
+// i.iter's current position. The span at the current position must be non-nil,
+// but may be Empty().
+func (i *DefragmentingIter) defragmentBackward() *Span {
 	if i.iterSpan.Empty() {
 		// An empty span will never be equal to another span; see checkEqual for
 		// why. To avoid loading non-empty range keys further ahead by calling Next,
@@ -360,16 +414,16 @@ func (i *DefragmentingIter) defragmentBackward() Span {
 		i.iterPos = iterPosCurr
 		return i.iterSpan
 	}
-	i.saveCurrent(i.iterSpan)
+	i.saveCurrent()
 
 	i.iterPos = iterPosPrev
 	i.iterSpan = i.iter.Prev()
-	for i.iterSpan.Valid() {
-		if i.cmp(i.curr.Start, i.iterSpan.End) != 0 {
+	for i.iterSpan != nil {
+		if !i.equal(i.curr.Start, i.iterSpan.End) {
 			// Not a continuation.
 			break
 		}
-		if !i.checkEqual(i.iterSpan, i.curr) {
+		if !i.checkEqual(i.iterSpan, &i.curr) {
 			// Not a continuation.
 			break
 		}
@@ -379,10 +433,10 @@ func (i *DefragmentingIter) defragmentBackward() Span {
 		i.iterSpan = i.iter.Prev()
 	}
 	i.curr.Keys = i.keysBuf
-	return i.curr
+	return &i.curr
 }
 
-func (i *DefragmentingIter) saveCurrent(span Span) {
+func (i *DefragmentingIter) saveCurrent() {
 	i.currBuf = i.currBuf[:0]
 	i.keysBuf = i.keysBuf[:0]
 	i.keyBuf = i.keyBuf[:0]
@@ -392,15 +446,19 @@ func (i *DefragmentingIter) saveCurrent(span Span) {
 	if cap(i.keyBuf) > bufferReuseMaxCapacity {
 		i.keyBuf = nil
 	}
-	i.curr = Span{
-		Start: i.saveBytes(span.Start),
-		End:   i.saveBytes(span.End),
+	if i.iterSpan == nil {
+		return
 	}
-	for j := range span.Keys {
+	i.curr = Span{
+		Start:     i.saveBytes(i.iterSpan.Start),
+		End:       i.saveBytes(i.iterSpan.End),
+		KeysOrder: i.iterSpan.KeysOrder,
+	}
+	for j := range i.iterSpan.Keys {
 		i.keysBuf = append(i.keysBuf, Key{
-			Trailer: span.Keys[j].Trailer,
-			Suffix:  i.saveBytes(span.Keys[j].Suffix),
-			Value:   i.saveBytes(span.Keys[j].Value),
+			Trailer: i.iterSpan.Keys[j].Trailer,
+			Suffix:  i.saveBytes(i.iterSpan.Keys[j].Suffix),
+			Value:   i.saveBytes(i.iterSpan.Keys[j].Value),
 		})
 	}
 	i.curr.Keys = i.keysBuf

vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go

@@ -10,7 +10,7 @@ package keyspan
 // the output Span, for example, to elice certain keys, or update the Span's
 // bounds if so desired. The output Span's Keys slice may be reused to reduce
 // allocations.
-type FilterFunc func(in Span, out *Span) (keep bool)
+type FilterFunc func(in *Span, out *Span) (keep bool)
 
 // filteringIter is a FragmentIterator that uses a FilterFunc to select which
 // Spans from the input iterator are returned in the output.
@@ -37,32 +37,32 @@ func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
 }
 
 // SeekGE implements FragmentIterator.
-func (i *filteringIter) SeekGE(key []byte) Span {
+func (i *filteringIter) SeekGE(key []byte) *Span {
 	return i.filter(i.iter.SeekGE(key), +1)
 }
 
 // SeekLT implements FragmentIterator.
-func (i *filteringIter) SeekLT(key []byte) Span {
+func (i *filteringIter) SeekLT(key []byte) *Span {
 	return i.filter(i.iter.SeekLT(key), -1)
 }
 
 // First implements FragmentIterator.
-func (i *filteringIter) First() Span {
+func (i *filteringIter) First() *Span {
 	return i.filter(i.iter.First(), +1)
 }
 
 // Last implements FragmentIterator.
-func (i *filteringIter) Last() Span {
+func (i *filteringIter) Last() *Span {
 	return i.filter(i.iter.Last(), -1)
 }
 
 // Next implements FragmentIterator.
-func (i *filteringIter) Next() Span {
+func (i *filteringIter) Next() *Span {
 	return i.filter(i.iter.Next(), +1)
 }
 
 // Prev implements FragmentIterator.
-func (i *filteringIter) Prev() Span {
+func (i *filteringIter) Prev() *Span {
 	return i.filter(i.iter.Prev(), -1)
 }
 
@@ -80,18 +80,18 @@ func (i *filteringIter) Close() error {
 // given Span. If the current Span is to be skipped, the iterator continues
 // iterating in the given direction until it lands on a Span that should be
 // returned, or the iterator becomes invalid.
-func (i *filteringIter) filter(span Span, dir int8) Span {
+func (i *filteringIter) filter(span *Span, dir int8) *Span {
 	if i.filterFn == nil {
 		return span
 	}
-	for i.Error() == nil && span.Valid() {
+	for i.Error() == nil && span != nil {
 		if keep := i.filterFn(span, &i.span); keep {
-			return i.span
+			return &i.span
 		}
 		if dir == +1 {
-			span = i.Next()
+			span = i.iter.Next()
 		} else {
-			span = i.Prev()
+			span = i.iter.Prev()
 		}
 	}
 	return span

vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go

@@ -183,9 +183,13 @@ func (f *Fragmenter) checkInvariants(buf []Span) {
 // deletion and range key blocks, so these keys are stable. Because of this key
 // stability, typically callers only need to perform a shallow clone of the Span
 // before Add-ing it to the fragmenter.
+//
+// Add requires the provided span's keys are sorted in Trailer descending order.
 func (f *Fragmenter) Add(s Span) {
 	if f.finished {
 		panic("pebble: span fragmenter already finished")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span keys unexpectedly not in trailer descending order")
 	}
 	if f.flushedKey != nil {
 		switch c := f.Cmp(s.Start, f.flushedKey); {
@@ -248,7 +252,7 @@ func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
 			// NB: A range deletion tombstone does not delete a point operation
 			// at the same sequence number, and broadly a span is not considered
 			// to cover a point operation at the same sequence number.
-			if s.Visible(snapshot).Covers(seqNum) {
+			if s.CoversAt(snapshot, seqNum) {
 				return true
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/keyspan/get.go

@@ -11,7 +11,7 @@ import "github.com/cockroachdb/pebble/internal/base"
 // parameter controls the visibility of spans (only spans older than the
 // snapshot sequence number are visible). The iterator must contain
 // fragmented spans: no span may overlap another.
-func Get(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) Span {
+func Get(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
 	// NB: We use SeekLT in order to land on the proper span for a search
 	// key that resides in the middle of a span. Consider the scenario:
 	//
@@ -25,15 +25,15 @@ func Get(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) S
 	// happens for the search key `e`. In that case SeekLT will land us
 	// on the span [a,e) and we'll have to move forward.
 	iterSpan := iter.SeekLT(key)
-	if !iterSpan.Valid() {
+	if iterSpan == nil {
 		iterSpan = iter.Next()
-		if !iterSpan.Valid() {
+		if iterSpan == nil {
 			// The iterator is empty.
-			return Span{}
+			return nil
 		}
 		if cmp(key, iterSpan.Start) < 0 {
 			// The search key lies before the first span.
-			return Span{}
+			return nil
 		}
 	}
 
@@ -43,11 +43,11 @@ func Get(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) S
 		// once to potentially land on a key with a start key exactly equal to
 		// key. (See the comment at the beginning of this function.)
 		iterSpan = iter.Next()
-		if !iterSpan.Valid() || cmp(key, iterSpan.Start) < 0 {
+		if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
 			// We've run out of spans or we've moved on to a span which
 			// starts after our search key.
-			return Span{}
+			return nil
 		}
 	}
-	return iterSpan.Visible(snapshot)
+	return iterSpan
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go

@@ -16,7 +16,7 @@ import "github.com/cockroachdb/pebble/internal/base"
 // keyspan.FragmentIterator with point keys.
 type InternalIteratorShim struct {
 	miter   MergingIter
-	span    Span
+	span    *Span
 	iterKey base.InternalKey
 }
 
@@ -31,62 +31,64 @@ func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator)
 
 // Span returns the span containing the full set of keys over the key span at
 // the current iterator position.
-func (i *InternalIteratorShim) Span() Span {
+func (i *InternalIteratorShim) Span() *Span {
 	return i.span
 }
 
 // SeekGE implements (base.InternalIterator).SeekGE.
 func (i *InternalIteratorShim) SeekGE(
-	key []byte, trySeekUsingNext bool,
-) (*base.InternalKey, []byte) {
+	key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
 	panic("unimplemented")
 }
 
 // SeekPrefixGE implements (base.InternalIterator).SeekPrefixGE.
 func (i *InternalIteratorShim) SeekPrefixGE(
-	prefix, key []byte, trySeekUsingNext bool,
-) (*base.InternalKey, []byte) {
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
 	panic("unimplemented")
 }
 
 // SeekLT implements (base.InternalIterator).SeekLT.
-func (i *InternalIteratorShim) SeekLT(key []byte) (*base.InternalKey, []byte) {
+func (i *InternalIteratorShim) SeekLT(
+	key []byte, flags base.SeekLTFlags,
+) (*base.InternalKey, base.LazyValue) {
 	panic("unimplemented")
 }
 
 // First implements (base.InternalIterator).First.
-func (i *InternalIteratorShim) First() (*base.InternalKey, []byte) {
+func (i *InternalIteratorShim) First() (*base.InternalKey, base.LazyValue) {
 	i.span = i.miter.First()
-	for i.span.Valid() && i.span.Empty() {
+	for i.span != nil && i.span.Empty() {
 		i.span = i.miter.Next()
 	}
-	if !i.span.Valid() {
-		return nil, nil
+	if i.span == nil {
+		return nil, base.LazyValue{}
 	}
 	i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
-	return &i.iterKey, i.span.End
+	return &i.iterKey, base.MakeInPlaceValue(i.span.End)
 }
 
 // Last implements (base.InternalIterator).Last.
-func (i *InternalIteratorShim) Last() (*base.InternalKey, []byte) {
+func (i *InternalIteratorShim) Last() (*base.InternalKey, base.LazyValue) {
 	panic("unimplemented")
 }
 
 // Next implements (base.InternalIterator).Next.
-func (i *InternalIteratorShim) Next() (*base.InternalKey, []byte) {
+func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) {
 	i.span = i.miter.Next()
-	for i.span.Valid() && i.span.Empty() {
+	for i.span != nil && i.span.Empty() {
 		i.span = i.miter.Next()
 	}
-	if !i.span.Valid() {
-		return nil, nil
+	if i.span == nil {
+		return nil, base.LazyValue{}
 	}
 	i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
-	return &i.iterKey, i.span.End
+	return &i.iterKey, base.MakeInPlaceValue(i.span.End)
 }
 
 // Prev implements (base.InternalIterator).Prev.
-func (i *InternalIteratorShim) Prev() (*base.InternalKey, []byte) {
+func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) {
 	panic("unimplemented")
 }
 

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -20,17 +20,17 @@ import (
 type FragmentIterator interface {
 	// SeekGE moves the iterator to the first span whose start key is greater
 	// than or equal to the given key.
-	SeekGE(key []byte) Span
+	SeekGE(key []byte) *Span
 
 	// SeekLT moves the iterator to the last span whose start key is less than
 	// the given key.
-	SeekLT(key []byte) Span
+	SeekLT(key []byte) *Span
 
 	// First moves the iterator to the first span.
-	First() Span
+	First() *Span
 
 	// Last moves the iterator to the last span.
-	Last() Span
+	Last() *Span
 
 	// Next moves the iterator to the next span.
 	//
@@ -38,7 +38,7 @@ type FragmentIterator interface {
 	// key/value pair due to either a prior call to SeekLT or Prev which
 	// returned an invalid span. It is not allowed to call Next when the
 	// previous call to SeekGE, SeekPrefixGE or Next returned an invalid span.
-	Next() Span
+	Next() *Span
 
 	// Prev moves the iterator to the previous span.
 	//
@@ -46,7 +46,7 @@ type FragmentIterator interface {
 	// key/value pair due to either a prior call to SeekGE or Next which
 	// returned an invalid span. It is not allowed to call Prev when the
 	// previous call to SeekLT or Prev returned an invalid span.
-	Prev() Span
+	Prev() *Span
 
 	// Error returns any accumulated error.
 	Error() error
@@ -81,7 +81,19 @@ var _ FragmentIterator = (*Iter)(nil)
 
 // NewIter returns a new iterator over a set of fragmented spans.
 func NewIter(cmp base.Compare, spans []Span) *Iter {
-	return &Iter{
+	i := &Iter{}
+	i.Init(cmp, spans)
+	return i
+}
+
+// Count returns the number of spans contained by Iter.
+func (i *Iter) Count() int {
+	return len(i.spans)
+}
+
+// Init initializes an Iter with the provided spans.
+func (i *Iter) Init(cmp base.Compare, spans []Span) {
+	*i = Iter{
 		cmp:   cmp,
 		spans: spans,
 		index: -1,
@@ -89,7 +101,7 @@ func NewIter(cmp base.Compare, spans []Span) *Iter {
 }
 
 // SeekGE implements FragmentIterator.SeekGE.
-func (i *Iter) SeekGE(key []byte) Span {
+func (i *Iter) SeekGE(key []byte) *Span {
 	// NB: manually inlined sort.Search is ~5% faster.
 	//
 	// Define f(-1) == false and f(n) == true.
@@ -108,13 +120,13 @@ func (i *Iter) SeekGE(key []byte) Span {
 	// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
 	// true => answer is i.index.
 	if i.index >= len(i.spans) {
-		return Span{}
+		return nil
 	}
-	return i.spans[i.index]
+	return &i.spans[i.index]
 }
 
 // SeekLT implements FragmentIterator.SeekLT.
-func (i *Iter) SeekLT(key []byte) Span {
+func (i *Iter) SeekLT(key []byte) *Span {
 	// NB: manually inlined sort.Search is ~5% faster.
 	//
 	// Define f(-1) == false and f(n) == true.
@@ -137,51 +149,51 @@ func (i *Iter) SeekLT(key []byte) Span {
 	// the largest whose key is < the key sought.
 	i.index--
 	if i.index < 0 {
-		return Span{}
+		return nil
 	}
-	return i.spans[i.index]
+	return &i.spans[i.index]
 }
 
 // First implements FragmentIterator.First.
-func (i *Iter) First() Span {
+func (i *Iter) First() *Span {
 	if len(i.spans) == 0 {
-		return Span{}
+		return nil
 	}
 	i.index = 0
-	return i.spans[i.index]
+	return &i.spans[i.index]
 }
 
 // Last implements FragmentIterator.Last.
-func (i *Iter) Last() Span {
+func (i *Iter) Last() *Span {
 	if len(i.spans) == 0 {
-		return Span{}
+		return nil
 	}
 	i.index = len(i.spans) - 1
-	return i.spans[i.index]
+	return &i.spans[i.index]
 }
 
 // Next implements FragmentIterator.Next.
-func (i *Iter) Next() Span {
+func (i *Iter) Next() *Span {
 	if i.index >= len(i.spans) {
-		return Span{}
+		return nil
 	}
 	i.index++
 	if i.index >= len(i.spans) {
-		return Span{}
+		return nil
 	}
-	return i.spans[i.index]
+	return &i.spans[i.index]
 }
 
 // Prev implements FragmentIterator.Prev.
-func (i *Iter) Prev() Span {
+func (i *Iter) Prev() *Span {
 	if i.index < 0 {
-		return Span{}
+		return nil
 	}
 	i.index--
 	if i.index < 0 {
-		return Span{}
+		return nil
 	}
-	return i.spans[i.index]
+	return &i.spans[i.index]
 }
 
 // Error implements FragmentIterator.Error.

vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go

@@ -40,8 +40,8 @@ type LevelIter struct {
 	// key spaces. This is an optimization to avoid unnecessarily loading files
 	// in cases where range keys are sparse and rare. dir is set by every
 	// positioning operation, straddleDir is set to dir whenever a straddling
-	// Span is synthesized, and straddle is Valid() whenever the last positioning
-	// operation returned a synthesized straddle span.
+	// Span is synthesized and the last positioning operation returned a
+	// synthesized straddle span.
 	//
 	// Note that when a straddle span is initialized, iterFile is modified to
 	// point to the next file in the straddleDir direction. A change of direction
@@ -53,7 +53,7 @@ type LevelIter struct {
 	// The iter for the current file. It is nil under any of the following conditions:
 	// - files.Current() == nil
 	// - err != nil
-	// - straddle.Valid(), in which case iterFile is not nil and points to the
+	// - straddleDir != 0, in which case iterFile is not nil and points to the
 	//   next file (in the straddleDir direction).
 	// - some other constraint, like the bounds in opts, caused the file at index to not
 	//   be relevant to the iteration.
@@ -178,7 +178,7 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur
 		return noFileLoaded
 	}
 	if indicator != fileAlreadyLoaded {
-		l.iter, l.err = l.newIter(l.files.Current(), &l.tableOpts)
+		l.iter, l.err = l.newIter(file, &l.tableOpts)
 		indicator = newFileLoaded
 	}
 	if l.err != nil {
@@ -188,123 +188,149 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur
 }
 
 // SeekGE implements keyspan.FragmentIterator.
-func (l *LevelIter) SeekGE(key []byte) Span {
+func (l *LevelIter) SeekGE(key []byte) *Span {
 	l.dir = +1
+	l.straddle = Span{}
+	l.straddleDir = 0
 	l.err = nil // clear cached iteration error
 
 	f := l.findFileGE(key)
 	if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
-		// Return a straddling key instead of loading the file.
-		l.iterFile = f
-		l.iter = nil
-		l.straddleDir = +1
-		// The synthetic span that we are creating starts at the seeked key. This
-		// is an optimization as it prevents us from loading the adjacent file's
-		// bounds, at the expense of this iterator appearing "inconsistent" to its
-		// callers i.e.:
-		//
-		// SeekGE(bb) -> {bb-c, empty}
-		// Next()     -> {c-d, RANGEKEYSET}
-		// Prev()     -> {a-c, empty}
-		//
-		// Seeing as the inconsistency will only be around empty spans, which are
-		// expected to be elided by one of the higher-level iterators (either
-		// top-level Iterator or the defragmenting iter), the entire iterator should
-		// still appear consistent to the user.
-		l.straddle = Span{
-			Start: key,
-			End:   f.SmallestRangeKey.UserKey,
-			Keys:  nil,
+		prevFile := l.files.Prev()
+		if prevFile != nil {
+			// We could unconditionally return an empty span between the seek key and
+			// f.SmallestRangeKey, however if this span is to the left of all range
+			// keys on this level, it could lead to inconsistent behaviour in relative
+			// positioning operations. Consider this example, with a b-c range key:
+			//
+			// SeekGE(a) -> a-b:{}
+			// Next() -> b-c{(#5,RANGEKEYSET,@4,foo)}
+			// Prev() -> nil
+			//
+			// Iterators higher up in the iterator stack rely on this sort of relative
+			// positioning consistency.
+			//
+			// TODO(bilal): Investigate ways to be able to return straddle spans in
+			// cases similar to the above, while still retaining correctness.
+			l.files.Next()
+			// Return a straddling key instead of loading the file.
+			l.iterFile = f
+			if err := l.Close(); err != nil {
+				return nil
+			}
+			l.straddleDir = +1
+			l.straddle = Span{
+				Start: prevFile.LargestRangeKey.UserKey,
+				End:   f.SmallestRangeKey.UserKey,
+				Keys:  nil,
+			}
+			return &l.straddle
 		}
-		return l.straddle
 	}
 	loadFileIndicator := l.loadFile(f, +1)
 	if loadFileIndicator == noFileLoaded {
-		return Span{}
+		return nil
 	}
-	if span := l.iter.SeekGE(key); span.Valid() {
+	if span := l.iter.SeekGE(key); span != nil {
 		return span
 	}
 	return l.skipEmptyFileForward()
 }
 
 // SeekLT implements keyspan.FragmentIterator.
-func (l *LevelIter) SeekLT(key []byte) Span {
+func (l *LevelIter) SeekLT(key []byte) *Span {
 	l.dir = -1
+	l.straddle = Span{}
+	l.straddleDir = 0
 	l.err = nil // clear cached iteration error
 
 	f := l.findFileLT(key)
 	if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
-		// Return a straddling key instead of loading the file.
-		l.iterFile = f
-		l.iter = nil
-		l.straddleDir = -1
-		// The synthetic span that we are creating ends at the seeked key. This
-		// is an optimization as it prevents us from loading the adjacent file's
-		// bounds, at the expense of this iterator appearing "inconsistent" to its
-		// callers i.e.:
-		//
-		// SeekLT(dd) -> {d-dd, empty}
-		// Prev()     -> {c-d, RANGEKEYSET}
-		// Next()     -> {d-e, empty}
-		//
-		// Seeing as the inconsistency will only be around empty spans, which are
-		// expected to be elided by one of the higher-level iterators (either
-		// top-level Iterator or the defragmenting iter), the entire iterator should
-		// still appear consistent to the user.
-		l.straddle = Span{
-			Start: f.LargestRangeKey.UserKey,
-			End:   key,
-			Keys:  nil,
+		nextFile := l.files.Next()
+		if nextFile != nil {
+			// We could unconditionally return an empty span between f.LargestRangeKey
+			// and the seek key, however if this span is to the right of all range keys
+			// on this level, it could lead to inconsistent behaviour in relative
+			// positioning operations. Consider this example, with a b-c range key:
+			//
+			// SeekLT(d) -> c-d:{}
+			// Prev() -> b-c{(#5,RANGEKEYSET,@4,foo)}
+			// Next() -> nil
+			//
+			// Iterators higher up in the iterator stack rely on this sort of relative
+			// positioning consistency.
+			//
+			// TODO(bilal): Investigate ways to be able to return straddle spans in
+			// cases similar to the above, while still retaining correctness.
+			l.files.Prev()
+			// Return a straddling key instead of loading the file.
+			l.iterFile = f
+			if err := l.Close(); err != nil {
+				return nil
+			}
+			l.straddleDir = -1
+			l.straddle = Span{
+				Start: f.LargestRangeKey.UserKey,
+				End:   nextFile.SmallestRangeKey.UserKey,
+				Keys:  nil,
+			}
+			return &l.straddle
 		}
-		return l.straddle
 	}
 	if l.loadFile(l.findFileLT(key), -1) == noFileLoaded {
-		return Span{}
+		return nil
 	}
-	if span := l.iter.SeekLT(key); span.Valid() {
+	if span := l.iter.SeekLT(key); span != nil {
 		return span
 	}
 	return l.skipEmptyFileBackward()
 }
 
 // First implements keyspan.FragmentIterator.
-func (l *LevelIter) First() Span {
+func (l *LevelIter) First() *Span {
 	l.dir = +1
+	l.straddle = Span{}
+	l.straddleDir = 0
 	l.err = nil // clear cached iteration error
 
 	if l.loadFile(l.files.First(), +1) == noFileLoaded {
-		return Span{}
+		return nil
 	}
-	if span := l.iter.First(); span.Valid() {
+	if span := l.iter.First(); span != nil {
 		return span
 	}
 	return l.skipEmptyFileForward()
 }
 
 // Last implements keyspan.FragmentIterator.
-func (l *LevelIter) Last() Span {
+func (l *LevelIter) Last() *Span {
 	l.dir = -1
+	l.straddle = Span{}
+	l.straddleDir = 0
 	l.err = nil // clear cached iteration error
 
 	if l.loadFile(l.files.Last(), -1) == noFileLoaded {
-		return Span{}
+		return nil
 	}
-	if span := l.iter.Last(); span.Valid() {
+	if span := l.iter.Last(); span != nil {
 		return span
 	}
 	return l.skipEmptyFileBackward()
 }
 
 // Next implements keyspan.FragmentIterator.
-func (l *LevelIter) Next() Span {
-	l.dir = +1
-	if l.err != nil || (l.iter == nil && l.iterFile == nil) {
-		return Span{}
+func (l *LevelIter) Next() *Span {
+	if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) {
+		return nil
 	}
+	if l.iter == nil && l.iterFile == nil {
+		// l.dir <= 0
+		return l.First()
+	}
+	l.dir = +1
 
 	if l.iter != nil {
-		if span := l.iter.Next(); span.Valid() {
+		if span := l.iter.Next(); span != nil {
 			return span
 		}
 	}
@@ -312,22 +338,26 @@ func (l *LevelIter) Next() Span {
 }
 
 // Prev implements keyspan.FragmentIterator.
-func (l *LevelIter) Prev() Span {
-	l.dir = -1
-	if l.err != nil || (l.iter == nil && l.iterFile == nil) {
-		return Span{}
+func (l *LevelIter) Prev() *Span {
+	if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) {
+		return nil
 	}
+	if l.iter == nil && l.iterFile == nil {
+		// l.dir >= 0
+		return l.Last()
+	}
+	l.dir = -1
 
 	if l.iter != nil {
-		if span := l.iter.Prev(); span.Valid() {
+		if span := l.iter.Prev(); span != nil {
 			return span
 		}
 	}
 	return l.skipEmptyFileBackward()
 }
 
-func (l *LevelIter) skipEmptyFileForward() Span {
-	if !l.straddle.Valid() && l.keyType == manifest.KeyTypeRange &&
+func (l *LevelIter) skipEmptyFileForward() *Span {
+	if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
 		l.iterFile != nil && l.iter != nil {
 		// We were at a file that had spans. Check if the next file that has
 		// spans is not directly adjacent to the current file i.e. there is a
@@ -337,7 +367,7 @@ func (l *LevelIter) skipEmptyFileForward() Span {
 		// Straddle spans are not created in rangedel mode.
 		if err := l.Close(); err != nil {
 			l.err = err
-			return Span{}
+			return nil
 		}
 		startKey := l.iterFile.LargestRangeKey.UserKey
 		// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -345,7 +375,7 @@ func (l *LevelIter) skipEmptyFileForward() Span {
 		// which it should be due to the Close() call above.
 		l.iterFile = l.files.Next()
 		if l.iterFile == nil {
-			return Span{}
+			return nil
 		}
 		endKey := l.iterFile.SmallestRangeKey.UserKey
 		if l.cmp(startKey, endKey) < 0 {
@@ -355,17 +385,18 @@ func (l *LevelIter) skipEmptyFileForward() Span {
 				Start: startKey,
 				End:   endKey,
 			}
-			l.straddleDir = l.dir
-			return l.straddle
+			l.straddleDir = +1
+			return &l.straddle
 		}
-	} else if l.straddle.Valid() && l.straddleDir < 0 {
+	} else if l.straddleDir < 0 {
 		// We were at a straddle key, but are now changing directions. l.iterFile
 		// was already moved backward by skipEmptyFileBackward, so advance it
 		// forward.
 		l.iterFile = l.files.Next()
 	}
 	l.straddle = Span{}
-	var span Span
+	l.straddleDir = 0
+	var span *Span
 	for span.Empty() {
 		fileToLoad := l.iterFile
 		if l.keyType == manifest.KeyTypePoint {
@@ -374,7 +405,7 @@ func (l *LevelIter) skipEmptyFileForward() Span {
 			fileToLoad = l.files.Next()
 		}
 		if l.loadFile(fileToLoad, +1) == noFileLoaded {
-			return Span{}
+			return nil
 		}
 		span = l.iter.First()
 		// In rangedel mode, we can expect to get empty files that we'd need to
@@ -386,18 +417,18 @@ func (l *LevelIter) skipEmptyFileForward() Span {
 	return span
 }
 
-func (l *LevelIter) skipEmptyFileBackward() Span {
+func (l *LevelIter) skipEmptyFileBackward() *Span {
 	// We were at a file that had spans. Check if the previous file that has
 	// spans is not directly adjacent to the current file i.e. there is a
 	// gap in the span keyspace between the two files. In that case, synthesize
 	// a "straddle span" in l.straddle and return that.
 	//
 	// Straddle spans are not created in rangedel mode.
-	if !l.straddle.Valid() && l.keyType == manifest.KeyTypeRange &&
+	if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
 		l.iterFile != nil && l.iter != nil {
 		if err := l.Close(); err != nil {
 			l.err = err
-			return Span{}
+			return nil
 		}
 		endKey := l.iterFile.SmallestRangeKey.UserKey
 		// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -405,7 +436,7 @@ func (l *LevelIter) skipEmptyFileBackward() Span {
 		// which it should be due to the Close() call above.
 		l.iterFile = l.files.Prev()
 		if l.iterFile == nil {
-			return Span{}
+			return nil
 		}
 		startKey := l.iterFile.LargestRangeKey.UserKey
 		if l.cmp(startKey, endKey) < 0 {
@@ -415,24 +446,25 @@ func (l *LevelIter) skipEmptyFileBackward() Span {
 				Start: startKey,
 				End:   endKey,
 			}
-			l.straddleDir = l.dir
-			return l.straddle
+			l.straddleDir = -1
+			return &l.straddle
 		}
-	} else if l.straddle.Valid() && l.straddleDir > 0 {
+	} else if l.straddleDir > 0 {
 		// We were at a straddle key, but are now changing directions. l.iterFile
 		// was already advanced forward by skipEmptyFileForward, so move it
 		// backward.
 		l.iterFile = l.files.Prev()
 	}
 	l.straddle = Span{}
-	var span Span
+	l.straddleDir = 0
+	var span *Span
 	for span.Empty() {
 		fileToLoad := l.iterFile
 		if l.keyType == manifest.KeyTypePoint {
 			fileToLoad = l.files.Prev()
 		}
 		if l.loadFile(fileToLoad, -1) == noFileLoaded {
-			return Span{}
+			return nil
 		}
 		span = l.iter.Last()
 		// In rangedel mode, we can expect to get empty files that we'd need to

vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/manifest"
 )
 
 // TODO(jackson): Consider implementing an optimization to seek lower levels
@@ -20,27 +21,47 @@ import (
 // seeks would require introducing key comparisons to switchTo{Min,Max}Heap
 // where there currently are none.
 
-// Transform defines a transform function to be applied to a Span. A Transform
-// takes a Span as input and writes the transformed Span to the provided output
-// *Span pointer. The output Span's Keys slice may be reused by Transform to
-// reduce allocations.
-type Transform func(cmp base.Compare, in Span, out *Span) error
+// Transformer defines a transformation to be applied to a Span.
+type Transformer interface {
+	// Transform takes a Span as input and writes the transformed Span to the
+	// provided output *Span pointer. The output Span's Keys slice may be reused
+	// by Transform to reduce allocations.
+	Transform(cmp base.Compare, in Span, out *Span) error
+}
 
-func noopTransform(_ base.Compare, s Span, dst *Span) error {
+// The TransformerFunc type is an adapter to allow the use of ordinary functions
+// as Transformers. If f is a function with the appropriate signature,
+// TransformerFunc(f) is a Transformer that calls f.
+type TransformerFunc func(base.Compare, Span, *Span) error
+
+// Transform calls f(cmp, in, out).
+func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
+	return tf(cmp, in, out)
+}
+
+var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
 	dst.Start, dst.End = s.Start, s.End
 	dst.Keys = append(dst.Keys[:0], s.Keys...)
 	return nil
-}
+})
 
 // visibleTransform filters keys that are invisible at the provided snapshot
 // sequence number.
-func visibleTransform(snapshot uint64) Transform {
-	return func(_ base.Compare, s Span, dst *Span) error {
-		s = s.Visible(snapshot)
+func visibleTransform(snapshot uint64) Transformer {
+	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
 		dst.Start, dst.End = s.Start, s.End
-		dst.Keys = append(dst.Keys[:0], s.Keys...)
+		dst.Keys = dst.Keys[:0]
+		for _, k := range s.Keys {
+			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
+			// because a batch's visible span keys are filtered when they're
+			// fragmented. There's no requirement to enforce visibility at
+			// iteration time.
+			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
+				dst.Keys = append(dst.Keys, k)
+			}
+		}
 		return nil
-	}
+	})
 }
 
 // MergingIter merges spans across levels of the LSM, exposing an iterator over
@@ -51,6 +72,10 @@ func visibleTransform(snapshot uint64) Transform {
 // overlapping keys are surfaced in a single Span. Key spans from one child
 // iterator may overlap key spans from another child iterator arbitrarily.
 //
+// The spans combined by MergingIter will return spans with keys sorted by
+// trailer descending. If the MergingIter is configured with a Transformer, it's
+// permitted to modify the ordering of the spans' keys returned by MergingIter.
+//
 // Algorithm
 //
 // The merging iterator wraps child iterators, merging and fragmenting spans
@@ -193,17 +218,17 @@ type MergingIter struct {
 	// buf is a buffer used to save [start, end) boundary keys.
 	buf []byte
 	// keys holds all of the keys across all levels that overlap the key span
-	// [start, end), sorted by sequence number and kind descending. This slice
-	// is reconstituted in synthesizeKeys from each mergingIterLevel's keys
-	// every time the [start, end) bounds change.
+	// [start, end), sorted by Trailer descending. This slice is reconstituted
+	// in synthesizeKeys from each mergingIterLevel's keys every time the
+	// [start, end) bounds change.
 	//
 	// Each element points into a child iterator's memory, so the keys may not
 	// be directly modified.
 	keys keysBySeqNumKind
-	// transform defines a function to be applied to a span before it's yielded
-	// to the user. A transform may filter individual keys contained within the
-	// span.
-	transform Transform
+	// transformer defines a transformation to be applied to a span before it's
+	// yielded to the user. Transforming may filter individual keys contained
+	// within the span.
+	transformer Transformer
 	// span holds the iterator's current span. This span is used as the
 	// destination for transforms. Every tranformed span overwrites the
 	// previous.
@@ -211,6 +236,17 @@ type MergingIter struct {
 
 	err error
 	dir int8
+
+	// alloc preallocates mergingIterLevel and mergingIterItems for use by the
+	// merging iterator. As long as the merging iterator is used with
+	// manifest.NumLevels+3 and fewer fragment iterators, the merging iterator
+	// will not need to allocate upon initialization. The value NumLevels+3
+	// mirrors the preallocated levels in iterAlloc used for point iterators.
+	// Invariant: cap(levels) == cap(items)
+	alloc struct {
+		levels [manifest.NumLevels + 3]mergingIterLevel
+		items  [manifest.NumLevels + 3]mergingIterItem
+	}
 }
 
 // MergingIter implements the FragmentIterator interface.
@@ -225,45 +261,82 @@ type mergingIterLevel struct {
 
 func (l *mergingIterLevel) next() {
 	if l.heapKey.kind == boundKindFragmentStart {
-		l.heapKey = endBound(l.heapKey.span)
+		l.heapKey = boundKey{
+			kind: boundKindFragmentEnd,
+			key:  l.heapKey.span.End,
+			span: l.heapKey.span,
+		}
 		return
 	}
-	l.heapKey = startBound(l.iter.Next())
+	if s := l.iter.Next(); s == nil {
+		l.heapKey = boundKey{kind: boundKindInvalid}
+	} else {
+		l.heapKey = boundKey{
+			kind: boundKindFragmentStart,
+			key:  s.Start,
+			span: s,
+		}
+	}
 }
 
 func (l *mergingIterLevel) prev() {
 	if l.heapKey.kind == boundKindFragmentEnd {
-		l.heapKey = startBound(l.heapKey.span)
+		l.heapKey = boundKey{
+			kind: boundKindFragmentStart,
+			key:  l.heapKey.span.Start,
+			span: l.heapKey.span,
+		}
 		return
 	}
-	l.heapKey = endBound(l.iter.Prev())
+	if s := l.iter.Prev(); s == nil {
+		l.heapKey = boundKey{kind: boundKindInvalid}
+	} else {
+		l.heapKey = boundKey{
+			kind: boundKindFragmentEnd,
+			key:  s.End,
+			span: s,
+		}
+	}
 }
 
 // Init initializes the merging iterator with the provided fragment iterators.
-func (m *MergingIter) Init(cmp base.Compare, transform Transform, iters ...FragmentIterator) {
+func (m *MergingIter) Init(cmp base.Compare, transformer Transformer, iters ...FragmentIterator) {
 	levels, items := m.levels, m.heap.items
 
 	*m = MergingIter{
-		heap:      mergingIterHeap{cmp: cmp},
-		transform: transform,
+		heap:        mergingIterHeap{cmp: cmp},
+		transformer: transformer,
 	}
-	// Invariant: cap(levels) == cap(items)
-	if cap(levels) < len(iters) {
-		m.levels = make([]mergingIterLevel, len(iters))
-		m.heap.items = make([]mergingIterItem, 0, len(iters))
-	} else {
+
+	// Invariant: cap(levels) >= cap(items)
+	// Invariant: cap(alloc.levels) == cap(alloc.items)
+	if len(iters) <= len(m.alloc.levels) {
+		// The slices allocated on the MergingIter struct are large enough.
+		m.levels = m.alloc.levels[:len(iters)]
+		m.heap.items = m.alloc.items[:0]
+	} else if len(iters) <= cap(levels) {
+		// The existing heap-allocated slices are large enough, so reuse them.
 		m.levels = levels[:len(iters)]
 		m.heap.items = items[:0]
+	} else {
+		// Heap allocate new slices.
+		m.levels = make([]mergingIterLevel, len(iters))
+		m.heap.items = make([]mergingIterItem, 0, len(iters))
 	}
 	for i := range m.levels {
-		m.levels[i] = mergingIterLevel{}
-		m.levels[i].iter = iters[i]
+		m.levels[i] = mergingIterLevel{iter: iters[i]}
 	}
 }
 
+// AddLevel adds a new level to the bottom of the merging iterator. AddLevel
+// must be called after Init and before any other method.
+func (m *MergingIter) AddLevel(iter FragmentIterator) {
+	m.levels = append(m.levels, mergingIterLevel{iter: iter})
+}
+
 // SeekGE moves the iterator to the first span with a start key greater than or
 // equal to key.
-func (m *MergingIter) SeekGE(key []byte) Span {
+func (m *MergingIter) SeekGE(key []byte) *Span {
 	m.invalidate() // clear state about current position
 	for i := range m.levels {
 		l := &m.levels[i]
@@ -277,24 +350,35 @@ func (m *MergingIter) SeekGE(key []byte) Span {
 		// Otherwise we use the start boundary of the next span which
 		// necessarily has a start ≥ key.
 		s := l.iter.SeekLT(key)
-		if s.Valid() && m.cmp(s.End, key) >= 0 {
+		if s != nil && m.cmp(s.End, key) >= 0 {
 			// s.End ≥ key
 			// We need to use this span's end bound.
-			l.heapKey = endBound(s)
+			l.heapKey = boundKey{
+				kind: boundKindFragmentEnd,
+				key:  s.End,
+				span: s,
+			}
 			continue
 		}
 		// s.End < key
 		// The span `s` ends before key. Next to the first span with a Start ≥
 		// key, and use that.
-		s = l.iter.Next()
-		l.heapKey = startBound(s)
+		if s = l.iter.Next(); s == nil {
+			l.heapKey = boundKey{kind: boundKindInvalid}
+		} else {
+			l.heapKey = boundKey{
+				kind: boundKindFragmentStart,
+				key:  s.Start,
+				span: s,
+			}
+		}
 	}
 	m.initMinHeap()
 	return m.findNextFragmentSet()
 }
 
 // SeekLT moves the iterator to the last span with a start key less than key.
-func (m *MergingIter) SeekLT(key []byte) Span {
+func (m *MergingIter) SeekLT(key []byte) *Span {
 	// TODO(jackson): Evaluate whether there's an implementation of SeekLT
 	// independent of SeekGE that is more efficient. It's tricky, because the
 	// span we should return might straddle `key` itself.
@@ -314,42 +398,56 @@ func (m *MergingIter) SeekLT(key []byte) Span {
 	// Start user key < key: [b,l)#1. This requires examining bounds both < 'c'
 	// (the 'b' of [b,m)#1's start key) and bounds ≥ 'c' (the 'l' of ([a,l)#2's
 	// end key).
-	if s := m.SeekGE(key); !s.Valid() && m.err != nil {
-		return Span{}
+	if s := m.SeekGE(key); s == nil && m.err != nil {
+		return nil
 	}
 	// Prev to the previous span.
 	return m.Prev()
 }
 
 // First seeks the iterator to the first span.
-func (m *MergingIter) First() Span {
+func (m *MergingIter) First() *Span {
 	m.invalidate() // clear state about current position
 	for i := range m.levels {
-		l := &m.levels[i]
-		l.heapKey = startBound(l.iter.First())
+		if s := m.levels[i].iter.First(); s == nil {
+			m.levels[i].heapKey = boundKey{kind: boundKindInvalid}
+		} else {
+			m.levels[i].heapKey = boundKey{
+				kind: boundKindFragmentStart,
+				key:  s.Start,
+				span: s,
+			}
+		}
 	}
 	m.initMinHeap()
 	return m.findNextFragmentSet()
 }
 
 // Last seeks the iterator to the last span.
-func (m *MergingIter) Last() Span {
+func (m *MergingIter) Last() *Span {
 	m.invalidate() // clear state about current position
 	for i := range m.levels {
-		l := &m.levels[i]
-		l.heapKey = endBound(l.iter.Last())
+		if s := m.levels[i].iter.Last(); s == nil {
+			m.levels[i].heapKey = boundKey{kind: boundKindInvalid}
+		} else {
+			m.levels[i].heapKey = boundKey{
+				kind: boundKindFragmentEnd,
+				key:  s.End,
+				span: s,
+			}
+		}
 	}
 	m.initMaxHeap()
 	return m.findPrevFragmentSet()
 }
 
 // Next advances the iterator to the next span.
-func (m *MergingIter) Next() Span {
+func (m *MergingIter) Next() *Span {
 	if m.err != nil {
-		return Span{}
+		return nil
 	}
 	if m.dir == +1 && (m.end == nil || m.start == nil) {
-		return Span{}
+		return nil
 	}
 	if m.dir != +1 {
 		m.switchToMinHeap()
@@ -358,12 +456,12 @@ func (m *MergingIter) Next() Span {
 }
 
 // Prev advances the iterator to the previous span.
-func (m *MergingIter) Prev() Span {
+func (m *MergingIter) Prev() *Span {
 	if m.err != nil {
-		return Span{}
+		return nil
 	}
 	if m.dir == -1 && (m.end == nil || m.start == nil) {
-		return Span{}
+		return nil
 	}
 	if m.dir != -1 {
 		m.switchToMaxHeap()
@@ -533,7 +631,7 @@ func (m *MergingIter) cmp(a, b []byte) int {
 	return m.heap.cmp(a, b)
 }
 
-func (m *MergingIter) findNextFragmentSet() Span {
+func (m *MergingIter) findNextFragmentSet() *Span {
 	// Each iteration of this loop considers a new merged span between unique
 	// user keys. An iteration may find that there exists no overlap for a given
 	// span, (eg, if the spans [a,b), [d, e) exist within level iterators, the
@@ -601,16 +699,16 @@ func (m *MergingIter) findNextFragmentSet() Span {
 		// we elide empty spans created by the mergingIter itself that don't overlap
 		// with any child iterator returned spans (i.e. empty spans that bridge two
 		// distinct child-iterator-defined spans).
-		if found, s := m.synthesizeKeys(+1); found && s.Valid() {
+		if found, s := m.synthesizeKeys(+1); found && s != nil {
 			return s
 		}
 	}
 	// Exhausted.
 	m.clear()
-	return Span{}
+	return nil
 }
 
-func (m *MergingIter) findPrevFragmentSet() Span {
+func (m *MergingIter) findPrevFragmentSet() *Span {
 	// Each iteration of this loop considers a new merged span between unique
 	// user keys. An iteration may find that there exists no overlap for a given
 	// span, (eg, if the spans [a,b), [d, e) exist within level iterators, the
@@ -677,13 +775,13 @@ func (m *MergingIter) findPrevFragmentSet() Span {
 		// we elide empty spans created by the mergingIter itself that don't overlap
 		// with any child iterator returned spans (i.e. empty spans that bridge two
 		// distinct child-iterator-defined spans).
-		if found, s := m.synthesizeKeys(-1); found && s.Valid() {
+		if found, s := m.synthesizeKeys(-1); found && s != nil {
 			return s
 		}
 	}
 	// Exhausted.
 	m.clear()
-	return Span{}
+	return nil
 }
 
 func (m *MergingIter) heapRoot() []byte {
@@ -703,7 +801,7 @@ func (m *MergingIter) heapRoot() []byte {
 //
 // The boolean return value, `found`, is true if the returned span overlaps
 // with a span returned by a child iterator.
-func (m *MergingIter) synthesizeKeys(dir int8) (bool, Span) {
+func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) {
 	if invariants.Enabled {
 		if m.cmp(m.start, m.end) >= 0 {
 			panic(fmt.Sprintf("pebble: invariant violation: span start ≥ end: %s >= %s", m.start, m.end))
@@ -719,19 +817,26 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, Span) {
 			found = true
 		}
 	}
+	// TODO(jackson): We should be able to remove this sort and instead
+	// guarantee that we'll return keys in the order of the levels they're from.
+	// With careful iterator construction, this would  guarantee that they're
+	// sorted by trailer descending for the range key iteration use case.
 	sort.Sort(&m.keys)
 
-	// Apply the configured transform. See VisibleTransform.
+	// Apply the configured transform. See visibleTransform.
 	s := Span{
-		Start: m.start,
-		End:   m.end,
-		Keys:  m.keys,
+		Start:     m.start,
+		End:       m.end,
+		Keys:      m.keys,
+		KeysOrder: ByTrailerDesc,
 	}
-	if err := m.transform(m.cmp, s, &m.span); err != nil {
+	// NB: m.heap.cmp is a base.Compare, whereas m.cmp is a method on
+	// MergingIter.
+	if err := m.transformer.Transform(m.heap.cmp, s, &m.span); err != nil {
 		m.err = err
-		return false, Span{}
+		return false, nil
 	}
-	return found, m.span
+	return found, &m.span
 }
 
 func (m *MergingIter) invalidate() {
@@ -885,28 +990,6 @@ func (h *mergingIterHeap) down(i0, n int) bool {
 	return i > i0
 }
 
-func startBound(s Span) boundKey {
-	if !s.Valid() {
-		return boundKey{kind: boundKindInvalid}
-	}
-	return boundKey{
-		kind: boundKindFragmentStart,
-		key:  s.Start,
-		span: s,
-	}
-}
-
-func endBound(s Span) boundKey {
-	if !s.Valid() {
-		return boundKey{kind: boundKindInvalid}
-	}
-	return boundKey{
-		kind: boundKindFragmentEnd,
-		key:  s.End,
-		span: s,
-	}
-}
-
 type boundKind int8
 
 const (
@@ -922,7 +1005,7 @@ type boundKey struct {
 	//
 	// If kind is boundKindFragmentStart, then key is span.Start. If kind is
 	// boundKindFragmentEnd, then key is span.End.
-	span Span
+	span *Span
 }
 
 func (k boundKey) valid() bool {

vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go

@@ -6,13 +6,9 @@ package keyspan
 
 import "github.com/cockroachdb/pebble/internal/base"
 
-// TODO(jackson): Move snapshot visiblity outside of SeekGE/SeekLE and into the
-// callsite. This should simplify the code some more.
-
 // SeekGE seeks to the span that contains the target key or the first span past
-// the target key. The snapshot parameter controls the visibility of keys (only
-// spans older than the snapshot sequence number are visible).
-func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) Span {
+// the target key.
+func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
 	// NB: We use SeekLT in order to land on the proper span for a search
 	// key that resides in the middle of a span. Consider the scenario:
 	//
@@ -29,27 +25,17 @@ func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64
 
 	// Invariant: key > iterSpan.Start
 
-	if !iterSpan.Valid() || cmp(key, iterSpan.End) >= 0 {
+	if iterSpan == nil || cmp(key, iterSpan.End) >= 0 {
 		// The current span lies entirely before the search key, or the iterator
 		// is exhausted. Advance the iterator to the next span which is
 		// guaranteed to lie at or past the search key.
 		iterSpan = iter.Next()
 	}
-	// The iterator is positioned at a valid iterSpan which is the first
-	// iterator position that satisfies the requirement that it contains or is
-	// past the target key. It may not be visible based on the snapshot. So now
-	// we only need to move forward and return the first span with visible keys.
-	iterSpan = iterSpan.Visible(snapshot)
-	for iterSpan.Valid() && iterSpan.Empty() {
-		iterSpan = iter.Next().Visible(snapshot)
-	}
 	return iterSpan
 }
 
-// SeekLE seeks to the span that contains or is before the target key.  The
-// snapshot parameter controls the visibility of keys (only spans older than the
-// snapshot sequence number are visible).
-func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) Span {
+// SeekLE seeks to the span that contains or is before the target key.
+func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
 	// NB: We use SeekLT in order to land on the proper span for a search
 	// key that resides in the middle of a span. Consider the scenario:
 	//
@@ -64,13 +50,13 @@ func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64
 	// and we'll have to move forward.
 	iterSpan := iter.SeekLT(key)
 
-	if !iterSpan.Valid() {
+	if iterSpan == nil {
 		// Advance the iterator once to see if the next span has a start key
 		// equal to key.
 		iterSpan = iter.Next()
-		if !iterSpan.Valid() || cmp(key, iterSpan.Start) < 0 {
+		if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
 			// The iterator is exhausted or we've hit the next span.
-			return Span{}
+			return nil
 		}
 	} else {
 		// Invariant: key > iterSpan.Start
@@ -79,20 +65,12 @@ func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64
 			// the next span contains the search key. If it doesn't, we'll backup
 			// and return to our earlier candidate.
 			iterSpan = iter.Next()
-			if !iterSpan.Valid() || cmp(key, iterSpan.Start) < 0 {
+			if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
 				// The next span is past our search key or there is no next span. Go
 				// back.
 				iterSpan = iter.Prev()
 			}
 		}
 	}
-
-	// We're positioned at a span that contains or is before the search key. It
-	// may not be visible based on the snapshot. So now we only need to move
-	// backward and return the first span with visible keys.
-	iterSpan = iterSpan.Visible(snapshot)
-	for iterSpan.Valid() && iterSpan.Empty() {
-		iterSpan = iter.Prev().Visible(snapshot)
-	}
 	return iterSpan
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go

@@ -5,6 +5,7 @@
 package keyspan // import "github.com/cockroachdb/pebble/internal/keyspan"
 
 import (
+	"bytes"
 	"fmt"
 	"sort"
 	"strconv"
@@ -30,16 +31,30 @@ type Span struct {
 	// non-nil, or both nil if representing an invalid Span.
 	Start, End []byte
 	// Keys holds the set of keys applied over the [Start, End) user key range.
-	// Keys is sorted by (SeqNum, Kind) descending. If SeqNum and Kind are
-	// equal, the order of Keys is undefined. Keys may be empty, even if Start
-	// and End are non-nil.
+	// Keys is sorted by (SeqNum, Kind) descending, unless otherwise specified
+	// by the context. If SeqNum and Kind are equal, the order of Keys is
+	// undefined. Keys may be empty, even if Start and End are non-nil.
 	//
 	// Keys are a decoded representation of the internal keys stored in batches
 	// or sstable blocks. A single internal key in a range key block may produce
 	// several decoded Keys.
-	Keys []Key
+	Keys      []Key
+	KeysOrder KeysOrder
 }
 
+// KeysOrder describes the ordering of Keys within a Span.
+type KeysOrder int8
+
+const (
+	// ByTrailerDesc indicates a Span's keys are sorted by Trailer descending.
+	// This is the default ordering, and the ordering used during physical
+	// storage.
+	ByTrailerDesc KeysOrder = iota
+	// BySuffixAsc indicates a Span's keys are sorted by Suffix ascending. This
+	// ordering is used during user iteration of range keys.
+	BySuffixAsc
+)
+
 // Key represents a single key applied over a span of user keys. A Key is
 // contained by a Span which specifies the span of user keys over which the Key
 // is applied.
@@ -65,8 +80,17 @@ func (k Key) Kind() base.InternalKeyKind {
 	return base.InternalKeyKind(k.Trailer & 0xff)
 }
 
+// Equal returns true if this Key is equal to the given key. Two keys are said
+// to be equal if the two Keys have equal trailers, suffix and value. Suffix
+// comparison uses the provided base.Compare func. Value comparison is bytewise.
+func (k Key) Equal(equal base.Equal, b Key) bool {
+	return k.Trailer == b.Trailer &&
+		equal(k.Suffix, b.Suffix) &&
+		bytes.Equal(k.Value, b.Value)
+}
+
 // Valid returns true if the span is defined.
-func (s Span) Valid() bool {
+func (s *Span) Valid() bool {
 	return s.Start != nil && s.End != nil
 }
 
@@ -75,15 +99,18 @@ func (s Span) Valid() bool {
 //
 // An Empty span may be produced by Visible, or be produced by iterators in
 // order to surface the gaps between keys.
-func (s Span) Empty() bool {
-	return len(s.Keys) == 0
+func (s *Span) Empty() bool {
+	return s == nil || len(s.Keys) == 0
 }
 
 // SmallestKey returns the smallest internal key defined by the span's keys.
-// It panics if the span contains no keys.
-func (s Span) SmallestKey() base.InternalKey {
+// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
+// contains no keys or its keys are sorted in a different order.
+func (s *Span) SmallestKey() base.InternalKey {
 	if len(s.Keys) == 0 {
 		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
 	}
 	// The first key has the highest (sequence number,kind) tuple.
 	return base.InternalKey{
@@ -98,10 +125,13 @@ func (s Span) SmallestKey() base.InternalKey {
 // with the maximal sequence number, ensuring all InternalKeys with the same
 // user key sort after the sentinel key.
 //
-// It panics if the span contains no keys.
-func (s Span) LargestKey() base.InternalKey {
+// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
+// contains no keys or its keys are sorted in a different order.
+func (s *Span) LargestKey() base.InternalKey {
 	if len(s.Keys) == 0 {
 		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
 	}
 	// The last key has the lowest (sequence number,kind) tuple.
 	kind := s.Keys[len(s.Keys)-1].Kind()
@@ -109,43 +139,144 @@ func (s Span) LargestKey() base.InternalKey {
 }
 
 // SmallestSeqNum returns the smallest sequence number of a key contained within
-// the span. It panics if the span contains no keys.
-func (s Span) SmallestSeqNum() uint64 {
+// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
+// the span contains no keys or its keys are sorted in a different order.
+func (s *Span) SmallestSeqNum() uint64 {
 	if len(s.Keys) == 0 {
 		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
 	}
+
 	return s.Keys[len(s.Keys)-1].SeqNum()
 }
 
 // LargestSeqNum returns the largest sequence number of a key contained within
-// the span. It panics if the span contains no keys.
-func (s Span) LargestSeqNum() uint64 {
+// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
+// the span contains no keys or its keys are sorted in a different order.
+func (s *Span) LargestSeqNum() uint64 {
 	if len(s.Keys) == 0 {
 		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
 	}
 	return s.Keys[0].SeqNum()
 }
 
+// TODO(jackson): Replace most of the calls to Visible with more targeted calls
+// that avoid the need to construct a new Span.
+
 // Visible returns a span with the subset of keys visible at the provided
-// sequence number.
-func (s Span) Visible(seqNum uint64) Span {
+// sequence number. It requires the Span's keys be in ByTrailerDesc order. It
+// panics if the span's keys are sorted in a different order.
+//
+// Visible may incur an allocation, so callers should prefer targeted,
+// non-allocating methods when possible.
+func (s Span) Visible(snapshot uint64) Span {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+
 	ret := Span{Start: s.Start, End: s.End}
+	if len(s.Keys) == 0 {
+		return ret
+	}
+
+	// Keys from indexed batches may force an allocation. The Keys slice is
+	// ordered by sequence number, so ordinarily we can return the trailing
+	// subslice containing keys with sequence numbers less than `seqNum`.
+	//
+	// However, batch keys are special. Only visible batch keys are included
+	// when an Iterator's batch spans are fragmented. They must always be
+	// visible.
+	//
+	// Batch keys can create a sandwich of visible batch keys at the beginning
+	// of the slice and visible committed keys at the end of the slice, forcing
+	// us to allocate a new slice and copy the contents.
+	//
+	// Care is taking to only incur an allocation only when batch keys and
+	// visible keys actually sandwich non-visible keys.
+
+	// lastBatchIdx and lastNonVisibleIdx are set to the last index of a batch
+	// key and a non-visible key respectively.
+	lastBatchIdx := -1
+	lastNonVisibleIdx := -1
 	for i := range s.Keys {
-		if base.Visible(s.Keys[i].SeqNum(), seqNum) {
-			ret.Keys = s.Keys[i:]
-			break
+		if seqNum := s.Keys[i].SeqNum(); seqNum&base.InternalKeySeqNumBatch != 0 {
+			// Batch key. Always visible.
+			lastBatchIdx = i
+		} else if seqNum >= snapshot {
+			// This key is not visible.
+			lastNonVisibleIdx = i
 		}
 	}
+
+	// In the following comments: b = batch, h = hidden, v = visible (committed).
+	switch {
+	case lastNonVisibleIdx == -1:
+		// All keys are visible.
+		//
+		// [b b b], [v v v] and [b b b v v v]
+		ret.Keys = s.Keys
+	case lastBatchIdx == -1:
+		// There are no batch keys, so we can return the continuous subslice
+		// starting after the last non-visible Key.
+		//
+		// h h h [v v v]
+		ret.Keys = s.Keys[lastNonVisibleIdx+1:]
+	case lastNonVisibleIdx == len(s.Keys)-1:
+		// While we have a batch key and non-visible keys, there are no
+		// committed visible keys. The 'sandwich' is missing the bottom layer,
+		// so we can return the continuous sublice at the beginning.
+		//
+		// [b b b] h h h
+		ret.Keys = s.Keys[0 : lastBatchIdx+1]
+	default:
+		// This is the problematic sandwich case. Allocate a new slice, copying
+		// the batch keys and the visible keys into it.
+		//
+		// [b b b] h h h [v v v]
+		ret.Keys = make([]Key, (lastBatchIdx+1)+(len(s.Keys)-lastNonVisibleIdx-1))
+		copy(ret.Keys, s.Keys[:lastBatchIdx+1])
+		copy(ret.Keys[lastBatchIdx+1:], s.Keys[lastNonVisibleIdx+1:])
+	}
 	return ret
 }
 
+// VisibleAt returns true if the span contains a key visible at the provided
+// snapshot. Keys with sequence numbers with the batch bit set are treated as
+// always visible.
+//
+// VisibleAt requires the Span's keys be in ByTrailerDesc order. It panics if
+// the span's keys are sorted in a different order.
+func (s *Span) VisibleAt(snapshot uint64) bool {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	if len(s.Keys) == 0 {
+		return false
+	} else if first := s.Keys[0].SeqNum(); first&base.InternalKeySeqNumBatch != 0 {
+		// Only visible batch keys are included when an Iterator's batch spans
+		// are fragmented. They must always be visible.
+		return true
+	} else {
+		// Otherwise we check the last key. Since keys are ordered decreasing in
+		// sequence number, the last key has the lowest sequence number of any
+		// of the span's keys. If any of the keys are visible, the last key must
+		// be visible. Or put differently: if the last key is not visible, then
+		// no key is visible.
+		return s.Keys[len(s.Keys)-1].SeqNum() < snapshot
+	}
+}
+
 // ShallowClone returns the span with a Keys slice owned by the span itself.
 // None of the key byte slices are cloned (see Span.DeepClone).
-func (s Span) ShallowClone() Span {
+func (s *Span) ShallowClone() Span {
 	c := Span{
-		Start: s.Start,
-		End:   s.End,
-		Keys:  make([]Key, len(s.Keys)),
+		Start:     s.Start,
+		End:       s.End,
+		Keys:      make([]Key, len(s.Keys)),
+		KeysOrder: s.KeysOrder,
 	}
 	copy(c.Keys, s.Keys)
 	return c
@@ -154,11 +285,12 @@ func (s Span) ShallowClone() Span {
 // DeepClone clones the span, creating copies of all contained slices. DeepClone
 // is intended for non-production code paths like tests, the level checker, etc
 // because it is allocation heavy.
-func (s Span) DeepClone() Span {
+func (s *Span) DeepClone() Span {
 	c := Span{
-		Start: make([]byte, len(s.Start)),
-		End:   make([]byte, len(s.End)),
-		Keys:  make([]Key, len(s.Keys)),
+		Start:     make([]byte, len(s.Start)),
+		End:       make([]byte, len(s.End)),
+		Keys:      make([]Key, len(s.Keys)),
+		KeysOrder: s.KeysOrder,
 	}
 	copy(c.Start, s.Start)
 	copy(c.End, s.End)
@@ -177,15 +309,48 @@ func (s Span) DeepClone() Span {
 }
 
 // Contains returns true if the specified key resides within the span's bounds.
-func (s Span) Contains(cmp base.Compare, key []byte) bool {
+func (s *Span) Contains(cmp base.Compare, key []byte) bool {
 	return cmp(s.Start, key) <= 0 && cmp(key, s.End) < 0
 }
 
 // Covers returns true if the span covers keys at seqNum.
+//
+// Covers requires the Span's keys be in ByTrailerDesc order. It panics if the
+// span's keys are sorted in a different order.
 func (s Span) Covers(seqNum uint64) bool {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
 	return !s.Empty() && s.Keys[0].SeqNum() > seqNum
 }
 
+// CoversAt returns true if the span contains a key that is visible at the
+// provided snapshot sequence number, and that key's sequence number is higher
+// than seqNum.
+//
+// Keys with sequence numbers with the batch bit set are treated as always
+// visible.
+//
+// CoversAt requires the Span's keys be in ByTrailerDesc order. It panics if the
+// span's keys are sorted in a different order.
+func (s *Span) CoversAt(snapshot, seqNum uint64) bool {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	// NB: A key is visible at `snapshot` if its sequence number is strictly
+	// less than `snapshot`. See base.Visible.
+	for i := range s.Keys {
+		if kseq := s.Keys[i].SeqNum(); kseq&base.InternalKeySeqNumBatch != 0 {
+			// Only visible batch keys are included when an Iterator's batch spans
+			// are fragmented. They must always be visible.
+			return kseq > seqNum
+		} else if kseq < snapshot {
+			return kseq > seqNum
+		}
+	}
+	return false
+}
+
 // String returns a string representation of the span.
 func (s Span) String() string {
 	return fmt.Sprint(prettySpan{Span: s, formatKey: base.DefaultFormatter})
@@ -223,10 +388,12 @@ func (s prettySpan) Format(fs fmt.State, c rune) {
 	fmt.Fprintf(fs, "}")
 }
 
-// SortKeys sorts a keys slice by trailer.
-func SortKeys(keys []Key) {
-	sorted := keysBySeqNumKind(keys)
-	sort.Sort(&sorted)
+// SortKeysByTrailer sorts a keys slice by trailer.
+func SortKeysByTrailer(keys *[]Key) {
+	// NB: keys is a pointer to a slice instead of a slice to avoid `sorted`
+	// escaping to the heap.
+	sorted := (*keysBySeqNumKind)(keys)
+	sort.Sort(sorted)
 }
 
 // ParseSpan parses the string representation of a Span. It's intended for

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -12,7 +12,7 @@ import "github.com/cockroachdb/pebble/internal/base"
 func Truncate(
 	cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
 ) FragmentIterator {
-	return Filter(iter, func(in Span, out *Span) (keep bool) {
+	return Filter(iter, func(in *Span, out *Span) (keep bool) {
 		out.Start, out.End = in.Start, in.End
 		out.Keys = append(out.Keys[:0], in.Keys...)
 

vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go

@@ -714,7 +714,17 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
 
 	iter := s.levelMetadata.Iter()
 	for f := iter.First(); f != nil; f = iter.Next() {
-		if !f.Compacting {
+		if invariants.Enabled {
+			if !bytes.Equal(s.orderedIntervals[f.minIntervalIndex].startKey.key, f.Smallest.UserKey) {
+				panic(fmt.Sprintf("f.minIntervalIndex in FileMetadata out of sync with intervals in L0Sublevels: %s != %s",
+					s.formatKey(s.orderedIntervals[f.minIntervalIndex].startKey.key), s.formatKey(f.Smallest.UserKey)))
+			}
+			if !bytes.Equal(s.orderedIntervals[f.maxIntervalIndex+1].startKey.key, f.Largest.UserKey) {
+				panic(fmt.Sprintf("f.maxIntervalIndex in FileMetadata out of sync with intervals in L0Sublevels: %s != %s",
+					s.formatKey(s.orderedIntervals[f.maxIntervalIndex+1].startKey.key), s.formatKey(f.Smallest.UserKey)))
+			}
+		}
+		if !f.IsCompacting() {
 			continue
 		}
 		for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
@@ -795,7 +805,7 @@ func (s *L0Sublevels) describe(verbose bool) string {
 			}
 			sumIntervals += intervals
 			totalBytes += f.Size
-			if f.Compacting {
+			if f.IsCompacting() {
 				numCompactingFiles++
 			}
 		}
@@ -1042,7 +1052,7 @@ func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error {
 				fmt.Fprintf(&buf, "bug %t, seed interval: %d: level %d, sl index %d, f.index %d, min %d, max %d, pre-min %d, pre-max %d, f.min %d, f.max %d, filenum: %d, isCompacting: %t\n%s\n",
 					c.isIntraL0, c.seedInterval, level, index, f.L0Index, min, max, c.preExtensionMinInterval, c.preExtensionMaxInterval,
 					f.minIntervalIndex, f.maxIntervalIndex,
-					f.FileNum, f.Compacting, s)
+					f.FileNum, f.IsCompacting(), s)
 				fmt.Fprintf(&buf, "files included:\n")
 				for _, f := range c.Files {
 					fmt.Fprintf(&buf, "filenum: %d, sl: %d, index: %d, [%d, %d]\n",
@@ -1386,7 +1396,7 @@ func (s *L0Sublevels) PickBaseCompaction(
 			return nil, errors.New("no seed file found in sublevel intervals")
 		}
 		consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1)
-		if f.Compacting {
+		if f.IsCompacting() {
 			if f.IsIntraL0Compacting {
 				// If we're picking a base compaction and we came across a
 				// seed file candidate that's being intra-L0 compacted, skip
@@ -1418,7 +1428,7 @@ func (s *L0Sublevels) PickBaseCompaction(
 				if cmp > 0 || (cmp == 0 && !s.orderedIntervals[c.maxIntervalIndex+1].startKey.isLargest) {
 					break
 				}
-				baseCompacting = baseCompacting || m.Compacting
+				baseCompacting = baseCompacting || m.IsCompacting()
 			}
 			if baseCompacting {
 				continue
@@ -1539,7 +1549,7 @@ func (s *L0Sublevels) extendFiles(
 		if f.minIntervalIndex > cFiles.maxIntervalIndex {
 			break
 		}
-		if f.Compacting {
+		if f.IsCompacting() {
 			return false
 		}
 		// Skip over files that are newer than earliestUnflushedSeqNum. This is
@@ -1590,7 +1600,7 @@ func (s *L0Sublevels) PickIntraL0Compaction(
 		stackDepthReduction := scoredInterval.score
 		for i := len(interval.files) - 1; i >= 0; i-- {
 			f = interval.files[i]
-			if f.Compacting {
+			if f.IsCompacting() {
 				break
 			}
 			consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1)
@@ -1614,7 +1624,7 @@ func (s *L0Sublevels) PickIntraL0Compaction(
 		if f == nil {
 			return nil, errors.New("no seed file found in sublevel intervals")
 		}
-		if f.Compacting {
+		if f.IsCompacting() {
 			// This file could be in a concurrent intra-L0 or base compaction.
 			// Try another interval.
 			continue
@@ -1666,7 +1676,7 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed(
 	for ; slIndex >= 0; slIndex-- {
 		f2 := interval.files[slIndex]
 		sl := f2.SubLevel
-		if f2.Compacting {
+		if f2.IsCompacting() {
 			break
 		}
 		c.seedIntervalStackDepthReduction++
@@ -1936,7 +1946,7 @@ func (s *L0Sublevels) extendCandidateToRectangle(
 		candidateHasAlreadyPickedFiles := false
 		for index = firstIndex; index <= lastIndex; index++ {
 			f := files[index]
-			if f.Compacting {
+			if f.IsCompacting() {
 				if nonCompactingFirst != -1 {
 					last := index - 1
 					// Prioritize runs of consecutive non-compacting files that
@@ -1988,7 +1998,7 @@ func (s *L0Sublevels) extendCandidateToRectangle(
 		}
 		for index := candidateNonCompactingFirst; index <= candidateNonCompactingLast; index++ {
 			f := files[index]
-			if f.Compacting {
+			if f.IsCompacting() {
 				// TODO(bilal): Do a logger.Fatalf instead of a panic, for
 				// cleaner unwinding and error messages.
 				panic(fmt.Sprintf("expected %s to not be compacting", f.FileNum))

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -481,6 +481,21 @@ func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
 	meta := i.seek(func(m *FileMetadata) bool {
 		return cmp(m.Largest.UserKey, userKey) >= 0
 	})
+	for meta != nil {
+		switch i.filter {
+		case KeyTypePointAndRange:
+			return meta
+		case KeyTypePoint:
+			if meta.HasPointKeys && cmp(meta.LargestPointKey.UserKey, userKey) >= 0 {
+				return meta
+			}
+		case KeyTypeRange:
+			if meta.HasRangeKeys && cmp(meta.LargestRangeKey.UserKey, userKey) >= 0 {
+				return meta
+			}
+		}
+		meta = i.Next()
+	}
 	return i.filteredNextFile(meta)
 }
 
@@ -497,6 +512,21 @@ func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata {
 		return cmp(m.Smallest.UserKey, userKey) >= 0
 	})
 	meta := i.Prev()
+	for meta != nil {
+		switch i.filter {
+		case KeyTypePointAndRange:
+			return meta
+		case KeyTypePoint:
+			if meta.HasPointKeys && cmp(meta.SmallestPointKey.UserKey, userKey) < 0 {
+				return meta
+			}
+		case KeyTypeRange:
+			if meta.HasRangeKeys && cmp(meta.SmallestRangeKey.UserKey, userKey) < 0 {
+				return meta
+			}
+		}
+		meta = i.Prev()
+	}
 	return i.filteredPrevFile(meta)
 }
 

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/vfs"
 )
 
@@ -43,15 +44,12 @@ type TableInfo struct {
 
 // TableStats contains statistics on a table used for compaction heuristics.
 type TableStats struct {
-	// Valid true if stats have been loaded for the table. The rest of the
-	// structure is populated only if true.
-	Valid bool
 	// The total number of entries in the table.
 	NumEntries uint64
 	// The number of point and range deletion entries in the table.
 	NumDeletions uint64
-	// NumRangeKeys is the total number of range keys in the table.
-	NumRangeKeys uint64
+	// NumRangeKeySets is the total number of range key sets in the table.
+	NumRangeKeySets uint64
 	// Estimate of the total disk space that may be dropped by this table's
 	// point deletions by compacting them.
 	PointDeletionsBytesEstimate uint64
@@ -77,6 +75,48 @@ const (
 	boundTypeRangeKey
 )
 
+// CompactionState is the compaction state of a file.
+//
+// The following shows the valid state transitions:
+//
+//    NotCompacting --> Compacting --> Compacted
+//          ^               |
+//          |               |
+//          +-------<-------+
+//
+// Input files to a compaction transition to Compacting when a compaction is
+// picked. A file that has finished compacting typically transitions into the
+// Compacted state, at which point it is effectively obsolete ("zombied") and
+// will eventually be removed from the LSM. A file that has been move-compacted
+// will transition from Compacting back into the NotCompacting state, signaling
+// that the file may be selected for a subsequent compaction. A failed
+// compaction will result in all input tables transitioning from Compacting to
+// NotCompacting.
+//
+// This state is in-memory only. It is not persisted to the manifest.
+type CompactionState uint8
+
+// CompactionStates.
+const (
+	CompactionStateNotCompacting CompactionState = iota
+	CompactionStateCompacting
+	CompactionStateCompacted
+)
+
+// String implements fmt.Stringer.
+func (s CompactionState) String() string {
+	switch s {
+	case CompactionStateNotCompacting:
+		return "NotCompacting"
+	case CompactionStateCompacting:
+		return "Compacting"
+	case CompactionStateCompacted:
+		return "Compacted"
+	default:
+		panic(fmt.Sprintf("pebble: unknown compaction state %d", s))
+	}
+}
+
 // FileMetadata holds the metadata for an on-disk table.
 type FileMetadata struct {
 	// Atomic contains fields which are accessed atomically. Go allocations
@@ -90,6 +130,10 @@ type FileMetadata struct {
 		// in pebble.Iterator after every after every positioning operation
 		// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
 		AllowedSeeks int64
+
+		// statsValid is 1 if stats have been loaded for the table. The
+		// TableStats structure is populated only if valid is 1.
+		statsValid uint32
 	}
 
 	// InitAllowedSeeks is the inital value of allowed seeks. This is used
@@ -146,12 +190,11 @@ type FileMetadata struct {
 	// pick L0 compactions. Only accurate for the most recent Version.
 	//
 	// IsIntraL0Compacting is set to True if this file is part of an intra-L0
-	// compaction. When it's true, Compacting must also be true. If Compacting
-	// is true and IsIntraL0Compacting is false for an L0 file, the file must
-	// be part of a compaction to Lbase.
+	// compaction. When it's true, IsCompacting must also return true. If
+	// Compacting is true and IsIntraL0Compacting is false for an L0 file, the
+	// file must be part of a compaction to Lbase.
 	IsIntraL0Compacting bool
-	// True if the file is actively being compacted. Protected by DB.mu.
-	Compacting bool
+	CompactionState     CompactionState
 	// True if compaction of this file has been explicitly requested.
 	// Previously, RocksDB and earlier versions of Pebble allowed this
 	// flag to be set by a user table property collector. Some earlier
@@ -169,9 +212,11 @@ type FileMetadata struct {
 	//
 	// Protected by DB.mu.
 	MarkedForCompaction bool
-	// HasPointKeys and HasRangeKeys track whether the table contains point and
-	// range keys, respectively.
+	// HasPointKeys tracks whether the table contains point keys (including
+	// RANGEDELs). If a table contains only range deletions, HasPointsKeys is
+	// still true.
 	HasPointKeys bool
+	// HasRangeKeys tracks whether the table contains any range keys.
 	HasRangeKeys bool
 	// smallestSet and largestSet track whether the overall bounds have been set.
 	boundsSet bool
@@ -181,6 +226,59 @@ type FileMetadata struct {
 	boundTypeSmallest, boundTypeLargest boundType
 }
 
+// SetCompactionState transitions this file's compaction state to the given
+// state. Protected by DB.mu.
+func (m *FileMetadata) SetCompactionState(to CompactionState) {
+	if invariants.Enabled {
+		transitionErr := func() error {
+			return errors.Newf("pebble: invalid compaction state transition: %s -> %s", m.CompactionState, to)
+		}
+		switch m.CompactionState {
+		case CompactionStateNotCompacting:
+			if to != CompactionStateCompacting {
+				panic(transitionErr())
+			}
+		case CompactionStateCompacting:
+			if to != CompactionStateCompacted && to != CompactionStateNotCompacting {
+				panic(transitionErr())
+			}
+		case CompactionStateCompacted:
+			panic(transitionErr())
+		default:
+			panic(fmt.Sprintf("pebble: unknown compaction state: %d", m.CompactionState))
+		}
+	}
+	m.CompactionState = to
+}
+
+// IsCompacting returns true if this file's compaction state is
+// CompactionStateCompacting. Protected by DB.mu.
+func (m *FileMetadata) IsCompacting() bool {
+	return m.CompactionState == CompactionStateCompacting
+}
+
+// StatsValid returns true if the table stats have been populated. If StatValid
+// returns true, the Stats field may be read (with or without holding the
+// database mutex).
+func (m *FileMetadata) StatsValid() bool {
+	return atomic.LoadUint32(&m.Atomic.statsValid) == 1
+}
+
+// StatsValidLocked returns true if the table stats have been populated.
+// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the
+// overhead of an atomic load. This is possible because table stats validity is
+// only set while DB.mu is held.
+func (m *FileMetadata) StatsValidLocked() bool {
+	return m.Atomic.statsValid == 1
+}
+
+// StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu
+// while populating TableStats and calling StatsMarkValud. Once stats are
+// populated, they must not be mutated.
+func (m *FileMetadata) StatsMarkValid() {
+	atomic.StoreUint32(&m.Atomic.statsValid, 1)
+}
+
 // ExtendPointKeyBounds attempts to extend the lower and upper point key bounds
 // and overall table bounds with the given smallest and largest keys. The
 // smallest and largest bounds may not be extended if the table already has a
@@ -654,6 +752,11 @@ type Version struct {
 
 	Levels [NumLevels]LevelMetadata
 
+	// RangeKeyLevels holds a subset of the same files as Levels that contain range
+	// keys (i.e. fileMeta.HasRangeKeys == true). The memory amplification of this
+	// duplication should be minimal, as range keys are expected to be rare.
+	RangeKeyLevels [NumLevels]LevelMetadata
+
 	// The callback to invoke when the last reference to a version is
 	// removed. Will be called with list.mu held.
 	Deleted func(obsolete []*FileMetadata)
@@ -787,6 +890,9 @@ func (v *Version) unrefFiles() []*FileMetadata {
 	for _, lm := range v.Levels {
 		obsolete = append(obsolete, lm.release()...)
 	}
+	for _, lm := range v.RangeKeyLevels {
+		obsolete = append(obsolete, lm.release()...)
+	}
 	return obsolete
 }
 

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -648,6 +648,11 @@ func (b *BulkVersionEdit) Apply(
 		} else {
 			v.Levels[level] = curr.Levels[level].clone()
 		}
+		if curr == nil || curr.RangeKeyLevels[level].tree.root == nil {
+			v.RangeKeyLevels[level] = makeLevelMetadata(cmp, level, nil /* files */)
+		} else {
+			v.RangeKeyLevels[level] = curr.RangeKeyLevels[level].clone()
+		}
 
 		if len(b.Added[level]) == 0 && len(b.Deleted[level]) == 0 {
 			// There are no edits on this level.
@@ -667,6 +672,7 @@ func (b *BulkVersionEdit) Apply(
 
 		// Some edits on this level.
 		lm := &v.Levels[level]
+		lmRange := &v.RangeKeyLevels[level]
 		addedFiles := b.Added[level]
 		deletedMap := b.Deleted[level]
 		if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
@@ -688,6 +694,16 @@ func (b *BulkVersionEdit) Apply(
 				err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
 				return nil, nil, err
 			}
+			if f.HasRangeKeys {
+				if obsolete := v.RangeKeyLevels[level].tree.delete(f); obsolete {
+					// Deleting a file from the B-Tree may decrement its
+					// reference count. However, because we cloned the
+					// previous level's B-Tree, this should never result in a
+					// file's reference count dropping to zero.
+					err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum)
+					return nil, nil, err
+				}
+			}
 		}
 
 		var sm, la *FileMetadata
@@ -714,6 +730,12 @@ func (b *BulkVersionEdit) Apply(
 			if err != nil {
 				return nil, nil, errors.Wrap(err, "pebble")
 			}
+			if f.HasRangeKeys {
+				err = lmRange.tree.insert(f)
+				if err != nil {
+					return nil, nil, errors.Wrap(err, "pebble")
+				}
+			}
 			removeZombie(f.FileNum)
 			// Track the keys with the smallest and largest keys, so that we can
 			// check consistency of the modified span.

vendor/github.com/cockroachdb/pebble/internal/rangedel/rangedel.go

@@ -13,7 +13,7 @@ import (
 // closure with the encoded internal keys that represent the Span's state. The
 // keys and values passed to emit are only valid until the closure returns.  If
 // emit returns an error, Encode stops and returns the error.
-func Encode(s keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
+func Encode(s *keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
 	for _, k := range s.Keys {
 		if k.Kind() != base.InternalKeyKindRangeDelete {
 			return base.CorruptionErrorf("pebble: rangedel.Encode cannot encode %s key", k.Kind())

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -11,117 +11,162 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manifest"
 )
 
-// InitUserIteration initializes the range key iterator stack for user
-// iteration. The resulting fragment iterator applies range key semantics,
-// defragments spans according to their user-observable state and removes all
-// Keys other than RangeKeySets describing the current state of range keys.
+// UserIteratorConfig holds state for constructing the range key iterator stack
+// for user iteration.
+type UserIteratorConfig struct {
+	snapshot   uint64
+	comparer   *base.Comparer
+	miter      keyspan.MergingIter
+	biter      keyspan.BoundedIter
+	diter      keyspan.DefragmentingIter
+	liters     [manifest.NumLevels]keyspan.LevelIter
+	litersUsed int
+	sortBuf    keysBySuffix
+}
+
+// Init initializes the range key iterator stack for user iteration. The
+// resulting fragment iterator applies range key semantics, defragments spans
+// according to their user-observable state and removes all Keys other than
+// RangeKeySets describing the current state of range keys. The resulting spans
+// contain Keys sorted by Suffix.
 //
 // The snapshot sequence number parameter determines which keys are visible. Any
 // keys not visible at the provided snapshot are ignored.
-func InitUserIteration(
-	cmp base.Compare,
+func (ui *UserIteratorConfig) Init(
+	comparer *base.Comparer,
 	snapshot uint64,
-	miter *keyspan.MergingIter,
-	diter *keyspan.DefragmentingIter,
-	levelIters ...keyspan.FragmentIterator,
+	lower, upper []byte,
+	hasPrefix *bool,
+	prefix *[]byte,
+	iters ...keyspan.FragmentIterator,
 ) keyspan.FragmentIterator {
-	miter.Init(cmp, userIterationTransform(snapshot), levelIters...)
-	diter.Init(cmp, miter, userIterationDefragmenter(), keyspan.StaticDefragmentReducer)
-	return diter
+	ui.snapshot = snapshot
+	ui.comparer = comparer
+	ui.miter.Init(comparer.Compare, ui, iters...)
+	ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
+	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer)
+	ui.litersUsed = 0
+	return &ui.diter
 }
 
-// userIterationTransform returns a keyspan.Transform for use with a
-// keyspan.MergingIter that transforms spans by resolving range keys at the
+// AddLevel adds a new level to the bottom of the iterator stack. AddLevel
+// must be called after Init and before any other method on the iterator.
+func (ui *UserIteratorConfig) AddLevel(iter keyspan.FragmentIterator) {
+	ui.miter.AddLevel(iter)
+}
+
+// NewLevelIter returns a pointer to a newly allocated or reused
+// keyspan.LevelIter. The caller is responsible for calling Init() on this
+// instance.
+func (ui *UserIteratorConfig) NewLevelIter() *keyspan.LevelIter {
+	if ui.litersUsed >= len(ui.liters) {
+		return &keyspan.LevelIter{}
+	}
+	ui.litersUsed++
+	return &ui.liters[ui.litersUsed-1]
+}
+
+// SetBounds propagates bounds to the iterator stack. The fragment iterator
+// interface ordinarily doesn't enforce bounds, so this is exposed as an
+// explicit method on the user iterator config.
+func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
+	ui.biter.SetBounds(lower, upper)
+}
+
+// Transform implements the keyspan.Transformer interface for use with a
+// keyspan.MergingIter. It transforms spans by resolving range keys at the
 // provided snapshot sequence number. Shadowing of keys is resolved (eg, removal
 // of unset keys, removal of keys overwritten by a set at the same suffix, etc)
 // and then non-RangeKeySet keys are removed. The resulting transformed spans
 // only contain RangeKeySets describing the state visible at the provided
-// sequence number.
-func userIterationTransform(snapshot uint64) keyspan.Transform {
-	return func(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
-		// Apply shadowing of keys.
-		if err := Coalesce(cmp, s.Visible(snapshot), dst); err != nil {
-			return err
-		}
-
-		// During user iteration over range keys, unsets and deletes don't
-		// matter. Remove them. This step helps logical defragmentation during
-		// iteration.
-		keys := dst.Keys
-		dst.Keys = dst.Keys[:0]
-		for i := range keys {
-			switch keys[i].Kind() {
-			case base.InternalKeyKindRangeKeySet:
-				dst.Keys = append(dst.Keys, keys[i])
-			case base.InternalKeyKindRangeKeyUnset:
-				// Skip.
-				continue
-			case base.InternalKeyKindRangeKeyDelete:
-				// Skip.
-				continue
-			default:
-				return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
-			}
-		}
-		return nil
+// sequence number, and hold their Keys sorted by Suffix.
+func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
+	// Apply shadowing of keys.
+	dst.Start = s.Start
+	dst.End = s.End
+	ui.sortBuf = keysBySuffix{
+		cmp:  cmp,
+		keys: dst.Keys[:0],
 	}
+	if err := coalesce(&ui.sortBuf, s.Visible(ui.snapshot).Keys, &dst.Keys); err != nil {
+		return err
+	}
+	// During user iteration over range keys, unsets and deletes don't
+	// matter. Remove them. This step helps logical defragmentation during
+	// iteration.
+	keys := dst.Keys
+	dst.Keys = dst.Keys[:0]
+	for i := range keys {
+		switch keys[i].Kind() {
+		case base.InternalKeyKindRangeKeySet:
+			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
+				panic("pebble: keys unexpectedly not in ascending suffix order")
+			}
+			dst.Keys = append(dst.Keys, keys[i])
+		case base.InternalKeyKindRangeKeyUnset:
+			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
+				panic("pebble: keys unexpectedly not in ascending suffix order")
+			}
+			// Skip.
+			continue
+		case base.InternalKeyKindRangeKeyDelete:
+			// Skip.
+			continue
+		default:
+			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
+		}
+	}
+	// coalesce results in dst.Keys being sorted by Suffix.
+	dst.KeysOrder = keyspan.BySuffixAsc
+	return nil
 }
 
-// userIterationDefragmenter constructs a DefragmentMethod that configures a
+// ShouldDefragment implements the DefragmentMethod interface and configures a
 // DefragmentingIter to defragment spans of range keys if their user-visible
 // state is identical. This defragmenting method assumes the provided spans have
-// already been transformed through UserIterationTransform, so all RangeKeySets
-// are user-visible sets. This defragmenter checks for equality between set
-// suffixes and values (ignoring sequence numbers). It's intended for use during
-// user iteration, when the wrapped keyspan iterator is merging spans across all
-// levels of the LSM.
-//
-// The returned defragmenting method is stateful, and must not be used on
-// multiple DefragmentingIters concurrently.
-func userIterationDefragmenter() keyspan.DefragmentMethod {
-	var bufA keysBySuffix
-	var bufB keysBySuffix
-	return func(cmp base.Compare, a, b keyspan.Span) bool {
-		// UserIterationDefragmenter must only be used on spans that have
-		// transformed by UserIterationTransform. The transform applies
-		// shadowing and removes all keys besides the resulting Sets. Since
-		// shadowing has been applied, each Set must set a unique suffix. If the
-		// two spans are equivalent, they must have the same number of range key
-		// sets.
-		if len(a.Keys) != len(b.Keys) {
-			return false
-		}
+// already been transformed through (UserIterationConfig).Transform, so all
+// RangeKeySets are user-visible sets and are already in Suffix order. This
+// defragmenter checks for equality between set suffixes and values (ignoring
+// sequence numbers). It's intended for use during user iteration, when the
+// wrapped keyspan iterator is merging spans across all levels of the LSM.
+func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
+	// This implementation must only be used on spans that have transformed by
+	// ui.Transform. The transform applies shadowing, removes all keys besides
+	// the resulting Sets and sorts the keys by suffix. Since shadowing has been
+	// applied, each Set must set a unique suffix. If the two spans are
+	// equivalent, they must have the same number of range key sets.
+	if len(a.Keys) != len(b.Keys) || len(a.Keys) == 0 {
+		return false
+	}
+	if a.KeysOrder != keyspan.BySuffixAsc || b.KeysOrder != keyspan.BySuffixAsc {
+		panic("pebble: range key span's keys unexpectedly not in ascending suffix order")
+	}
 
-		// The keys in both spans must be all sets. In order to be equivalent
-		// during user iteration, they must set identical suffix-value mappings.
-		// They do not need to have the same sequence number.
-		//
-		// Currently, the Keys are sorted by Trailer (as per the keyspan.Span
-		// and keyspan.FragmentIterator interfaces). Collect all the keys in
-		// both spans into buffers and sort them by suffix for comparison.
-		bufA.cmp = cmp
-		bufA.keys = append(bufA.keys[:0], a.Keys...)
-		sort.Sort(bufA)
-		bufB.cmp = cmp
-		bufB.keys = append(bufB.keys[:0], b.Keys...)
-		sort.Sort(bufB)
-
-		for i := range bufA.keys {
-			if bufA.keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
-				bufB.keys[i].Kind() != base.InternalKeyKindRangeKeySet {
+	ret := true
+	for i := range a.Keys {
+		if invariants.Enabled {
+			if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
+				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
 				panic("pebble: unexpected non-RangeKeySet during defragmentation")
 			}
-			if cmp(bufA.keys[i].Suffix, bufB.keys[i].Suffix) != 0 {
-				return false
-			}
-			if !bytes.Equal(bufA.keys[i].Value, bufB.keys[i].Value) {
-				return false
+			if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||
+				ui.comparer.Compare(b.Keys[i].Suffix, b.Keys[i-1].Suffix) < 0) {
+				panic("pebble: range keys not ordered by suffix during defragmentation")
 			}
 		}
-		return true
+		if !equal(a.Keys[i].Suffix, b.Keys[i].Suffix) {
+			ret = false
+			break
+		}
+		if !bytes.Equal(a.Keys[i].Value, b.Keys[i].Value) {
+			ret = false
+			break
+		}
 	}
+	return ret
 }
 
 // Coalesce imposes range key semantics and coalesces range keys with the same
@@ -157,18 +202,28 @@ func userIterationDefragmenter() keyspan.DefragmentMethod {
 // keys do not affect one another. Ingested sstables are expected to be
 // consistent with respect to the set/unset suffixes: A given suffix should be
 // set or unset but not both.
-func Coalesce(cmp base.Compare, span keyspan.Span, dst *keyspan.Span) error {
+//
+// The resulting dst Keys slice is sorted by Trailer.
+func Coalesce(cmp base.Compare, keys []keyspan.Key, dst *[]keyspan.Key) error {
 	// TODO(jackson): Currently, Coalesce doesn't actually perform the sequence
 	// number promotion described in the comment above.
-
 	keysBySuffix := keysBySuffix{
 		cmp:  cmp,
-		keys: dst.Keys[:0],
+		keys: (*dst)[:0],
 	}
+	if err := coalesce(&keysBySuffix, keys, dst); err != nil {
+		return err
+	}
+	// coalesce left the keys in *dst sorted by suffix. Re-sort them by trailer.
+	keyspan.SortKeysByTrailer(dst)
+	return nil
+}
+
+func coalesce(keysBySuffix *keysBySuffix, keys []keyspan.Key, dst *[]keyspan.Key) error {
 	var deleted bool
-	for i := 0; i < len(span.Keys) && !deleted; i++ {
-		k := span.Keys[i]
-		if invariants.Enabled && i > 0 && k.Trailer > span.Keys[i-1].Trailer {
+	for i := 0; i < len(keys) && !deleted; i++ {
+		k := keys[i]
+		if invariants.Enabled && i > 0 && k.Trailer > keys[i-1].Trailer {
 			panic("pebble: invariant violation: span keys unordered")
 		}
 
@@ -210,26 +265,13 @@ func Coalesce(cmp base.Compare, span keyspan.Span, dst *keyspan.Span) error {
 		}
 	}
 
-	// Update the span with the (potentially reduced) keys slice, and re-sort it
-	// by Trailer.
-	*dst = keyspan.Span{
-		Start: span.Start,
-		End:   span.End,
-		Keys:  keysBySuffix.keys,
-	}
-	keyspan.SortKeys(dst.Keys)
+	// Update the span with the (potentially reduced) keys slice.
+	// NB: We don't re-sort by Trailer. The exported Coalesce function however
+	// will.
+	*dst = keysBySuffix.keys
 	return nil
 }
 
-// SortBySuffix sorts the provided keys by suffix.
-func SortBySuffix(cmp base.Compare, keys []keyspan.Key) {
-	bySuffix := keysBySuffix{
-		cmp:  cmp,
-		keys: keys,
-	}
-	sort.Sort(bySuffix)
-}
-
 type keysBySuffix struct {
 	cmp  base.Compare
 	keys []keyspan.Key
@@ -238,7 +280,7 @@ type keysBySuffix struct {
 // get searches for suffix among the first n keys in keys. If the suffix is
 // found, it returns the index of the item with the suffix. If the suffix is not
 // found, it returns n.
-func (s keysBySuffix) get(n int, suffix []byte) (i int) {
+func (s *keysBySuffix) get(n int, suffix []byte) (i int) {
 	// Binary search for the suffix to see if there's an existing key with the
 	// suffix. Only binary search among the first n items. get is called while
 	// appending new keys with suffixes that may sort before existing keys.
@@ -254,6 +296,6 @@ func (s keysBySuffix) get(n int, suffix []byte) (i int) {
 	return n
 }
 
-func (s keysBySuffix) Len() int           { return len(s.keys) }
-func (s keysBySuffix) Less(i, j int) bool { return s.cmp(s.keys[i].Suffix, s.keys[j].Suffix) < 0 }
-func (s keysBySuffix) Swap(i, j int)      { s.keys[i], s.keys[j] = s.keys[j], s.keys[i] }
+func (s *keysBySuffix) Len() int           { return len(s.keys) }
+func (s *keysBySuffix) Less(i, j int) bool { return s.cmp(s.keys[i].Suffix, s.keys[j].Suffix) < 0 }
+func (s *keysBySuffix) Swap(i, j int)      { s.keys[i], s.keys[j] = s.keys[j], s.keys[i] }

vendor/github.com/cockroachdb/pebble/internal/rangekey/rangekey.go

@@ -61,7 +61,7 @@ import (
 // closure with the encoded internal keys that represent the Span's state. The
 // keys and values passed to emit are only valid until the closure returns.
 // If emit returns an error, Encode stops and returns the error.
-func Encode(s keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
+func Encode(s *keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
 	enc := Encoder{Emit: emit}
 	return enc.Encode(s)
 }
@@ -82,7 +82,7 @@ type Encoder struct {
 //
 // The encoded key-value pair passed to Emit is only valid until the closure
 // completes.
-func (e *Encoder) Encode(s keyspan.Span) error {
+func (e *Encoder) Encode(s *keyspan.Span) error {
 	if s.Empty() {
 		return nil
 	}
@@ -127,7 +127,7 @@ func (e *Encoder) Encode(s keyspan.Span) error {
 
 // flush constructs internal keys for accumulated key state, and emits the
 // internal keys.
-func (e *Encoder) flush(s keyspan.Span, seqNum uint64, del bool) error {
+func (e *Encoder) flush(s *keyspan.Span, seqNum uint64, del bool) error {
 	if len(e.sets) > 0 {
 		ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeySet)
 		l := EncodedSetValueLen(s.End, e.sets)

vendor/github.com/cockroachdb/pebble/internal/testkeys/strconv.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2013 The Perkeep Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testkeys
+
+import (
+	"strconv"
+
+	"github.com/cockroachdb/errors"
+)
+
+// parseUintBytes is like strconv.ParseUint, but using a []byte. Use of this
+// function avoids an allocation when parsing an integer out of a []byte.
+//
+// This function is copied from go4.org/strconv.
+func parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) {
+	var cutoff, maxVal uint64
+
+	if bitSize == 0 {
+		bitSize = int(strconv.IntSize)
+	}
+
+	s0 := s
+	switch {
+	case len(s) < 1:
+		err = strconv.ErrSyntax
+		goto Error
+
+	case 2 <= base && base <= 36:
+		// valid base; nothing to do
+
+	case base == 0:
+		// Look for octal, hex prefix.
+		switch {
+		case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'):
+			base = 16
+			s = s[2:]
+			if len(s) < 1 {
+				err = strconv.ErrSyntax
+				goto Error
+			}
+		case s[0] == '0':
+			base = 8
+		default:
+			base = 10
+		}
+
+	default:
+		err = errors.New("invalid base " + strconv.Itoa(base))
+		goto Error
+	}
+
+	n = 0
+	cutoff = cutoff64(base)
+	maxVal = 1<<uint(bitSize) - 1
+
+	for i := 0; i < len(s); i++ {
+		var v byte
+		d := s[i]
+		switch {
+		case '0' <= d && d <= '9':
+			v = d - '0'
+		case 'a' <= d && d <= 'z':
+			v = d - 'a' + 10
+		case 'A' <= d && d <= 'Z':
+			v = d - 'A' + 10
+		default:
+			n = 0
+			err = strconv.ErrSyntax
+			goto Error
+		}
+		if int(v) >= base {
+			n = 0
+			err = strconv.ErrSyntax
+			goto Error
+		}
+
+		if n >= cutoff {
+			// n*base overflows
+			n = 1<<64 - 1
+			err = strconv.ErrRange
+			goto Error
+		}
+		n *= uint64(base)
+
+		n1 := n + uint64(v)
+		if n1 < n || n1 > maxVal {
+			// n+v overflows
+			n = 1<<64 - 1
+			err = strconv.ErrRange
+			goto Error
+		}
+		n = n1
+	}
+
+	return n, nil
+
+Error:
+	return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
+}
+
+// Return the first number n such that n*base >= 1<<64.
+func cutoff64(base int) uint64 {
+	if base < 2 {
+		return 0
+	}
+	return (1<<64-1)/uint64(base) + 1
+}

vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go

@@ -0,0 +1,349 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+// Package testkeys provides facilities for generating and comparing
+// human-readable test keys for use in tests and benchmarks. This package
+// provides a single Comparer implementation that compares all keys generated
+// by this package.
+//
+// Keys generated by this package may optionally have a 'suffix' encoding an
+// MVCC timestamp. This suffix is of the form "@<integer>". Comparisons on the
+// suffix are performed using integer value, not the byte representation.
+package testkeys
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+const alpha = "abcdefghijklmnopqrstuvwxyz"
+
+const suffixDelim = '@'
+
+// MaxSuffixLen is the maximum length of a suffix generated by this package.
+var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64))
+
+// Comparer is the comparer for test keys generated by this package.
+var Comparer *base.Comparer = &base.Comparer{
+	Compare: compare,
+	Equal:   func(a, b []byte) bool { return compare(a, b) == 0 },
+	AbbreviatedKey: func(k []byte) uint64 {
+		return base.DefaultComparer.AbbreviatedKey(k[:split(k)])
+	},
+	FormatKey: base.DefaultFormatter,
+	Separator: func(dst, a, b []byte) []byte {
+		ai := split(a)
+		if ai == len(a) {
+			return append(dst, a...)
+		}
+		bi := split(b)
+		if bi == len(b) {
+			return append(dst, a...)
+		}
+
+		// If the keys are the same just return a.
+		if bytes.Equal(a[:ai], b[:bi]) {
+			return append(dst, a...)
+		}
+		n := len(dst)
+		dst = base.DefaultComparer.Separator(dst, a[:ai], b[:bi])
+		// Did it pick a separator different than a[:ai] -- if not we can't do better than a.
+		buf := dst[n:]
+		if bytes.Equal(a[:ai], buf) {
+			return append(dst[:n], a...)
+		}
+		// The separator is > a[:ai], so return it
+		return dst
+	},
+	Successor: func(dst, a []byte) []byte {
+		ai := split(a)
+		if ai == len(a) {
+			return append(dst, a...)
+		}
+		n := len(dst)
+		dst = base.DefaultComparer.Successor(dst, a[:ai])
+		// Did it pick a successor different than a[:ai] -- if not we can't do better than a.
+		buf := dst[n:]
+		if bytes.Equal(a[:ai], buf) {
+			return append(dst[:n], a...)
+		}
+		// The successor is > a[:ai], so return it.
+		return dst
+	},
+	ImmediateSuccessor: func(dst, a []byte) []byte {
+		// TODO(jackson): Consider changing this Comparer to only support
+		// representable prefix keys containing characters a-z.
+		ai := split(a)
+		if ai != len(a) {
+			panic("pebble: ImmediateSuccessor invoked with a non-prefix key")
+		}
+		return append(append(dst, a...), 0x00)
+	},
+	Split: split,
+	Name:  "pebble.internal.testkeys",
+}
+
+func compare(a, b []byte) int {
+	ai, bi := split(a), split(b)
+	if v := bytes.Compare(a[:ai], b[:bi]); v != 0 {
+		return v
+	}
+
+	if len(a[ai:]) == 0 {
+		if len(b[bi:]) == 0 {
+			return 0
+		}
+		return -1
+	} else if len(b[bi:]) == 0 {
+		return +1
+	}
+	return compareTimestamps(a[ai:], b[bi:])
+}
+
+func split(a []byte) int {
+	i := bytes.LastIndexByte(a, suffixDelim)
+	if i >= 0 {
+		return i
+	}
+	return len(a)
+}
+
+func compareTimestamps(a, b []byte) int {
+	ai, err := parseUintBytes(bytes.TrimPrefix(a, []byte{suffixDelim}), 10, 64)
+	if err != nil {
+		panic(fmt.Sprintf("invalid test mvcc timestamp %q", a))
+	}
+	bi, err := parseUintBytes(bytes.TrimPrefix(b, []byte{suffixDelim}), 10, 64)
+	if err != nil {
+		panic(fmt.Sprintf("invalid test mvcc timestamp %q", b))
+	}
+	switch {
+	case ai < bi:
+		return +1
+	case ai > bi:
+		return -1
+	default:
+		return 0
+	}
+}
+
+// Keyspace describes a finite keyspace of unsuffixed test keys.
+type Keyspace interface {
+	// Count returns the number of keys that exist within this keyspace.
+	Count() int
+
+	// MaxLen returns the maximum length, in bytes, of a key within this
+	// keyspace. This is only guaranteed to return an upper bound.
+	MaxLen() int
+
+	// Slice returns the sub-keyspace from index i, inclusive, to index j,
+	// exclusive. The receiver is unmodified.
+	Slice(i, j int) Keyspace
+
+	// EveryN returns a key space that includes 1 key for every N keys in the
+	// original keyspace. The receiver is unmodified.
+	EveryN(n int) Keyspace
+
+	key(buf []byte, i int) int
+}
+
+// Divvy divides the provided keyspace into N equal portions, containing
+// disjoint keys evenly distributed across the keyspace.
+func Divvy(ks Keyspace, n int) []Keyspace {
+	ret := make([]Keyspace, n)
+	for i := 0; i < n; i++ {
+		ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
+	}
+	return ret
+}
+
+// Alpha constructs a keyspace consisting of all keys containing characters a-z,
+// with at most `maxLength` characters.
+func Alpha(maxLength int) Keyspace {
+	return alphabet{
+		alphabet:  []byte(alpha),
+		maxLength: maxLength,
+		increment: 1,
+	}
+}
+
+// KeyAt returns the i-th key within the keyspace with a suffix encoding the
+// timestamp t.
+func KeyAt(k Keyspace, i int, t int) []byte {
+	b := make([]byte, k.MaxLen()+MaxSuffixLen)
+	return b[:WriteKeyAt(b, k, i, t)]
+}
+
+// WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
+// suffix encoding the timestamp t suffix. It returns the number of bytes
+// written.
+func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int {
+	n := WriteKey(dst, k, i)
+	n += WriteSuffix(dst[n:], t)
+	return n
+}
+
+// Suffix returns the test keys suffix representation of timestamp t.
+func Suffix(t int) []byte {
+	b := make([]byte, MaxSuffixLen)
+	return b[:WriteSuffix(b, t)]
+}
+
+// SuffixLen returns the exact length of the given suffix when encoded.
+func SuffixLen(t int) int {
+	// Begin at 1 for the '@' delimiter, 1 for a single digit.
+	n := 2
+	t /= 10
+	for t > 0 {
+		t /= 10
+		n++
+	}
+	return n
+}
+
+// ParseSuffix returns the integer representation of the encoded suffix.
+func ParseSuffix(s []byte) (int, error) {
+	return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
+}
+
+// WriteSuffix writes the test keys suffix representation of timestamp t to dst,
+// returning the number of bytes written.
+func WriteSuffix(dst []byte, t int) int {
+	dst[0] = suffixDelim
+	n := 1
+	n += len(strconv.AppendInt(dst[n:n], int64(t), 10))
+	return n
+}
+
+// Key returns the i-th unsuffixed key within the keyspace.
+func Key(k Keyspace, i int) []byte {
+	b := make([]byte, k.MaxLen())
+	return b[:k.key(b, i)]
+}
+
+// WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
+// returns the number of bytes written.
+func WriteKey(dst []byte, k Keyspace, i int) int {
+	return k.key(dst, i)
+}
+
+type alphabet struct {
+	alphabet  []byte
+	maxLength int
+	headSkip  int
+	tailSkip  int
+	increment int
+}
+
+func (a alphabet) Count() int {
+	// Calculate the total number of keys, ignoring the increment.
+	total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip)
+
+	// The increment dictates that we take every N keys, where N = a.increment.
+	// Consider a total containing the 5 keys:
+	//   a  b  c  d  e
+	//   ^     ^     ^
+	// If the increment is 2, this keyspace includes 'a', 'c' and 'e'. After
+	// dividing by the increment, there may be remainder. If there is, there's
+	// one additional key in the alphabet.
+	count := total / a.increment
+	if total%a.increment > 0 {
+		count++
+	}
+	return count
+}
+
+func (a alphabet) MaxLen() int {
+	return a.maxLength
+}
+
+func (a alphabet) Slice(i, j int) Keyspace {
+	s := a
+	s.headSkip += i
+	s.tailSkip += a.Count() - j
+	return s
+}
+
+func (a alphabet) EveryN(n int) Keyspace {
+	s := a
+	s.increment *= n
+	return s
+}
+
+func keyCount(n, l int) int {
+	// The number of representable keys in the keyspace is a function of the
+	// length of the alphabet n and the max key length l. Consider how the
+	// number of representable keys grows as l increases:
+	//
+	// l = 1: n
+	// l = 2: n + n^2
+	// l = 3: n + n^2 + n^3
+	// ...
+	// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
+	return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1)
+}
+
+func (a alphabet) key(buf []byte, idx int) int {
+	// This function generates keys of length 1..maxKeyLength, pulling
+	// characters from the alphabet. The idx determines which key to generate,
+	// generating the i-th lexicographically next key.
+	//
+	// The index to use is advanced by `headSkip`, allowing a keyspace to encode
+	// a subregion of the keyspace.
+	//
+	// Eg, alphabet = `ab`, maxKeyLength = 3:
+	//
+	//           aaa aab     aba abb         baa bab     bba bbb
+	//       aa          ab              ba          bb
+	//   a                           b
+	//   0   1   2   3   4   5   6   7   8   9   10  11  12  13
+	//
+	return generateAlphabetKey(buf, a.alphabet, (idx*a.increment)+a.headSkip,
+		keyCount(len(a.alphabet), a.maxLength))
+}
+
+func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
+	if keyCount == 0 || i > keyCount || i < 0 {
+		return 0
+	}
+
+	// Of the keyCount keys in the generative keyspace, how many are there
+	// starting with a particular character?
+	keysPerCharacter := keyCount / len(alphabet)
+
+	// Find the character that the key at index i starts with and set it.
+	characterIdx := i / keysPerCharacter
+	buf[0] = alphabet[characterIdx]
+
+	// Consider characterIdx = 0, pointing to 'a'.
+	//
+	//           aaa aab     aba abb         baa bab     bba bbb
+	//       aa          ab              ba          bb
+	//   a                           b
+	//   0   1   2   3   4   5   6   7   8   9   10  11  12  13
+	//  \_________________________/
+	//    |keysPerCharacter| keys
+	//
+	// In our recursive call, we reduce the problem to:
+	//
+	//           aaa aab     aba abb
+	//       aa          ab
+	//       0   1   2   3   4   5
+	//     \________________________/
+	//    |keysPerCharacter-1| keys
+	//
+	// In the subproblem, there are keysPerCharacter-1 keys (eliminating the
+	// just 'a' key, plus any keys beginning with any other character).
+	//
+	// The index i is also offset, reduced by the count of keys beginning with
+	// characters earlier in the alphabet (keysPerCharacter*characterIdx) and
+	// the key consisting of just the 'a' (-1).
+	i = i - keysPerCharacter*characterIdx - 1
+	return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
+}