Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go

@@ -79,6 +79,10 @@ func (it *flushIterator) Next() (*base.InternalKey, base.LazyValue) {
 	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
+func (it *flushIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	panic("pebble: NextPrefix unimplemented")
+}
+
 func (it *flushIterator) Prev() (*base.InternalKey, base.LazyValue) {
 	panic("pebble: Prev unimplemented")
 }

vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go

@@ -194,6 +194,13 @@ func (it *Iterator) Next() (*base.InternalKey, base.LazyValue) {
 	return &it.key, base.MakeInPlaceValue(it.value())
 }
 
+// NextPrefix advances to the next position with a new prefix. Returns the key
+// and value if the iterator is pointing at a valid entry, and (nil, nil)
+// otherwise.
+func (it *Iterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return it.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext())
+}
+
 // Prev moves to the previous position. Returns the key and value if the
 // iterator is pointing at a valid entry, and (nil, nil) otherwise.
 func (it *Iterator) Prev() (*base.InternalKey, base.LazyValue) {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go

@@ -68,7 +68,7 @@ const (
 // keys.
 var ErrRecordExists = errors.New("record with this key already exists")
 
-// Skiplist is a fast, cocnurrent skiplist implementation that supports forward
+// Skiplist is a fast, concurrent skiplist implementation that supports forward
 // and backward iteration. See batchskl.Skiplist for a non-concurrent
 // skiplist. Keys and values are immutable once added to the skiplist and
 // deletion is not supported. Instead, higher-level code is expected to add new

vendor/github.com/cockroachdb/pebble/internal/base/comparer.go

@@ -70,11 +70,11 @@ type Successor func(dst, a []byte) []byte
 // returns the smallest key that is larger than the given prefix a.
 // ImmediateSuccessor must return a prefix key k such that:
 //
-//   Split(k) == len(k) and Compare(k, a) > 0
+//	Split(k) == len(k) and Compare(k, a) > 0
 //
 // and there exists no representable k2 such that:
 //
-//   Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
+//	Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
 //
 // As an example, an implementation built on the natural byte ordering using
 // bytes.Compare could append a `\0` to `a`.
@@ -98,24 +98,23 @@ type ImmediateSuccessor func(dst, a []byte) []byte
 //
 // The returned prefix must have the following properties:
 //
-// 1) The prefix must be a byte prefix:
+//  1. The prefix must be a byte prefix:
 //
-//    bytes.HasPrefix(a, prefix(a))
+//     bytes.HasPrefix(a, prefix(a))
 //
-// 2) A key consisting of just a prefix must sort before all other keys with
-//    that prefix:
+//  2. A key consisting of just a prefix must sort before all other keys with
+//     that prefix:
 //
-//    Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
+//     Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
 //
-// 3) Prefixes must be used to order keys before suffixes:
+//  3. Prefixes must be used to order keys before suffixes:
 //
-//    If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
+//     If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
 //
-// 4) Suffixes themselves must be valid keys and comparable, respecting the same
-//    ordering as within a key.
-//
-//    If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
+//  4. Suffixes themselves must be valid keys and comparable, respecting the same
+//     ordering as within a key.
 //
+//     If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
 type Split func(a []byte) int
 
 // Comparer defines a total ordering over the space of []byte keys: a 'less

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -20,6 +20,31 @@ type FileNum uint64
 // String returns a string representation of the file number.
 func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
 
+// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
+// called if the caller can ensure that the FileNum belongs to a physical file
+// on disk. These could be manifests, log files, physical sstables on disk, the
+// options file, but not virtual sstables.
+func (fn FileNum) DiskFileNum() DiskFileNum {
+	return DiskFileNum{fn}
+}
+
+// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
+// Note that a FileNum is an internal DB identifier and it could belong to files
+// which don't exist on disk. An example would be virtual sstable FileNums.
+// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
+// FileNum to DiskFileNum may not be valid and care should be taken to prove
+// that the FileNum actually exists on disk.
+type DiskFileNum struct {
+	fn FileNum
+}
+
+func (dfn DiskFileNum) String() string { return dfn.fn.String() }
+
+// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
+func (dfn DiskFileNum) FileNum() FileNum {
+	return dfn.fn
+}
+
 // FileType enumerates the types of files found in a DB.
 type FileType int
 
@@ -36,92 +61,92 @@ const (
 )
 
 // MakeFilename builds a filename from components.
-func MakeFilename(fileType FileType, fileNum FileNum) string {
+func MakeFilename(fileType FileType, dfn DiskFileNum) string {
 	switch fileType {
 	case FileTypeLog:
-		return fmt.Sprintf("%s.log", fileNum)
+		return fmt.Sprintf("%s.log", dfn)
 	case FileTypeLock:
 		return "LOCK"
 	case FileTypeTable:
-		return fmt.Sprintf("%s.sst", fileNum)
+		return fmt.Sprintf("%s.sst", dfn)
 	case FileTypeManifest:
-		return fmt.Sprintf("MANIFEST-%s", fileNum)
+		return fmt.Sprintf("MANIFEST-%s", dfn)
 	case FileTypeCurrent:
 		return "CURRENT"
 	case FileTypeOptions:
-		return fmt.Sprintf("OPTIONS-%s", fileNum)
+		return fmt.Sprintf("OPTIONS-%s", dfn)
 	case FileTypeOldTemp:
-		return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum)
+		return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
 	case FileTypeTemp:
-		return fmt.Sprintf("temporary.%s.dbtmp", fileNum)
+		return fmt.Sprintf("temporary.%s.dbtmp", dfn)
 	}
 	panic("unreachable")
 }
 
 // MakeFilepath builds a filepath from components.
-func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string {
-	return fs.PathJoin(dirname, MakeFilename(fileType, fileNum))
+func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
+	return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
 }
 
 // ParseFilename parses the components from a filename.
-func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) {
+func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
 	filename = fs.PathBase(filename)
 	switch {
 	case filename == "CURRENT":
-		return FileTypeCurrent, 0, true
+		return FileTypeCurrent, DiskFileNum{0}, true
 	case filename == "LOCK":
-		return FileTypeLock, 0, true
+		return FileTypeLock, DiskFileNum{0}, true
 	case strings.HasPrefix(filename, "MANIFEST-"):
-		fileNum, ok = parseFileNum(filename[len("MANIFEST-"):])
+		dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
 		if !ok {
 			break
 		}
-		return FileTypeManifest, fileNum, true
+		return FileTypeManifest, dfn, true
 	case strings.HasPrefix(filename, "OPTIONS-"):
-		fileNum, ok = parseFileNum(filename[len("OPTIONS-"):])
+		dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
 		if !ok {
 			break
 		}
-		return FileTypeOptions, fileNum, ok
+		return FileTypeOptions, dfn, ok
 	case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
 		s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
-		fileNum, ok = parseFileNum(s)
+		dfn, ok = parseDiskFileNum(s)
 		if !ok {
 			break
 		}
-		return FileTypeOldTemp, fileNum, ok
+		return FileTypeOldTemp, dfn, ok
 	case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
 		s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
-		fileNum, ok = parseFileNum(s)
+		dfn, ok = parseDiskFileNum(s)
 		if !ok {
 			break
 		}
-		return FileTypeTemp, fileNum, ok
+		return FileTypeTemp, dfn, ok
 	default:
 		i := strings.IndexByte(filename, '.')
 		if i < 0 {
 			break
 		}
-		fileNum, ok = parseFileNum(filename[:i])
+		dfn, ok = parseDiskFileNum(filename[:i])
 		if !ok {
 			break
 		}
 		switch filename[i+1:] {
 		case "sst":
-			return FileTypeTable, fileNum, true
+			return FileTypeTable, dfn, true
 		case "log":
-			return FileTypeLog, fileNum, true
+			return FileTypeLog, dfn, true
 		}
 	}
-	return 0, fileNum, false
+	return 0, dfn, false
 }
 
-func parseFileNum(s string) (fileNum FileNum, ok bool) {
+func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
 	u, err := strconv.ParseUint(s, 10, 64)
 	if err != nil {
-		return fileNum, false
+		return dfn, false
 	}
-	return FileNum(u), true
+	return DiskFileNum{FileNum(u)}, true
 }
 
 // A Fataler fatals a process with a message when called.

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -60,6 +60,12 @@ const (
 	InternalKeyKindRangeKeyUnset InternalKeyKind = 20
 	InternalKeyKindRangeKeySet   InternalKeyKind = 21
 
+	// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
+	// the WAL entry for ingested sstables that are added to the flushable
+	// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
+	// batch, or in an sstable.
+	InternalKeyKindIngestSST InternalKeyKind = 22
+
 	// This maximum value isn't part of the file format. It's unlikely,
 	// but future extensions may increase this value.
 	//
@@ -69,7 +75,7 @@ const (
 	// which sorts 'less than or equal to' any other valid internalKeyKind, when
 	// searching for any kind of internal key formed by a certain user key and
 	// seqNum.
-	InternalKeyKindMax InternalKeyKind = 21
+	InternalKeyKindMax InternalKeyKind = 22
 
 	// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
 	// zero sequence number.
@@ -111,6 +117,7 @@ var internalKeyKindNames = []string{
 	InternalKeyKindRangeKeySet:    "RANGEKEYSET",
 	InternalKeyKindRangeKeyUnset:  "RANGEKEYUNSET",
 	InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
+	InternalKeyKindIngestSST:      "INGESTSST",
 	InternalKeyKindInvalid:        "INVALID",
 }
 
@@ -187,6 +194,7 @@ var kindsMap = map[string]InternalKeyKind{
 	"DEL":           InternalKeyKindDelete,
 	"SINGLEDEL":     InternalKeyKindSingleDelete,
 	"RANGEDEL":      InternalKeyKindRangeDelete,
+	"LOGDATA":       InternalKeyKindLogData,
 	"SET":           InternalKeyKindSet,
 	"MERGE":         InternalKeyKindMerge,
 	"INVALID":       InternalKeyKindInvalid,
@@ -195,6 +203,7 @@ var kindsMap = map[string]InternalKeyKind{
 	"RANGEKEYSET":   InternalKeyKindRangeKeySet,
 	"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
 	"RANGEKEYDEL":   InternalKeyKindRangeKeyDelete,
+	"INGESTSST":     InternalKeyKindIngestSST,
 }
 
 // ParseInternalKey parses the string representation of an internal key. The
@@ -362,9 +371,14 @@ func (k *InternalKey) SetKind(kind InternalKeyKind) {
 	k.Trailer = (k.Trailer &^ 0xff) | uint64(kind)
 }
 
-// Kind returns the kind compoment of the key.
+// Kind returns the kind component of the key.
 func (k InternalKey) Kind() InternalKeyKind {
-	return InternalKeyKind(k.Trailer & 0xff)
+	return TrailerKind(k.Trailer)
+}
+
+// TrailerKind returns the key kind of the key trailer.
+func TrailerKind(trailer uint64) InternalKeyKind {
+	return InternalKeyKind(trailer & 0xff)
 }
 
 // Valid returns true if the key has a valid kind.
@@ -383,6 +397,13 @@ func (k InternalKey) Clone() InternalKey {
 	}
 }
 
+// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
+// reusing any space already used for the current UserKey.
+func (k *InternalKey) CopyFrom(k2 InternalKey) {
+	k.UserKey = append(k.UserKey[:0], k2.UserKey...)
+	k.Trailer = k2.Trailer
+}
+
 // String returns a string representation of the key.
 func (k InternalKey) String() string {
 	return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())
@@ -413,7 +434,11 @@ type prettyInternalKey struct {
 }
 
 func (k prettyInternalKey) Format(s fmt.State, c rune) {
-	fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
+	if seqNum := k.SeqNum(); seqNum == InternalKeySeqNumMax {
+		fmt.Fprintf(s, "%s#inf,%s", k.formatKey(k.UserKey), k.Kind())
+	} else {
+		fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
+	}
 }
 
 // ParsePrettyInternalKey parses the pretty string representation of an
@@ -425,6 +450,11 @@ func ParsePrettyInternalKey(s string) InternalKey {
 	if !ok {
 		panic(fmt.Sprintf("unknown kind: %q", x[2]))
 	}
-	seqNum, _ := strconv.ParseUint(x[1], 10, 64)
+	var seqNum uint64
+	if x[1] == "max" || x[1] == "inf" {
+		seqNum = InternalKeySeqNumMax
+	} else {
+		seqNum, _ = strconv.ParseUint(x[1], 10, 64)
+	}
 	return MakeInternalKey([]byte(ukey), seqNum, kind)
 }

vendor/github.com/cockroachdb/pebble/internal/base/iterator.go

@@ -4,7 +4,10 @@
 
 package base
 
-import "fmt"
+import (
+	"fmt"
+	"time"
+)
 
 // InternalIterator iterates over a DB's key/value pairs in key order. Unlike
 // the Iterator interface, the returned keys are InternalKeys composed of the
@@ -40,7 +43,9 @@ import "fmt"
 // context is not a strict byte prefix, but defined by byte equality for the
 // result of the Comparer.Split method. An InternalIterator is not required to
 // support prefix iteration mode, and can implement SeekPrefixGE by forwarding
-// to SeekGE.
+// to SeekGE. When the iteration prefix is exhausted, it is not valid to call
+// Next on an internal iterator that's already returned (nil,nilv) or a key
+// beyond the prefix.
 //
 // Bounds, [lower, upper), can be set on iterators, either using the SetBounds()
 // function in the interface, or in implementation specific ways during iterator
@@ -152,6 +157,22 @@ type InternalIterator interface {
 	// SeekPrefixGE or Next returned (nil, nilv).
 	Next() (*InternalKey, LazyValue)
 
+	// NextPrefix moves the iterator to the next key/value pair with a different
+	// prefix than the key at the current iterator position. Returns the key and
+	// value if the iterator is pointing at a valid entry, and (nil, nil)
+	// otherwise. Note that NextPrefix only checks the upper bound. It is up to
+	// the caller to ensure that key is greater than or equal to the lower
+	// bound.
+	//
+	// NextPrefix is passed the immediate successor to the current prefix key. A
+	// valid implementation of NextPrefix is to call SeekGE with succKey.
+	//
+	// It is not allowed to call NextPrefix when the previous call was a reverse
+	// positioning operation or a call to a forward positioning method that
+	// returned (nil, nilv). It is also not allowed to call NextPrefix when the
+	// iterator is in prefix iteration mode.
+	NextPrefix(succKey []byte) (*InternalKey, LazyValue)
+
 	// Prev moves the iterator to the previous key/value pair. Returns the key
 	// and value if the iterator is pointing at a valid entry, and (nil, nilv)
 	// otherwise. Note that Prev only checks the lower bound. It is up to the
@@ -330,7 +351,11 @@ type InternalIteratorStats struct {
 	BlockBytes uint64
 	// Subset of BlockBytes that were in the block cache.
 	BlockBytesInCache uint64
-
+	// BlockReadDuration accumulates the duration spent fetching blocks
+	// due to block cache misses.
+	// TODO(sumeer): this currently excludes the time spent in Reader creation,
+	// and in reading the rangedel and rangekey blocks. Fix that.
+	BlockReadDuration time.Duration
 	// The following can repeatedly count the same points if they are iterated
 	// over multiple times. Additionally, they may count a point twice when
 	// switching directions. The latter could be improved if needed.
@@ -339,7 +364,7 @@ type InternalIteratorStats struct {
 	// included.
 	KeyBytes uint64
 	// Bytes in values that were iterated over. Currently, only point values are
-	// included.
+	// included. For separated values, this is the size of the handle.
 	ValueBytes uint64
 	// The count of points iterated over.
 	PointCount uint64
@@ -347,14 +372,38 @@ type InternalIteratorStats struct {
 	// can be useful for discovering instances of
 	// https://github.com/cockroachdb/pebble/issues/1070.
 	PointsCoveredByRangeTombstones uint64
+
+	// Stats related to points in value blocks encountered during iteration.
+	// These are useful to understand outliers, since typical user facing
+	// iteration should tend to only look at the latest point, and hence have
+	// the following stats close to 0.
+	SeparatedPointValue struct {
+		// Count is a count of points that were in value blocks. This is not a
+		// subset of PointCount: PointCount is produced by mergingIter and if
+		// positioned once, and successful in returning a point, will have a
+		// PointCount of 1, regardless of how many sstables (and memtables etc.)
+		// in the heap got positioned. The count here includes every sstable
+		// iterator that got positioned in the heap.
+		Count uint64
+		// ValueBytes represent the total byte length of the values (in value
+		// blocks) of the points corresponding to Count.
+		ValueBytes uint64
+		// ValueBytesFetched is the total byte length of the values (in value
+		// blocks) that were retrieved.
+		ValueBytesFetched uint64
+	}
 }
 
 // Merge merges the stats in from into the given stats.
 func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
 	s.BlockBytes += from.BlockBytes
 	s.BlockBytesInCache += from.BlockBytesInCache
+	s.BlockReadDuration += from.BlockReadDuration
 	s.KeyBytes += from.KeyBytes
 	s.ValueBytes += from.ValueBytes
 	s.PointCount += from.PointCount
 	s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones
+	s.SeparatedPointValue.Count += from.SeparatedPointValue.Count
+	s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
+	s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
 }

vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go

@@ -16,12 +16,8 @@ import "github.com/cockroachdb/pebble/internal/invariants"
 // it can call the ShortAttributeExtractor to extract the attribute and store
 // it together with the key. This allows for cheap retrieval of
 // AttributeAndLen on the read-path, without doing a more expensive retrieval
-// of the value.
-//
-// In general, the extraction code may want to also look at the key to decide
-// how to treat the value. Our current needs can be satisfied by configuring a
-// keyspan for which the ShortAttributeExtractor should be used, so we do not
-// expose the key to the extractor.
+// of the value. In general, the extraction code may want to also look at the
+// key to decide how to treat the value, hence the key* parameters.
 //
 // Write path performance: The ShortAttributeExtractor func cannot be inlined,
 // so we will pay the cost of this function call. However, we will only pay
@@ -38,7 +34,8 @@ const MaxShortAttribute = 7
 
 // ShortAttributeExtractor is an extractor that given the value, will return
 // the ShortAttribute.
-type ShortAttributeExtractor func(value []byte) (ShortAttribute, error)
+type ShortAttributeExtractor func(
+	key []byte, keyPrefixLen int, value []byte) (ShortAttribute, error)
 
 // AttributeAndLen represents the pair of value length and the short
 // attribute.
@@ -70,24 +67,26 @@ type AttributeAndLen struct {
 // Memory management:
 // This is subtle, but important for performance.
 //
-// - A LazyValue returned by an InternalIterator or Iterator is unstable in
-//   that repositioning the iterator will invalidate the memory inside it. A
-//   caller wishing to maintain that LazyValue needs to call
-//   LazyValue.Clone(). Note that this does not fetch the value if it is not
-//   in-place. Additionally, Clone() *must* only be called if
-//   LazyValue.Value() has *not* been called, since (a) it only makes a
-//   promise about stability of LazyValue and not the underlying value, (b) if
-//   LazyValue.Value() has been called already, the caller has the value
-//   []byte slice so it is unnecessary to fiddle around with LazyValue.
+// A LazyValue returned by an InternalIterator or Iterator is unstable in that
+// repositioning the iterator will invalidate the memory inside it. A caller
+// wishing to maintain that LazyValue needs to call LazyValue.Clone(). Note
+// that this does not fetch the value if it is not in-place. Clone() should
+// ideally not be called if LazyValue.Value() has been called, since the
+// cloned LazyValue will forget the extracted/fetched value, and calling
+// Value() on this clone will cause the value to be extracted again. That is,
+// Clone() does not make any promise about the memory stability of the
+// underlying value.
 //
-// - A user of an iterator that calls LazyValue.Value() wants as much as
-//   possible for the returned value []byte to point to iterator owned memory.
-//   - [P1] The underlying iterator that owns that memory also needs a promise
+// A user of an iterator that calls LazyValue.Value() wants as much as
+// possible for the returned value []byte to point to iterator owned memory.
+//
+//  1. [P1] The underlying iterator that owns that memory also needs a promise
 //     from that user that at any time there is at most one value []byte slice
 //     that the caller is expecting it to maintain. Otherwise, the underlying
 //     iterator has to maintain multiple such []byte slices which results in
 //     more complicated and inefficient code.
-//   - [P2] The underlying iterator, in order to make the promise that it is
+//
+//  2. [P2] The underlying iterator, in order to make the promise that it is
 //     maintaining the one value []byte slice, also needs a way to know when
 //     it is relieved of that promise. One way it is relieved of that promise
 //     is by being told that it is being repositioned. Typically, the owner of
@@ -105,28 +104,31 @@ type AttributeAndLen struct {
 //     the caller. This will be true if the callee attempted to use buf and
 //     either successfully used it or allocated a new []byte slice.
 //
-//  To ground the above in reality, we consider three examples of callers of
-//  LazyValue.Value():
-//  - Iterator: it calls LazyValue.Value for its own use when merging values.
-//    When merging during reverse iteration, it may have cloned the LazyValue.
-//    In this case it calls LazyValue.Value() on the cloned value, merges it,
-//    and then calls LazyValue.Value() on the current iterator position and
-//    merges it. So it is honoring P1.
-//  - Iterator on behalf of Iterator clients: The Iterator.Value() method
-//    needs to call LazyValue.Value(). The client of Iterator is satisfying P1
-//    because of the inherent Iterator interface constraint, i.e., it is calling
-//    Iterator.Value() on the current Iterator position. It is possible that
-//    the Iterator has cloned this LazyValue (for the reverse iteration case),
-//    which the client is unaware of, so the underlying sstable iterator may
-//    not be able to satisfy P2. This is ok because Iterator will call
-//    LazyValue.Value with its (reusable) owned buffer.
-//  - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
-//    since during reverse iteration in order to find the highest version that
-//    satisfies a read it needs to clone the LazyValue, step back the iterator
-//    and then decide whether it needs the value from the previously cloned
-//    LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
-//    similar to the previous case in that it will call LazyValue.Value with
-//    its (reusable) owned buffer.
+// To ground the above in reality, we consider three examples of callers of
+// LazyValue.Value():
+//
+//   - Iterator: it calls LazyValue.Value for its own use when merging values.
+//     When merging during reverse iteration, it may have cloned the LazyValue.
+//     In this case it calls LazyValue.Value() on the cloned value, merges it,
+//     and then calls LazyValue.Value() on the current iterator position and
+//     merges it. So it is honoring P1.
+//
+//   - Iterator on behalf of Iterator clients: The Iterator.Value() method
+//     needs to call LazyValue.Value(). The client of Iterator is satisfying P1
+//     because of the inherent Iterator interface constraint, i.e., it is calling
+//     Iterator.Value() on the current Iterator position. It is possible that
+//     the Iterator has cloned this LazyValue (for the reverse iteration case),
+//     which the client is unaware of, so the underlying sstable iterator may
+//     not be able to satisfy P2. This is ok because Iterator will call
+//     LazyValue.Value with its (reusable) owned buffer.
+//
+//   - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
+//     since during reverse iteration in order to find the highest version that
+//     satisfies a read it needs to clone the LazyValue, step back the iterator
+//     and then decide whether it needs the value from the previously cloned
+//     LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
+//     similar to the previous case in that it will call LazyValue.Value with
+//     its (reusable) owned buffer.
 //
 // Corollary: callers that directly use InternalIterator can know that they
 // have done nothing to interfere with promise P2 can pass in a nil buf and be
@@ -154,9 +156,24 @@ type LazyValue struct {
 }
 
 // LazyFetcher supports fetching a lazy value.
+//
+// Fetcher and Attribute are to be initialized at creation time. The fields
+// are arranged to reduce the sizeof this struct.
 type LazyFetcher struct {
-	// ValueFetcher, given a handle, returns the value. It is acceptable to call
-	// the ValueFetcher as long as the DB is open. However, one should assume
+	// Fetcher, given a handle, returns the value.
+	Fetcher ValueFetcher
+	err     error
+	value   []byte
+	// Attribute includes the short attribute and value length.
+	Attribute   AttributeAndLen
+	fetched     bool
+	callerOwned bool
+}
+
+// ValueFetcher is an interface for fetching a value.
+type ValueFetcher interface {
+	// Fetch returns the value, given the handle. It is acceptable to call the
+	// ValueFetcher.Fetch as long as the DB is open. However, one should assume
 	// there is a fast-path when the iterator tree has not moved off the sstable
 	// iterator that initially provided this LazyValue. Hence, to utilize this
 	// fast-path the caller should try to decide whether it needs the value or
@@ -166,13 +183,8 @@ type LazyFetcher struct {
 	// If the fetcher attempted to use buf *and* len(buf) was insufficient, it
 	// will allocate a new slice for the value. In either case it will set
 	// callerOwned to true.
-	ValueFetcher func(handle []byte, buf []byte) (val []byte, callerOwned bool, err error)
-	// Attribute includes the short attribute and value length.
-	Attribute   AttributeAndLen
-	fetched     bool
-	value       []byte
-	callerOwned bool
-	err         error
+	Fetch(
+		handle []byte, valLen int32, buf []byte) (val []byte, callerOwned bool, err error)
 }
 
 // Value returns the underlying value.
@@ -197,7 +209,8 @@ func (lv *LazyValue) fetchValue(buf []byte) (val []byte, callerOwned bool, err e
 	f := lv.Fetcher
 	if !f.fetched {
 		f.fetched = true
-		f.value, f.callerOwned, f.err = lv.Fetcher.ValueFetcher(lv.ValueOrHandle, buf)
+		f.value, f.callerOwned, f.err = f.Fetcher.Fetch(
+			lv.ValueOrHandle, lv.Fetcher.Attribute.ValueLen, buf)
 	}
 	return f.value, f.callerOwned, f.err
 }
@@ -219,18 +232,46 @@ func (lv *LazyValue) Len() int {
 	return int(lv.Fetcher.Attribute.ValueLen)
 }
 
-// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
-// REQUIRES: LazyValue.Value() has not been called.
-// Ensure that this is only called before LazyValue.Value is called. We don't
-// actually care if it was in-place but we don't want confusing things about whether
-// we need to clone the value inside the fetcher.
-func (lv *LazyValue) Clone(buf []byte) (LazyValue, []byte) {
-	if invariants.Enabled && lv.Fetcher != nil && lv.Fetcher.fetched {
-		panic("value has already been fetched")
+// TryGetShortAttribute returns the ShortAttribute and a bool indicating
+// whether the ShortAttribute was populated.
+func (lv *LazyValue) TryGetShortAttribute() (ShortAttribute, bool) {
+	if lv.Fetcher == nil {
+		return 0, false
+	}
+	return lv.Fetcher.Attribute.ShortAttribute, true
+}
+
+// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
+// The fetcher parameter must be non-nil and may be over-written and used
+// inside the returned LazyValue -- this is needed to avoid an allocation.
+// Most callers have at most K cloned LazyValues, where K is hard-coded, so
+// they can have a pool of exactly K LazyFetcher structs they can reuse in
+// these calls. The alternative of allocating LazyFetchers from a sync.Pool is
+// not viable since we have no code trigger for returning to the pool
+// (LazyValues are simply GC'd).
+//
+// NB: It is highly preferable that LazyValue.Value() has not been called,
+// since the Clone will forget any previously extracted value, and a future
+// call to Value will cause it to be fetched again. We do this since we don't
+// want to reason about whether or not to clone an already extracted value
+// inside the Fetcher (we don't). Property P1 applies here too: if lv1.Value()
+// has been called, and then lv2 is created as a clone of lv1, then calling
+// lv2.Value() can invalidate any backing memory maintained inside the fetcher
+// for lv1 (even though these are the same values). We initially prohibited
+// calling LazyValue.Clone() if LazyValue.Value() has been called, but there
+// is at least one complex caller (pebbleMVCCScanner inside CockroachDB) where
+// it is not easy to prove this invariant.
+func (lv *LazyValue) Clone(buf []byte, fetcher *LazyFetcher) (LazyValue, []byte) {
+	var lvCopy LazyValue
+	if lv.Fetcher != nil {
+		*fetcher = LazyFetcher{
+			Fetcher:   lv.Fetcher.Fetcher,
+			Attribute: lv.Fetcher.Attribute,
+			// Not copying anything that has been extracted.
+		}
+		lvCopy.Fetcher = fetcher
 	}
-	// INVARIANT: LazyFetcher.value is nil, so there is nothing to copy there.
 	vLen := len(lv.ValueOrHandle)
-	lvCopy := LazyValue{Fetcher: lv.Fetcher}
 	if vLen == 0 {
 		return lvCopy, buf
 	}
@@ -244,13 +285,3 @@ func (lv *LazyValue) Clone(buf []byte) (LazyValue, []byte) {
 func MakeInPlaceValue(val []byte) LazyValue {
 	return LazyValue{ValueOrHandle: val}
 }
-
-// TODO(sumeer): test that callers are adhering to promise P1. In the sstable
-// iterators, when invariants are enabled, create a testingFetcher struct that
-// contains the sstable name and the []byte backing for the last fetched
-// value. The sstable iterator, instead of returning an in-place value, will
-// return a copy of the key in ValueOrHandle, and a ValueFetcher implemented
-// by testingFetcher.valueFetcher. Each time this method is called, it will
-// mangle the last fetched value []byte slice, allocate a new one, open the
-// sstable, seek to the key, and copy the value into this new byte slice and
-// return.

vendor/github.com/cockroachdb/pebble/internal/base/logger.go

@@ -0,0 +1,144 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"sync"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// Logger defines an interface for writing log messages.
+type Logger interface {
+	Infof(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+}
+type defaultLogger struct{}
+
+// DefaultLogger logs to the Go stdlib logs.
+var DefaultLogger defaultLogger
+
+var _ Logger = DefaultLogger
+
+// Infof implements the Logger.Infof interface.
+func (defaultLogger) Infof(format string, args ...interface{}) {
+	_ = log.Output(2, fmt.Sprintf(format, args...))
+}
+
+// Fatalf implements the Logger.Fatalf interface.
+func (defaultLogger) Fatalf(format string, args ...interface{}) {
+	_ = log.Output(2, fmt.Sprintf(format, args...))
+	os.Exit(1)
+}
+
+// InMemLogger implements Logger using an in-memory buffer (used for testing).
+// The buffer can be read via String() and cleared via Reset().
+type InMemLogger struct {
+	mu struct {
+		sync.Mutex
+		buf bytes.Buffer
+	}
+}
+
+var _ Logger = (*InMemLogger)(nil)
+
+// Reset clears the internal buffer.
+func (b *InMemLogger) Reset() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.mu.buf.Reset()
+}
+
+// String returns the current internal buffer.
+func (b *InMemLogger) String() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.mu.buf.String()
+}
+
+// Infof is part of the Logger interface.
+func (b *InMemLogger) Infof(format string, args ...interface{}) {
+	s := fmt.Sprintf(format, args...)
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.mu.buf.Write([]byte(s))
+	if n := len(s); n == 0 || s[n-1] != '\n' {
+		b.mu.buf.Write([]byte("\n"))
+	}
+}
+
+// Fatalf is part of the Logger interface.
+func (b *InMemLogger) Fatalf(format string, args ...interface{}) {
+	b.Infof(format, args...)
+	runtime.Goexit()
+}
+
+// LoggerAndTracer defines an interface for logging and tracing.
+type LoggerAndTracer interface {
+	Logger
+	// Eventf formats and emits a tracing log, if tracing is enabled in the
+	// current context.
+	Eventf(ctx context.Context, format string, args ...interface{})
+	// IsTracingEnabled returns true if tracing is enabled. It can be used as an
+	// optimization to avoid calling Eventf (which will be a noop when tracing
+	// is not enabled) to avoid the overhead of boxing the args.
+	IsTracingEnabled(ctx context.Context) bool
+}
+
+// LoggerWithNoopTracer wraps a logger and does no tracing.
+type LoggerWithNoopTracer struct {
+	Logger
+}
+
+var _ LoggerAndTracer = &LoggerWithNoopTracer{}
+
+// Eventf implements LoggerAndTracer.
+func (*LoggerWithNoopTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
+	if invariants.Enabled && ctx == nil {
+		panic("Eventf context is nil")
+	}
+}
+
+// IsTracingEnabled implements LoggerAndTracer.
+func (*LoggerWithNoopTracer) IsTracingEnabled(ctx context.Context) bool {
+	if invariants.Enabled && ctx == nil {
+		panic("IsTracingEnabled ctx is nil")
+	}
+	return false
+}
+
+// NoopLoggerAndTracer does no logging and tracing. Remember that struct{} is
+// special cased in Go and does not incur an allocation when it backs the
+// interface LoggerAndTracer.
+type NoopLoggerAndTracer struct{}
+
+var _ LoggerAndTracer = NoopLoggerAndTracer{}
+
+// Infof implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) Infof(format string, args ...interface{}) {}
+
+// Fatalf implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) Fatalf(format string, args ...interface{}) {}
+
+// Eventf implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
+	if invariants.Enabled && ctx == nil {
+		panic("Eventf context is nil")
+	}
+}
+
+// IsTracingEnabled implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) IsTracingEnabled(ctx context.Context) bool {
+	if invariants.Enabled && ctx == nil {
+		panic("IsTracingEnabled ctx is nil")
+	}
+	return false
+}

vendor/github.com/cockroachdb/pebble/internal/base/merger.go

@@ -23,7 +23,7 @@ type Merge func(key, value []byte) (ValueMerger, error)
 //
 // The merge operation must be associative. That is, for the values A, B, C:
 //
-//   Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
+//	Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
 //
 // Examples of merge operators are integer addition, list append, and string
 // concatenation.

vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go

@@ -0,0 +1,54 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+// This file defines sequence numbers that are reserved for foreign keys i.e.
+// internal keys coming from other Pebble instances and existing in shared
+// storage, as those will "slot below" any internal keys added by our own Pebble
+// instance. Any keys created by this Pebble instance need to be greater than
+// all reserved sequence numbers (i.e. >= SeqNumStart).
+const (
+	// SeqNumZero is the zero sequence number, set by compactions if they can
+	// guarantee there are no keys underneath an internal key.
+	SeqNumZero = uint64(0)
+
+	// SeqNumL6Point is the sequence number reserved for foreign point keys in L6.
+	// This sequence number must be lower than SeqNumL6RangeKey for range key
+	// masking to work correctly.
+	SeqNumL6Point = uint64(1)
+
+	// SeqNumL6RangeKey is the sequence number reserved for foreign range keys in
+	// L6. Only RangeKeySets are expected at this level.
+	SeqNumL6RangeKey = uint64(2)
+
+	// SeqNumL5RangeDel is the sequence number reserved for foreign range deletes
+	// in L5. These keys could delete L6 points.
+	SeqNumL5RangeDel = uint64(3)
+
+	// SeqNumL5Point is the sequence number reserved for foreign point keys in L5.
+	// Any sst-local range deletions would have already been applied to these keys,
+	// so they can safely get a sequence number higher than SeqNumL5RangeDel.
+	// However they must have a sequence number lower than SeqNumL5RangeKey* for
+	// range key masking to work correctly.
+	SeqNumL5Point = uint64(4)
+
+	// SeqNumL5RangeKeyUnsetDel is the sequence number reserved for foreign
+	// range key unsets/deletes in L5. These operations could apply to L6
+	// RangeKeySets, so this sequence number must be > SeqNumL6RangeKey.
+	SeqNumL5RangeKeyUnsetDel = uint64(5)
+
+	// SeqNumL5RangeKeySet is the sequence number reserved for foreign range key
+	// Sets in L5. These operations could apply to L6 RangeKeySets, so this
+	// sequence number must be > SeqNumL6RangeKey. Any SST-local rangekey
+	// unsets/dels have already been applied to them, so their sequence number must
+	// be > SeqNumL5RangeKeyUnsetDel.
+	SeqNumL5RangeKeySet = uint64(6)
+
+	// Sequence numbers 7-9 are reserved for future use.
+
+	// SeqNumStart is the first sequence number assigned to a key written by
+	// ourselves.
+	SeqNumStart = uint64(10)
+)

vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go

@@ -26,7 +26,7 @@ import "github.com/cockroachdb/pebble/internal/rawalloc"
 type A []byte
 
 const chunkAllocMinSize = 512
-const chunkAllocMaxSize = 16384
+const chunkAllocMaxSize = 512 << 10 // 512 KB
 
 func (a A) reserve(n int) A {
 	allocSize := cap(a) * 2
@@ -59,3 +59,11 @@ func (a A) Copy(src []byte) (A, []byte) {
 	copy(alloc, src)
 	return a, alloc
 }
+
+// Reset returns the current chunk, resetting allocated memory back to none.
+// Future allocations will use memory previously allocated by previous calls to
+// Alloc or Copy, so the caller must know know that none of the previously
+// allocated byte slices are still in use.
+func (a A) Reset() A {
+	return a[:0]
+}

vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go

@@ -33,7 +33,7 @@ import (
 type fileKey struct {
 	// id is the namespace for fileNums.
 	id      uint64
-	fileNum base.FileNum
+	fileNum base.DiskFileNum
 }
 
 type key struct {
@@ -113,7 +113,7 @@ type shard struct {
 	countTest int64
 }
 
-func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
+func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
 	c.mu.RLock()
 	var value *Value
 	if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
@@ -131,7 +131,7 @@ func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
 	return Handle{value: value}
 }
 
-func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
+func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
 	if n := value.refs(); n != 1 {
 		panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
 	}
@@ -222,7 +222,7 @@ func (c *shard) checkConsistency() {
 }
 
 // Delete deletes the cached value for the specified file and offset.
-func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
+func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
 	// The common case is there is nothing to delete, so do a quick check with
 	// shared lock.
 	k := key{fileKey{id, fileNum}, offset}
@@ -246,7 +246,7 @@ func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
 }
 
 // EvictFile evicts all of the cache values for the specified file.
-func (c *shard) EvictFile(id uint64, fileNum base.FileNum) {
+func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -603,7 +603,7 @@ type Metrics struct {
 // efficient eviction of all of the blocks for a file which is used when an
 // sstable is deleted from disk.
 //
-// Memory Management
+// # Memory Management
 //
 // In order to reduce pressure on the Go GC, manual memory management is
 // performed for the data stored in the cache. Manual memory management is
@@ -639,9 +639,9 @@ type Cache struct {
 // creator of the cache should usually release their reference after the DB is
 // created.
 //
-//   c := cache.New(...)
-//   defer c.Unref()
-//   d, err := pebble.Open(pebble.Options{Cache: c})
+//	c := cache.New(...)
+//	defer c.Unref()
+//	d, err := pebble.Open(pebble.Options{Cache: c})
 func New(size int64) *Cache {
 	return newShards(size, 2*runtime.GOMAXPROCS(0))
 }
@@ -683,7 +683,7 @@ func newShards(size int64, shards int) *Cache {
 	return c
 }
 
-func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard {
+func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
 	if id == 0 {
 		panic("pebble: 0 cache ID is invalid")
 	}
@@ -698,10 +698,11 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
 		h ^= uint64(id & 0xff)
 		id >>= 8
 	}
+	fileNumVal := uint64(fileNum.FileNum())
 	for i := 0; i < 8; i++ {
 		h *= prime64
-		h ^= uint64(fileNum & 0xff)
-		fileNum >>= 8
+		h ^= uint64(fileNumVal) & 0xff
+		fileNumVal >>= 8
 	}
 	for i := 0; i < 8; i++ {
 		h *= prime64
@@ -738,7 +739,7 @@ func (c *Cache) Unref() {
 
 // Get retrieves the cache value for the specified file and offset, returning
 // nil if no value is present.
-func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
+func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
 	return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
 }
 
@@ -746,17 +747,17 @@ func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
 // existing value if present. A Handle is returned which provides faster
 // retrieval of the cached value than Get (lock-free and avoidance of the map
 // lookup). The value must have been allocated by Cache.Alloc.
-func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
+func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
 	return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
 }
 
 // Delete deletes the cached value for the specified file and offset.
-func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) {
+func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
 	c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
 }
 
 // EvictFile evicts all of the cache values for the specified file.
-func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) {
+func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
 	if id == 0 {
 		panic("pebble: 0 cache ID is invalid")
 	}

vendor/github.com/cockroachdb/pebble/internal/cache/entry.go

@@ -30,7 +30,7 @@ func (p entryType) String() string {
 // Using manual memory management for entries is technically a volation of the
 // Cgo pointer rules:
 //
-//   https://golang.org/cmd/cgo/#hdr-Passing_pointers
+//	https://golang.org/cmd/cgo/#hdr-Passing_pointers
 //
 // Specifically, Go pointers should not be stored in C allocated memory. The
 // reason for this rule is that the Go GC will not look at C allocated memory

vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go

@@ -24,7 +24,7 @@ func robinHoodHash(k key, shift uint32) uint32 {
 	const m = 11400714819323198485
 	h := hashSeed
 	h ^= k.id * m
-	h ^= uint64(k.fileNum) * m
+	h ^= uint64(k.fileNum.FileNum()) * m
 	h ^= k.offset * m
 	return uint32(h >> shift)
 }
@@ -95,11 +95,11 @@ func (e robinHoodEntries) free() {
 // table is 2, maxDist is computed as 4 and the actual size of the entry slice
 // is 6.
 //
-//   +---+---+---+---+---+---+
-//   | 0 | 1 | 2 | 3 | 4 | 5 |
-//   +---+---+---+---+---+---+
-//           ^
-//          size
+//	+---+---+---+---+---+---+
+//	| 0 | 1 | 2 | 3 | 4 | 5 |
+//	+---+---+---+---+---+---+
+//	        ^
+//	       size
 //
 // In this scenario, the target entry for a key will always be in the range
 // [0,1]. Valid entries may reside in the range [0,4] due to the linear probing

vendor/github.com/cockroachdb/pebble/internal/crc/crc.go

@@ -10,7 +10,9 @@
 // look like a checksum.
 //
 // To calculate the uint32 checksum of some data:
+//
 //	var u uint32 = crc.New(data).Value()
+//
 // In pebble, the uint32 value is then stored in little-endian format.
 package crc // import "github.com/cockroachdb/pebble/internal/crc"
 

vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go

@@ -7,9 +7,11 @@ package fastrand
 import _ "unsafe" // required by go:linkname
 
 // Uint32 returns a lock free uint32 value.
+//
 //go:linkname Uint32 runtime.fastrand
 func Uint32() uint32
 
 // Uint32n returns a lock free uint32 value in the interval [0, n).
+//
 //go:linkname Uint32n runtime.fastrandn
 func Uint32n(n uint32) uint32

vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go

@@ -66,6 +66,9 @@ type BoundedIter struct {
 // indicating whether the iterator is in prefix iteration mode and the prefix
 // key if it is. This is used to exclude spans that are outside the iteration
 // prefix.
+//
+// hasPrefix and prefix are allowed to be nil, however if hasPrefix != nil,
+// prefix must also not be nil.
 func (i *BoundedIter) Init(
 	cmp base.Compare,
 	split base.Split,
@@ -146,7 +149,7 @@ func (i *BoundedIter) Next() *Span {
 		}
 		// Similarly, if the span extends to the next prefix and we're in prefix
 		// iteration mode, we can avoid advancing.
-		if i.iterSpan != nil && *i.hasPrefix {
+		if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
 			ei := i.split(i.iterSpan.End)
 			if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 {
 				i.pos = posAtUpperLimit
@@ -178,7 +181,7 @@ func (i *BoundedIter) Prev() *Span {
 		}
 		// Similarly, if the span extends to or beyond the current prefix and
 		// we're in prefix iteration mode, we can avoid advancing.
-		if i.iterSpan != nil && *i.hasPrefix {
+		if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
 			si := i.split(i.iterSpan.Start)
 			if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 {
 				i.pos = posAtLowerLimit
@@ -215,7 +218,7 @@ func (i *BoundedIter) SetBounds(lower, upper []byte) {
 
 func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span {
 	// Compare to the prefix's bounds, if in prefix iteration mode.
-	if span != nil && *i.hasPrefix {
+	if span != nil && i.hasPrefix != nil && *i.hasPrefix {
 		si := i.split(span.Start)
 		if i.cmp(span.Start[:si], *i.prefix) > 0 {
 			// This span starts at a prefix that sorts after our current prefix.
@@ -242,7 +245,7 @@ func (i *BoundedIter) checkForwardBound(span *Span) *Span {
 
 func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span {
 	// Compare to the prefix's bounds, if in prefix iteration mode.
-	if span != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
+	if span != nil && i.hasPrefix != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
 		// This span ends before the current prefix.
 		span = nil
 	}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -8,14 +8,20 @@ import (
 	"bytes"
 
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/bytealloc"
 	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
-// that DefragmentingIter will reuse. Buffers greater than this will be
+// that DefragmentingIter will reuse. Buffers larger than this will be
 // discarded and reallocated as necessary.
 const bufferReuseMaxCapacity = 10 << 10 // 10 KB
 
+// keysReuseMaxCapacity is the maximum capacity of a []keyspan.Key buffer that
+// DefragmentingIter will reuse. Buffers larger than this will be discarded and
+// reallocated as necessary.
+const keysReuseMaxCapacity = 100
+
 // DefragmentMethod configures the defragmentation performed by the
 // DefragmentingIter.
 type DefragmentMethod interface {
@@ -113,30 +119,21 @@ const (
 //
 // Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may
 // land on a physical fragment in the middle of several fragments that must be
-// defragmented. A seek first degfragments in the opposite direction of
-// iteration to find the beginning of the defragmented span, and then
-// defragments in the iteration direction, ensuring it's found a whole
-// defragmented span.
+// defragmented. A seek that lands in a fragment straddling the seek key must
+// first degfragment in the opposite direction of iteration to find the
+// beginning of the defragmented span, and then defragments in the iteration
+// direction, ensuring it's found a whole defragmented span.
 type DefragmentingIter struct {
+	// DefragmentingBuffers holds buffers used for copying iterator state.
+	*DefragmentingBuffers
 	comparer *base.Comparer
 	equal    base.Equal
 	iter     FragmentIterator
 	iterSpan *Span
 	iterPos  iterPos
 
-	// curr holds the span at the current iterator position. currBuf is a buffer
-	// for use when copying user keys for curr. keysBuf is a buffer for use when
-	// copying Keys for curr. currBuf is cleared between positioning methods.
-	//
-	// keyBuf is a buffer specifically for the defragmented start key when
-	// defragmenting backwards or the defragmented end key when defragmenting
-	// forwards. These bounds are overwritten repeatedly during defragmentation,
-	// and the defragmentation routines overwrite keyBuf repeatedly to store
-	// these extended bounds.
-	curr    Span
-	currBuf []byte
-	keysBuf []Key
-	keyBuf  []byte
+	// curr holds the span at the current iterator position.
+	curr Span
 
 	// method is a comparison function for two spans. method is called when two
 	// spans are abutting to determine whether they may be defragmented.
@@ -148,20 +145,53 @@ type DefragmentingIter struct {
 	reduce DefragmentReducer
 }
 
+// DefragmentingBuffers holds buffers used for copying iterator state.
+type DefragmentingBuffers struct {
+	// currBuf is a buffer for use when copying user keys for curr. currBuf is
+	// cleared between positioning methods.
+	currBuf bytealloc.A
+	// keysBuf is a buffer for use when copying Keys for DefragmentingIter.curr.
+	keysBuf []Key
+	// keyBuf is a buffer specifically for the defragmented start key when
+	// defragmenting backwards or the defragmented end key when defragmenting
+	// forwards. These bounds are overwritten repeatedly during defragmentation,
+	// and the defragmentation routines overwrite keyBuf repeatedly to store
+	// these extended bounds.
+	keyBuf []byte
+}
+
+// PrepareForReuse discards any excessively large buffers.
+func (bufs *DefragmentingBuffers) PrepareForReuse() {
+	if cap(bufs.currBuf) > bufferReuseMaxCapacity {
+		bufs.currBuf = nil
+	}
+	if cap(bufs.keyBuf) > bufferReuseMaxCapacity {
+		bufs.keyBuf = nil
+	}
+	if cap(bufs.keysBuf) > keysReuseMaxCapacity {
+		bufs.keysBuf = nil
+	}
+}
+
 // Assert that *DefragmentingIter implements the FragmentIterator interface.
 var _ FragmentIterator = (*DefragmentingIter)(nil)
 
 // Init initializes the defragmenting iter using the provided defragment
 // method.
 func (i *DefragmentingIter) Init(
-	comparer *base.Comparer, iter FragmentIterator, equal DefragmentMethod, reducer DefragmentReducer,
+	comparer *base.Comparer,
+	iter FragmentIterator,
+	equal DefragmentMethod,
+	reducer DefragmentReducer,
+	bufs *DefragmentingBuffers,
 ) {
 	*i = DefragmentingIter{
-		comparer: comparer,
-		equal:    comparer.Equal,
-		iter:     iter,
-		method:   equal,
-		reduce:   reducer,
+		DefragmentingBuffers: bufs,
+		comparer:             comparer,
+		equal:                comparer.Equal,
+		iter:                 iter,
+		method:               equal,
+		reduce:               reducer,
 	}
 }
 
@@ -175,8 +205,9 @@ func (i *DefragmentingIter) Close() error {
 	return i.iter.Close()
 }
 
-// SeekGE seeks the iterator to the first span with a start key greater than or
-// equal to key and returns it.
+// SeekGE moves the iterator to the first span covering a key greater than or
+// equal to the given key. This is equivalent to seeking to the first span with
+// an end key greater than the given key.
 func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 	i.iterSpan = i.iter.SeekGE(key)
 	if i.iterSpan == nil {
@@ -186,30 +217,28 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 		i.iterPos = iterPosCurr
 		return i.iterSpan
 	}
-	// Save the current span and peek backwards.
-	i.saveCurrent()
-	i.iterSpan = i.iter.Prev()
-	if i.iterSpan != nil && i.equal(i.curr.Start, i.iterSpan.End) && i.checkEqual(i.iterSpan, &i.curr) {
-		// A continuation. The span we originally landed on and defragmented
-		// backwards has a true Start key < key. To obey the FragmentIterator
-		// contract, we must not return this defragmented span. Defragment
-		// forward to finish defragmenting the span in the forward direction.
-		i.defragmentForward()
-
-		// Now we must be on a span that truly has a defragmented Start key >
-		// key.
+	// If the span starts strictly after key, we know there mustn't be an
+	// earlier span that ends at i.iterSpan.Start, otherwise i.iter would've
+	// returned that span instead.
+	if i.comparer.Compare(i.iterSpan.Start, key) > 0 {
 		return i.defragmentForward()
 	}
 
-	// The span previous to i.curr does not defragment, so we should return it.
-	// Next the underlying iterator back onto the span we previously saved to
-	// i.curr and then defragment forward.
-	i.iterSpan = i.iter.Next()
+	// The span we landed on has a Start bound ≤ key. There may be additional
+	// fragments before this span. Defragment backward to find the start of the
+	// defragmented span.
+	i.defragmentBackward()
+	if i.iterPos == iterPosPrev {
+		// Next once back onto the span.
+		i.iterSpan = i.iter.Next()
+	}
+	// Defragment the full span from its start.
 	return i.defragmentForward()
 }
 
-// SeekLT seeks the iterator to the last span with a start key less than
-// key and returns it.
+// SeekLT moves the iterator to the last span covering a key less than the
+// given key. This is equivalent to seeking to the last span with a start
+// key less than the given key.
 func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 	i.iterSpan = i.iter.SeekLT(key)
 	if i.iterSpan == nil {
@@ -219,7 +248,16 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 		i.iterPos = iterPosCurr
 		return i.iterSpan
 	}
-	// Defragment forward to find the end of the defragmented span.
+	// If the span ends strictly before key, we know there mustn't be a later
+	// span that starts at i.iterSpan.End, otherwise i.iter would've returned
+	// that span instead.
+	if i.comparer.Compare(i.iterSpan.End, key) < 0 {
+		return i.defragmentBackward()
+	}
+
+	// The span we landed on has a End bound ≥ key. There may be additional
+	// fragments after this span. Defragment forward to find the end of the
+	// defragmented span.
 	i.defragmentForward()
 	if i.iterPos == iterPosNext {
 		// Prev once back onto the span.
@@ -437,15 +475,9 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
 }
 
 func (i *DefragmentingIter) saveCurrent() {
-	i.currBuf = i.currBuf[:0]
+	i.currBuf.Reset()
 	i.keysBuf = i.keysBuf[:0]
 	i.keyBuf = i.keyBuf[:0]
-	if cap(i.currBuf) > bufferReuseMaxCapacity {
-		i.currBuf = nil
-	}
-	if cap(i.keyBuf) > bufferReuseMaxCapacity {
-		i.keyBuf = nil
-	}
 	if i.iterSpan == nil {
 		return
 	}
@@ -468,7 +500,6 @@ func (i *DefragmentingIter) saveBytes(b []byte) []byte {
 	if b == nil {
 		return nil
 	}
-	ret := append(i.currBuf, b...)
-	i.currBuf = ret[len(ret):]
-	return ret
+	i.currBuf, b = i.currBuf.Copy(b)
+	return b
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go

@@ -108,20 +108,20 @@ func (f *Fragmenter) checkInvariants(buf []Span) {
 // increasing start key order. That is, Add must be called with a series
 // of spans like:
 //
-//   a---e
-//     c---g
-//     c-----i
-//            j---n
-//            j-l
+//	a---e
+//	  c---g
+//	  c-----i
+//	         j---n
+//	         j-l
 //
 // We need to fragment the spans at overlap points. In the above
 // example, we'd create:
 //
-//   a-c-e
-//     c-e-g
-//     c-e-g-i
-//            j-l-n
-//            j-l
+//	a-c-e
+//	  c-e-g
+//	  c-e-g-i
+//	         j-l-n
+//	         j-l
 //
 // The fragments need to be output sorted by start key, and for equal start
 // keys, sorted by descending sequence number. This last part requires a mild
@@ -134,42 +134,42 @@ func (f *Fragmenter) checkInvariants(buf []Span) {
 //
 // Walking through the example above, we start with:
 //
-//   a---e
+//	a---e
 //
 // Next we add [c,g) resulting in:
 //
-//   a-c-e
-//     c---g
+//	a-c-e
+//	  c---g
 //
 // The fragment [a,c) is flushed leaving the pending spans as:
 //
-//   c-e
-//   c---g
+//	c-e
+//	c---g
 //
 // The next span is [c,i):
 //
-//   c-e
-//   c---g
-//   c-----i
+//	c-e
+//	c---g
+//	c-----i
 //
 // No fragments are flushed. The next span is [j,n):
 //
-//   c-e
-//   c---g
-//   c-----i
-//          j---n
+//	c-e
+//	c---g
+//	c-----i
+//	       j---n
 //
 // The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments
 // by their end key, then split the fragments on the end keys:
 //
-//   c-e
-//   c-e-g
-//   c-e---i
+//	c-e
+//	c-e-g
+//	c-e---i
 //
 // The [c,e) fragments all get flushed leaving:
 //
-//   e-g
-//   e---i
+//	e-g
+//	e---i
 //
 // This process continues until there are no more fragments to flush.
 //
@@ -229,16 +229,35 @@ func (f *Fragmenter) Add(s Span) {
 	f.pending = append(f.pending, s)
 }
 
-// Covers returns true if the specified key is covered by one of the pending
-// spans. The key must be consistent with the ordering of the
-// spans. That is, it is invalid to specify a key here that is out of
-// order with the span start keys passed to Add.
-func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
+// Cover is returned by Framenter.Covers and describes a span's relationship to
+// a key at a particular snapshot.
+type Cover int8
+
+const (
+	// NoCover indicates the tested key does not fall within the span's bounds,
+	// or the span contains no keys with sequence numbers higher than the key's.
+	NoCover Cover = iota
+	// CoversInvisibly indicates the tested key does fall within the span's
+	// bounds and the span contains at least one key with a higher sequence
+	// number, but none visible at the provided snapshot.
+	CoversInvisibly
+	// CoversVisibly indicates the tested key does fall within the span's
+	// bounds, and the span constains at least one key with a sequence number
+	// higher than the key's sequence number that is visible at the provided
+	// snapshot.
+	CoversVisibly
+)
+
+// Covers returns an enum indicating whether the specified key is covered by one
+// of the pending keys. The provided key must be consistent with the ordering of
+// the spans. That is, it is invalid to specify a key here that is out of order
+// with the span start keys passed to Add.
+func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) Cover {
 	if f.finished {
 		panic("pebble: span fragmenter already finished")
 	}
 	if len(f.pending) == 0 {
-		return false
+		return NoCover
 	}
 
 	if f.Cmp(f.pending[0].Start, key.UserKey) > 0 {
@@ -246,18 +265,31 @@ func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
 			f.Format(f.pending[0].Start), key.Pretty(f.Format)))
 	}
 
+	cover := NoCover
 	seqNum := key.SeqNum()
 	for _, s := range f.pending {
 		if f.Cmp(key.UserKey, s.End) < 0 {
 			// NB: A range deletion tombstone does not delete a point operation
 			// at the same sequence number, and broadly a span is not considered
 			// to cover a point operation at the same sequence number.
-			if s.CoversAt(snapshot, seqNum) {
-				return true
+
+			for i := range s.Keys {
+				if kseq := s.Keys[i].SeqNum(); kseq > seqNum {
+					// This key from the span has a higher sequence number than
+					// `key`. It covers `key`, although the span's key might not
+					// be visible if its snapshot is too high.
+					//
+					// Batch keys are always be visible.
+					if kseq < snapshot || kseq&base.InternalKeySeqNumBatch != 0 {
+						return CoversVisibly
+					}
+					// s.Keys[i] is not visible.
+					cover = CoversInvisibly
+				}
 			}
 		}
 	}
-	return false
+	return cover
 }
 
 // Empty returns true if all fragments added so far have finished flushing.
@@ -270,20 +302,20 @@ func (f *Fragmenter) Empty() bool {
 // emitting of spans which straddle an sstable boundary. Consider
 // the scenario:
 //
-//     a---------k#10
-//          f#8
-//          f#7
-///
+//	a---------k#10
+//	     f#8
+//	     f#7
+//
 // Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will
 // flush this span:
 //
-//    a-------g#10
-//         f#8
-//         f#7
+//	a-------g#10
+//	     f#8
+//	     f#7
 //
 // And leave this one in f.pending:
 //
-//            g----k#10
+//	g----k#10
 //
 // WARNING: The fragmenter could hold on to the specified end key. Ensure it's
 // a safe byte slice that could outlast the current sstable output, and one
@@ -374,8 +406,8 @@ func (f *Fragmenter) truncateAndFlush(key []byte) {
 // we want to flush (but not truncate) all range tombstones that start at or
 // before the first key in the next sstable. Consider:
 //
-//   a---e#10
-//   a------h#9
+//	a---e#10
+//	a------h#9
 //
 // If a compaction splits the sstables at key c we want the first sstable to
 // contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally

vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go

@@ -29,9 +29,9 @@ type SpanMask interface {
 	SpanChanged(*Span)
 	// SkipPoint is invoked by the interleaving iterator whenever the iterator
 	// encounters a point key covered by a Span. If SkipPoint returns true, the
-	// interleaving iterator skips the point key without returning it. This is
-	// used during range key iteration to skip over point keys 'masked' by range
-	// keys.
+	// interleaving iterator skips the point key and all larger keys with the
+	// same prefix. This is used during range key iteration to skip over point
+	// keys 'masked' by range keys.
 	SkipPoint(userKey []byte) bool
 }
 
@@ -79,13 +79,13 @@ type SpanMask interface {
 // SeekGE or SeekPrefixGE search key. Consider, for example SetBounds('c', 'e'),
 // with an iterator containing the Span [a,z):
 //
-//     First()     = `c#72057594037927935,21`        Span() = [c,e)
-//     SeekGE('d') = `d#72057594037927935,21`        Span() = [c,e)
+//	First()     = `c#72057594037927935,21`        Span() = [c,e)
+//	SeekGE('d') = `d#72057594037927935,21`        Span() = [c,e)
 //
 // InterleavedIter does not interleave synthetic markers for spans that do not
 // contain any keys.
 //
-// SpanMask
+// # SpanMask
 //
 // InterelavingIter takes a SpanMask parameter that may be used to configure the
 // behavior of the iterator. See the documentation on the SpanMask type.
@@ -479,6 +479,30 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) {
 	return i.interleaveForward(i.lower, nil /* prefix */)
 }
 
+// NextPrefix implements (base.InternalIterator).NextPrefix.
+func (i *InterleavingIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	if i.dir == -1 {
+		panic("pebble: cannot switch directions with NextPrefix")
+	}
+
+	// Refresh the point key if the current point key has already been
+	// interleaved, OR if the next uninterleaved point key has the old suffix
+	// (it's less than succKey).
+	if i.pointKeyInterleaved || (i.pointKey != nil && i.cmp(i.pointKey.UserKey, succKey) < 0) {
+		i.pointKey, i.pointVal = i.pointIter.NextPrefix(succKey)
+		i.pointKeyInterleaved = false
+	}
+	// If we already interleaved the current span start key, and the point key
+	// is ≥ the span's end key, move to the next span.
+	if i.keyspanInterleaved && i.pointKey != nil && i.span != nil &&
+		i.cmp(i.pointKey.UserKey, i.span.End) >= 0 {
+		i.span = i.keyspanIter.Next()
+		i.checkForwardBound(nil)
+		i.savedKeyspan()
+	}
+	return i.interleaveForward(i.lower, nil)
+}
+
 // Prev implements (base.InternalIterator).Prev.
 func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
 	if i.dir == +1 {
@@ -677,9 +701,29 @@ func (i *InterleavingIter) interleaveForward(
 				// if we have stepped outside of the span last saved as a mask,
 				// so that the decision to skip is made with the correct
 				// knowledge of the covering span.
-				i.maybeUpdateMask(true /*covered */)
+				i.maybeUpdateMask(true /* covered */)
 
 				if i.mask != nil && i.mask.SkipPoint(i.pointKey.UserKey) {
+					if i.prefix {
+						// During prefix-iteration node, once a point is masked,
+						// all subsequent keys with the same prefix must also be
+						// masked according to the key ordering. We can stop and
+						// return nil.
+						//
+						// NB: The above is not just an optimization. During
+						// prefix-iteration mode, the internal iterator contract
+						// prohibits us from Next-ing beyond the first key
+						// beyond the iteration prefix. If we didn't already
+						// stop early, we would need to check if this masked
+						// point is already beyond the prefix.
+						return i.yieldNil()
+					}
+					// TODO(jackson): If we thread a base.Comparer through to
+					// InterleavingIter so that we have access to
+					// ImmediateSuccessor, we could use NextPrefix. We'd need to
+					// tweak the SpanMask interface slightly, but it's probably
+					// worthwhile.
+
 					i.pointKey, i.pointVal = i.pointIter.Next()
 					// We may have just invalidated the invariant that
 					// ensures the span's End is > the point key, so
@@ -780,27 +824,16 @@ func (i *InterleavingIter) interleaveBackward() (*base.InternalKey, base.LazyVal
 	}
 }
 
-// keyspanSeekGE seeks the keyspan iterator to the first span covering k ≥ key.
-// Note that this differs from the FragmentIterator.SeekGE semantics, which
-// seek to the first span with a start key ≥ key.
-func (i *InterleavingIter) keyspanSeekGE(key []byte, prefix []byte) {
-	// Seek using SeekLT to look for a span that starts before key, with an end
-	// boundary extending beyond key.
-	i.span = i.keyspanIter.SeekLT(key)
-	if i.span == nil || i.cmp(i.span.End, key) <= 0 {
-		// The iterator is exhausted in the reverse direction, or the span we
-		// found ends before key. Next to the first key with a start ≥ key.
-		i.span = i.keyspanIter.Next()
-	}
+// keyspanSeekGE seeks the keyspan iterator to the first span covering a key ≥ k.
+func (i *InterleavingIter) keyspanSeekGE(k []byte, prefix []byte) {
+	i.span = i.keyspanIter.SeekGE(k)
 	i.checkForwardBound(prefix)
 	i.savedKeyspan()
 }
 
-// keyspanSeekLT seeks the keyspan iterator to the last span covering k < key.
-// Note that this differs from the FragmentIterator.SeekLT semantics, which
-// seek to the last span with a start key < key.
-func (i *InterleavingIter) keyspanSeekLT(key []byte) {
-	i.span = i.keyspanIter.SeekLT(key)
+// keyspanSeekLT seeks the keyspan iterator to the last span covering a key < k.
+func (i *InterleavingIter) keyspanSeekLT(k []byte) {
+	i.span = i.keyspanIter.SeekLT(k)
 	i.checkBackwardBound()
 	// The current span's start key is not guaranteed to be less than key,
 	// because of the bounds enforcement. Consider the following example:
@@ -813,7 +846,7 @@ func (i *InterleavingIter) keyspanSeekLT(key []byte) {
 	//
 	// This problem is a consequence of the SeekLT's exclusive search key and
 	// the fact that we don't perform bounds truncation at every leaf iterator.
-	if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, key) >= 0 {
+	if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, k) >= 0 {
 		i.span = nil
 	}
 	i.savedKeyspan()

vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go

@@ -16,6 +16,7 @@ import "github.com/cockroachdb/pebble/internal/base"
 // keyspan.FragmentIterator with point keys.
 type InternalIteratorShim struct {
 	miter   MergingIter
+	mbufs   MergingBuffers
 	span    *Span
 	iterKey base.InternalKey
 }
@@ -26,7 +27,7 @@ var _ base.InternalIterator = &InternalIteratorShim{}
 // Init initializes the internal iterator shim to merge the provided fragment
 // iterators.
 func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) {
-	i.miter.Init(cmp, noopTransform, iters...)
+	i.miter.Init(cmp, noopTransform, &i.mbufs, iters...)
 }
 
 // Span returns the span containing the full set of keys over the key span at
@@ -87,6 +88,11 @@ func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) {
 	return &i.iterKey, base.MakeInPlaceValue(i.span.End)
 }
 
+// NextPrefix implements (base.InternalIterator).NextPrefix.
+func (i *InternalIteratorShim) NextPrefix([]byte) (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
 // Prev implements (base.InternalIterator).Prev.
 func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) {
 	panic("unimplemented")

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -18,12 +18,14 @@ import (
 // longer lifetimes but implementations need only guarantee stability until the
 // next positioning method.
 type FragmentIterator interface {
-	// SeekGE moves the iterator to the first span whose start key is greater
-	// than or equal to the given key.
+	// SeekGE moves the iterator to the first span covering a key greater than
+	// or equal to the given key. This is equivalent to seeking to the first
+	// span with an end key greater than the given key.
 	SeekGE(key []byte) *Span
 
-	// SeekLT moves the iterator to the last span whose start key is less than
-	// the given key.
+	// SeekLT moves the iterator to the last span covering a key less than the
+	// given key. This is equivalent to seeking to the last span with a start
+	// key less than the given key.
 	SeekLT(key []byte) *Span
 
 	// First moves the iterator to the first span.
@@ -58,7 +60,8 @@ type FragmentIterator interface {
 	Close() error
 }
 
-// TableNewSpanIter creates a new iterator for spans for the given file.
+// TableNewSpanIter creates a new iterator for range key spans for the given
+// file.
 type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
 
 // SpanIterOptions is a subset of IterOptions that are necessary to instantiate
@@ -104,6 +107,9 @@ func (i *Iter) Init(cmp base.Compare, spans []Span) {
 func (i *Iter) SeekGE(key []byte) *Span {
 	// NB: manually inlined sort.Search is ~5% faster.
 	//
+	// Define f(j) = false iff the span i.spans[j] is strictly before `key`
+	// (equivalently, i.spans[j].End ≤ key.)
+	//
 	// Define f(-1) == false and f(n) == true.
 	// Invariant: f(index-1) == false, f(upper) == true.
 	i.index = 0
@@ -111,12 +117,13 @@ func (i *Iter) SeekGE(key []byte) *Span {
 	for i.index < upper {
 		h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
 		// i.index ≤ h < upper
-		if i.cmp(key, i.spans[h].Start) > 0 {
+		if i.cmp(key, i.spans[h].End) >= 0 {
 			i.index = h + 1 // preserves f(i-1) == false
 		} else {
 			upper = h // preserves f(j) == true
 		}
 	}
+
 	// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
 	// true => answer is i.index.
 	if i.index >= len(i.spans) {

vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go

@@ -8,25 +8,23 @@ import (
 	"fmt"
 
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/manifest"
 )
 
-// Logger defines an interface for writing log messages.
-type Logger interface {
-	Infof(format string, args ...interface{})
-	Fatalf(format string, args ...interface{})
-}
-
 // LevelIter provides a merged view of spans from sstables in a level.
 // It takes advantage of level invariants to only have one sstable span block
 // open at one time, opened using the newIter function passed in.
 type LevelIter struct {
-	logger Logger
-	cmp    base.Compare
-	// Denotes if this level iter should read point key spans (i.e. rangedels,
-	// or range keys. If key type is Point, no straddle spans are emitted between
-	// files, and point key bounds are used to find files instead of range key
-	// bounds.
+	cmp base.Compare
+	// Denotes the kind of key the level iterator should read. If the key type
+	// is KeyTypePoint, the level iterator will read range tombstones (which
+	// only affect point keys). If the key type is KeyTypeRange, the level
+	// iterator will read range keys. It is invalid to configure an iterator
+	// with the KeyTypePointAndRange key type.
+	//
+	// If key type is KeyTypePoint, no straddle spans are emitted between files,
+	// and point key bounds are used to find files instead of range key bounds.
 	//
 	// TODO(bilal): Straddle spans can safely be produced in rangedel mode once
 	// we can guarantee that we will never read sstables in a level that split
@@ -50,14 +48,17 @@ type LevelIter struct {
 	dir         int
 	straddle    Span
 	straddleDir int
-	// The iter for the current file. It is nil under any of the following conditions:
+	// The iter for the current file (iterFile). It is nil under any of the
+	// following conditions:
 	// - files.Current() == nil
 	// - err != nil
 	// - straddleDir != 0, in which case iterFile is not nil and points to the
 	//   next file (in the straddleDir direction).
 	// - some other constraint, like the bounds in opts, caused the file at index to not
 	//   be relevant to the iteration.
-	iter     FragmentIterator
+	iter FragmentIterator
+	// iterFile holds the current file.
+	// INVARIANT: iterFile = files.Current()
 	iterFile *manifest.FileMetadata
 	newIter  TableNewSpanIter
 	files    manifest.LevelIterator
@@ -72,18 +73,17 @@ type LevelIter struct {
 // LevelIter implements the keyspan.FragmentIterator interface.
 var _ FragmentIterator = (*LevelIter)(nil)
 
-// newLevelIter returns a LevelIter.
-func newLevelIter(
+// NewLevelIter returns a LevelIter.
+func NewLevelIter(
 	opts SpanIterOptions,
 	cmp base.Compare,
 	newIter TableNewSpanIter,
 	files manifest.LevelIterator,
 	level manifest.Level,
-	logger Logger,
 	keyType manifest.KeyType,
 ) *LevelIter {
 	l := &LevelIter{}
-	l.Init(opts, cmp, newIter, files, level, logger, keyType)
+	l.Init(opts, cmp, newIter, files, level, keyType)
 	return l
 }
 
@@ -94,18 +94,19 @@ func (l *LevelIter) Init(
 	newIter TableNewSpanIter,
 	files manifest.LevelIterator,
 	level manifest.Level,
-	logger Logger,
 	keyType manifest.KeyType,
 ) {
 	l.err = nil
 	l.level = level
-	l.logger = logger
 	l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
 	l.cmp = cmp
 	l.iterFile = nil
 	l.newIter = newIter
 	switch keyType {
-	case manifest.KeyTypePoint, manifest.KeyTypeRange:
+	case manifest.KeyTypePoint:
+		l.keyType = keyType
+		l.files = files.Filter(keyType)
+	case manifest.KeyTypeRange:
 		l.keyType = keyType
 		l.files = files.Filter(keyType)
 	default:
@@ -196,7 +197,9 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
 
 	f := l.findFileGE(key)
 	if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
+		// Peek at the previous file.
 		prevFile := l.files.Prev()
+		l.files.Next()
 		if prevFile != nil {
 			// We could unconditionally return an empty span between the seek key and
 			// f.SmallestRangeKey, however if this span is to the left of all range
@@ -212,11 +215,10 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
 			//
 			// TODO(bilal): Investigate ways to be able to return straddle spans in
 			// cases similar to the above, while still retaining correctness.
-			l.files.Next()
 			// Return a straddling key instead of loading the file.
 			l.iterFile = f
 			if err := l.Close(); err != nil {
-				return nil
+				return l.verify(nil)
 			}
 			l.straddleDir = +1
 			l.straddle = Span{
@@ -224,15 +226,15 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
 				End:   f.SmallestRangeKey.UserKey,
 				Keys:  nil,
 			}
-			return &l.straddle
+			return l.verify(&l.straddle)
 		}
 	}
 	loadFileIndicator := l.loadFile(f, +1)
 	if loadFileIndicator == noFileLoaded {
-		return nil
+		return l.verify(nil)
 	}
 	if span := l.iter.SeekGE(key); span != nil {
-		return span
+		return l.verify(span)
 	}
 	return l.skipEmptyFileForward()
 }
@@ -246,7 +248,9 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
 
 	f := l.findFileLT(key)
 	if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
+		// Peek at the next file.
 		nextFile := l.files.Next()
+		l.files.Prev()
 		if nextFile != nil {
 			// We could unconditionally return an empty span between f.LargestRangeKey
 			// and the seek key, however if this span is to the right of all range keys
@@ -262,11 +266,10 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
 			//
 			// TODO(bilal): Investigate ways to be able to return straddle spans in
 			// cases similar to the above, while still retaining correctness.
-			l.files.Prev()
 			// Return a straddling key instead of loading the file.
 			l.iterFile = f
 			if err := l.Close(); err != nil {
-				return nil
+				return l.verify(nil)
 			}
 			l.straddleDir = -1
 			l.straddle = Span{
@@ -274,14 +277,14 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
 				End:   nextFile.SmallestRangeKey.UserKey,
 				Keys:  nil,
 			}
-			return &l.straddle
+			return l.verify(&l.straddle)
 		}
 	}
-	if l.loadFile(l.findFileLT(key), -1) == noFileLoaded {
-		return nil
+	if l.loadFile(f, -1) == noFileLoaded {
+		return l.verify(nil)
 	}
 	if span := l.iter.SeekLT(key); span != nil {
-		return span
+		return l.verify(span)
 	}
 	return l.skipEmptyFileBackward()
 }
@@ -294,10 +297,10 @@ func (l *LevelIter) First() *Span {
 	l.err = nil // clear cached iteration error
 
 	if l.loadFile(l.files.First(), +1) == noFileLoaded {
-		return nil
+		return l.verify(nil)
 	}
 	if span := l.iter.First(); span != nil {
-		return span
+		return l.verify(span)
 	}
 	return l.skipEmptyFileForward()
 }
@@ -310,10 +313,10 @@ func (l *LevelIter) Last() *Span {
 	l.err = nil // clear cached iteration error
 
 	if l.loadFile(l.files.Last(), -1) == noFileLoaded {
-		return nil
+		return l.verify(nil)
 	}
 	if span := l.iter.Last(); span != nil {
-		return span
+		return l.verify(span)
 	}
 	return l.skipEmptyFileBackward()
 }
@@ -321,7 +324,7 @@ func (l *LevelIter) Last() *Span {
 // Next implements keyspan.FragmentIterator.
 func (l *LevelIter) Next() *Span {
 	if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) {
-		return nil
+		return l.verify(nil)
 	}
 	if l.iter == nil && l.iterFile == nil {
 		// l.dir <= 0
@@ -331,7 +334,7 @@ func (l *LevelIter) Next() *Span {
 
 	if l.iter != nil {
 		if span := l.iter.Next(); span != nil {
-			return span
+			return l.verify(span)
 		}
 	}
 	return l.skipEmptyFileForward()
@@ -340,7 +343,7 @@ func (l *LevelIter) Next() *Span {
 // Prev implements keyspan.FragmentIterator.
 func (l *LevelIter) Prev() *Span {
 	if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) {
-		return nil
+		return l.verify(nil)
 	}
 	if l.iter == nil && l.iterFile == nil {
 		// l.dir >= 0
@@ -350,7 +353,7 @@ func (l *LevelIter) Prev() *Span {
 
 	if l.iter != nil {
 		if span := l.iter.Prev(); span != nil {
-			return span
+			return l.verify(span)
 		}
 	}
 	return l.skipEmptyFileBackward()
@@ -367,7 +370,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
 		// Straddle spans are not created in rangedel mode.
 		if err := l.Close(); err != nil {
 			l.err = err
-			return nil
+			return l.verify(nil)
 		}
 		startKey := l.iterFile.LargestRangeKey.UserKey
 		// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -375,7 +378,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
 		// which it should be due to the Close() call above.
 		l.iterFile = l.files.Next()
 		if l.iterFile == nil {
-			return nil
+			return l.verify(nil)
 		}
 		endKey := l.iterFile.SmallestRangeKey.UserKey
 		if l.cmp(startKey, endKey) < 0 {
@@ -386,7 +389,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
 				End:   endKey,
 			}
 			l.straddleDir = +1
-			return &l.straddle
+			return l.verify(&l.straddle)
 		}
 	} else if l.straddleDir < 0 {
 		// We were at a straddle key, but are now changing directions. l.iterFile
@@ -405,7 +408,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
 			fileToLoad = l.files.Next()
 		}
 		if l.loadFile(fileToLoad, +1) == noFileLoaded {
-			return nil
+			return l.verify(nil)
 		}
 		span = l.iter.First()
 		// In rangedel mode, we can expect to get empty files that we'd need to
@@ -414,7 +417,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
 			break
 		}
 	}
-	return span
+	return l.verify(span)
 }
 
 func (l *LevelIter) skipEmptyFileBackward() *Span {
@@ -428,7 +431,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
 		l.iterFile != nil && l.iter != nil {
 		if err := l.Close(); err != nil {
 			l.err = err
-			return nil
+			return l.verify(nil)
 		}
 		endKey := l.iterFile.SmallestRangeKey.UserKey
 		// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -436,7 +439,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
 		// which it should be due to the Close() call above.
 		l.iterFile = l.files.Prev()
 		if l.iterFile == nil {
-			return nil
+			return l.verify(nil)
 		}
 		startKey := l.iterFile.LargestRangeKey.UserKey
 		if l.cmp(startKey, endKey) < 0 {
@@ -447,7 +450,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
 				End:   endKey,
 			}
 			l.straddleDir = -1
-			return &l.straddle
+			return l.verify(&l.straddle)
 		}
 	} else if l.straddleDir > 0 {
 		// We were at a straddle key, but are now changing directions. l.iterFile
@@ -464,7 +467,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
 			fileToLoad = l.files.Prev()
 		}
 		if l.loadFile(fileToLoad, -1) == noFileLoaded {
-			return nil
+			return l.verify(nil)
 		}
 		span = l.iter.Last()
 		// In rangedel mode, we can expect to get empty files that we'd need to
@@ -474,7 +477,22 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
 			break
 		}
 	}
-	return span
+	return l.verify(span)
+}
+
+// verify is invoked whenever a span is returned from an iterator positioning
+// method to a caller. During invariant builds, it asserts invariants to the
+// caller.
+func (l *LevelIter) verify(s *Span) *Span {
+	// NB: Do not add any logic outside the invariants.Enabled conditional to
+	// ensure that verify is always compiled away in production builds.
+	if invariants.Enabled {
+		if f := l.files.Current(); f != l.iterFile {
+			panic(fmt.Sprintf("LevelIter.files.Current (%s) and l.iterFile (%s) diverged",
+				f, l.iterFile))
+		}
+	}
+	return s
 }
 
 // Error implements keyspan.FragmentIterator.

vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go

@@ -21,6 +21,10 @@ import (
 // seeks would require introducing key comparisons to switchTo{Min,Max}Heap
 // where there currently are none.
 
+// TODO(jackson): There are several opportunities to use base.Equal in the
+// MergingIter implementation, but will require a bit of plumbing to thread the
+// Equal function.
+
 // Transformer defines a transformation to be applied to a Span.
 type Transformer interface {
 	// Transform takes a Span as input and writes the transformed Span to the
@@ -45,9 +49,9 @@ var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst
 	return nil
 })
 
-// visibleTransform filters keys that are invisible at the provided snapshot
+// VisibleTransform filters keys that are invisible at the provided snapshot
 // sequence number.
-func visibleTransform(snapshot uint64) Transformer {
+func VisibleTransform(snapshot uint64) Transformer {
 	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
 		dst.Start, dst.End = s.Start, s.End
 		dst.Keys = dst.Keys[:0]
@@ -76,34 +80,34 @@ func visibleTransform(snapshot uint64) Transformer {
 // trailer descending. If the MergingIter is configured with a Transformer, it's
 // permitted to modify the ordering of the spans' keys returned by MergingIter.
 //
-// Algorithm
+// # Algorithm
 //
 // The merging iterator wraps child iterators, merging and fragmenting spans
 // across levels. The high-level algorithm is:
 //
-//     1. Initialize the heap with bound keys from child iterators' spans.
-//     2. Find the next [or previous] two unique user keys' from bounds.
-//     3. Consider the span formed between the two unique user keys a candidate
-//        span.
-//     4. Determine if any of the child iterators' spans overlap the candidate
-//        span.
-//         4a. If any of the child iterator's current bounds are end keys
-//             (during forward iteration) or start keys (during reverse
-//             iteration), then all the spans with that bound overlap the
-//             candidate span.
-//         4b. Apply the configured transform, which may remove keys.
-//         4b. If no spans overlap, forget the smallest (forward iteration)
-//             or largest (reverse iteration) unique user key and advance
-//             the iterators to the next unique user key. Start again from 3.
+//  1. Initialize the heap with bound keys from child iterators' spans.
+//  2. Find the next [or previous] two unique user keys' from bounds.
+//  3. Consider the span formed between the two unique user keys a candidate
+//     span.
+//  4. Determine if any of the child iterators' spans overlap the candidate
+//     span.
+//     4a. If any of the child iterator's current bounds are end keys
+//     (during forward iteration) or start keys (during reverse
+//     iteration), then all the spans with that bound overlap the
+//     candidate span.
+//     4b. Apply the configured transform, which may remove keys.
+//     4c. If no spans overlap, forget the smallest (forward iteration)
+//     or largest (reverse iteration) unique user key and advance
+//     the iterators to the next unique user key. Start again from 3.
 //
-// Detailed algorithm
+// # Detailed algorithm
 //
 // Each level (i0, i1, ...) has a user-provided input FragmentIterator. The
 // merging iterator steps through individual boundaries of the underlying
 // spans separately. If the underlying FragmentIterator has fragments
 // [a,b){#2,#1} [b,c){#1} the mergingIterLevel.{next,prev} step through:
 //
-//     (a, start), (b, end), (b, start), (c, end)
+//	(a, start), (b, end), (b, start), (c, end)
 //
 // Note that (a, start) and (b, end) are observed ONCE each, despite two keys
 // sharing those bounds. Also note that (b, end) and (b, start) are two distinct
@@ -119,18 +123,18 @@ func visibleTransform(snapshot uint64) Transformer {
 // determine which span is next, but it's also responsible for fragmenting
 // overlapping spans. Consider the example:
 //
-//            i0:     b---d e-----h
-//            i1:   a---c         h-----k
-//            i2:   a------------------------------p
+//	       i0:     b---d e-----h
+//	       i1:   a---c         h-----k
+//	       i2:   a------------------------------p
 //
-//     fragments:   a-b-c-d-e-----h-----k----------p
+//	fragments:   a-b-c-d-e-----h-----k----------p
 //
 // None of the individual child iterators contain a span with the exact bounds
 // [c,d), but the merging iterator must produce a span [c,d). To accomplish
 // this, the merging iterator visits every span between unique boundary user
 // keys. In the above example, this is:
 //
-//     [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p)
+//	[a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p)
 //
 // The merging iterator first initializes the heap to prepare for iteration.
 // The description below discusses the mechanics of forward iteration after a
@@ -141,9 +145,9 @@ func visibleTransform(snapshot uint64) Transformer {
 // mergingIterLevel to the first bound of the first fragment. In the above
 // example, this seeks the child iterators to:
 //
-//     i0: (b, boundKindFragmentStart, [ [b,d) ])
-//     i1: (a, boundKindFragmentStart, [ [a,c) ])
-//     i2: (a, boundKindFragmentStart, [ [a,p) ])
+//	i0: (b, boundKindFragmentStart, [ [b,d) ])
+//	i1: (a, boundKindFragmentStart, [ [a,c) ])
+//	i2: (a, boundKindFragmentStart, [ [a,p) ])
 //
 // After fixing up the heap, the root of the heap is a boundKey with the
 // smallest user key ('a' in the example). Once the heap is setup for iteration
@@ -160,9 +164,9 @@ func visibleTransform(snapshot uint64) Transformer {
 // In the above example, this results in m.start = 'a', m.end = 'b' and child
 // iterators in the following positions:
 //
-//     i0: (b, boundKindFragmentStart, [ [b,d) ])
-//     i1: (c, boundKindFragmentEnd,   [ [a,c) ])
-//     i2: (p, boundKindFragmentEnd,   [ [a,p) ])
+//	i0: (b, boundKindFragmentStart, [ [b,d) ])
+//	i1: (c, boundKindFragmentEnd,   [ [a,c) ])
+//	i2: (p, boundKindFragmentEnd,   [ [a,p) ])
 //
 // With the user key bounds of the next merged span established,
 // findNextFragmentSet must determine which, if any, fragments overlap the span.
@@ -191,7 +195,7 @@ func visibleTransform(snapshot uint64) Transformer {
 // findNextFragmentSet loops, repeating the above process again until it finds a
 // span that does contain keys.
 //
-// Memory safety
+// # Memory safety
 //
 // The FragmentIterator interface only guarantees stability of a Span and its
 // associated slices until the next positioning method is called. Adjacent Spans
@@ -207,24 +211,14 @@ func visibleTransform(snapshot uint64) Transformer {
 // accommodate this, find{Next,Prev}FragmentSet copy the initial boundary if the
 // subsequent Next/Prev would move to the next span.
 type MergingIter struct {
-	levels []mergingIterLevel
-	heap   mergingIterHeap
+	*MergingBuffers
 	// start and end hold the bounds for the span currently under the
 	// iterator position.
 	//
 	// Invariant: None of the levels' iterators contain spans with a bound
 	// between start and end. For all bounds b, b ≤ start || b ≥ end.
 	start, end []byte
-	// buf is a buffer used to save [start, end) boundary keys.
-	buf []byte
-	// keys holds all of the keys across all levels that overlap the key span
-	// [start, end), sorted by Trailer descending. This slice is reconstituted
-	// in synthesizeKeys from each mergingIterLevel's keys every time the
-	// [start, end) bounds change.
-	//
-	// Each element points into a child iterator's memory, so the keys may not
-	// be directly modified.
-	keys keysBySeqNumKind
+
 	// transformer defines a transformation to be applied to a span before it's
 	// yielded to the user. Transforming may filter individual keys contained
 	// within the span.
@@ -233,9 +227,8 @@ type MergingIter struct {
 	// destination for transforms. Every tranformed span overwrites the
 	// previous.
 	span Span
-
-	err error
-	dir int8
+	err  error
+	dir  int8
 
 	// alloc preallocates mergingIterLevel and mergingIterItems for use by the
 	// merging iterator. As long as the merging iterator is used with
@@ -249,6 +242,35 @@ type MergingIter struct {
 	}
 }
 
+// MergingBuffers holds buffers used while merging keyspans.
+type MergingBuffers struct {
+	// keys holds all of the keys across all levels that overlap the key span
+	// [start, end), sorted by Trailer descending. This slice is reconstituted
+	// in synthesizeKeys from each mergingIterLevel's keys every time the
+	// [start, end) bounds change.
+	//
+	// Each element points into a child iterator's memory, so the keys may not
+	// be directly modified.
+	keys keysBySeqNumKind
+	// levels holds levels allocated by MergingIter.init. The MergingIter will
+	// prefer use of its `manifest.NumLevels+3` array, so this slice will be
+	// longer if set.
+	levels []mergingIterLevel
+	// heap holds a slice for the merging iterator heap allocated by
+	// MergingIter.init. The MergingIter will prefer use of its
+	// `manifest.NumLevels+3` items array, so this slice will be longer if set.
+	heap mergingIterHeap
+	// buf is a buffer used to save [start, end) boundary keys.
+	buf []byte
+}
+
+// PrepareForReuse discards any excessively large buffers.
+func (bufs *MergingBuffers) PrepareForReuse() {
+	if cap(bufs.buf) > bufferReuseMaxCapacity {
+		bufs.buf = nil
+	}
+}
+
 // MergingIter implements the FragmentIterator interface.
 var _ FragmentIterator = (*MergingIter)(nil)
 
@@ -300,13 +322,15 @@ func (l *mergingIterLevel) prev() {
 }
 
 // Init initializes the merging iterator with the provided fragment iterators.
-func (m *MergingIter) Init(cmp base.Compare, transformer Transformer, iters ...FragmentIterator) {
-	levels, items := m.levels, m.heap.items
-
+func (m *MergingIter) Init(
+	cmp base.Compare, transformer Transformer, bufs *MergingBuffers, iters ...FragmentIterator,
+) {
 	*m = MergingIter{
-		heap:        mergingIterHeap{cmp: cmp},
-		transformer: transformer,
+		MergingBuffers: bufs,
+		transformer:    transformer,
 	}
+	m.heap.cmp = cmp
+	levels, items := m.levels, m.heap.items
 
 	// Invariant: cap(levels) >= cap(items)
 	// Invariant: cap(alloc.levels) == cap(alloc.items)
@@ -334,38 +358,70 @@ func (m *MergingIter) AddLevel(iter FragmentIterator) {
 	m.levels = append(m.levels, mergingIterLevel{iter: iter})
 }
 
-// SeekGE moves the iterator to the first span with a start key greater than or
-// equal to key.
+// SeekGE moves the iterator to the first span covering a key greater than
+// or equal to the given key. This is equivalent to seeking to the first
+// span with an end key greater than the given key.
 func (m *MergingIter) SeekGE(key []byte) *Span {
 	m.invalidate() // clear state about current position
+
+	// SeekGE(k) seeks to the first span with an end key greater than the given
+	// key. The merged span M that we're searching for might straddle the seek
+	// `key`. In this case, the M.Start may be a key ≤ the seek key.
+	//
+	// Consider a SeekGE(dog) in the following example.
+	//
+	//            i0:     b---d e-----h
+	//            i1:   a---c         h-----k
+	//            i2:   a------------------------------p
+	//        merged:   a-b-c-d-e-----h-----k----------p
+	//
+	// The merged span M containing 'dog' is [d,e). The 'd' of the merged span
+	// comes from i0's [b,d)'s end boundary. The [b,d) span does not cover any
+	// key >= dog, so we cannot find the span by positioning the child iterators
+	// using a SeekGE(dog).
+	//
+	// Instead, if we take all the child iterators' spans bounds:
+	//                  a b c d e     h     k          p
+	// We want to partition them into keys ≤ `key` and keys > `key`.
+	//                        dog
+	//                         │
+	//                  a b c d│e     h     k          p
+	//                         │
+	// The largest key on the left of the partition forms the merged span's
+	// start key, and the smallest key on the right of the partition forms the
+	// merged span's end key. Recharacterized:
+	//
+	//   M.Start: the largest boundary ≤ k of any child span
+	//   M.End:   the smallest boundary > k of any child span
+	//
+	// The FragmentIterator interface doesn't implement seeking by all bounds,
+	// it implements seeking by containment. A SeekGE(k) will ensure we observe
+	// all start boundaries ≥ k and all end boundaries > k but does not ensure
+	// we observe end boundaries = k or any boundaries < k.  A SeekLT(k) will
+	// ensure we observe all start boundaries < k and all end boundaries ≤ k but
+	// does not ensure we observe any start boundaries = k or any boundaries >
+	// k. This forces us to seek in one direction and step in the other.
+	//
+	// In a SeekGE, we want to end up oriented in the forward direction when
+	// complete, so we begin with searching for M.Start by SeekLT-ing every
+	// child iterator to `k`.  For every child span found, we determine the
+	// largest bound ≤ `k` and use it to initialize our max heap. The resulting
+	// root of the max heap is a preliminary value for `M.Start`.
 	for i := range m.levels {
 		l := &m.levels[i]
-
-		// A SeekGE requires we position each level at the smallest bound ≥ key.
-		// We must search through both inclusive start and exclusive end bounds.
-		// Note that this search requirement differs from FragmentIterator's
-		// .SeekGE'semantics, which returns the span with the smallest start key
-		// ≥ key. To remedy this difference, we find the last span less than
-		// key. If its end boundary is greater than or equal to key, we use it.
-		// Otherwise we use the start boundary of the next span which
-		// necessarily has a start ≥ key.
 		s := l.iter.SeekLT(key)
-		if s != nil && m.cmp(s.End, key) >= 0 {
-			// s.End ≥ key
-			// We need to use this span's end bound.
+		if s == nil {
+			l.heapKey = boundKey{kind: boundKindInvalid}
+		} else if m.cmp(s.End, key) <= 0 {
 			l.heapKey = boundKey{
 				kind: boundKindFragmentEnd,
 				key:  s.End,
 				span: s,
 			}
-			continue
-		}
-		// s.End < key
-		// The span `s` ends before key. Next to the first span with a Start ≥
-		// key, and use that.
-		if s = l.iter.Next(); s == nil {
-			l.heapKey = boundKey{kind: boundKindInvalid}
 		} else {
+			// s.End > key && s.Start < key
+			// We need to use this span's start bound, since that's the largest
+			// bound ≤ key.
 			l.heapKey = boundKey{
 				kind: boundKindFragmentStart,
 				key:  s.Start,
@@ -373,36 +429,201 @@ func (m *MergingIter) SeekGE(key []byte) *Span {
 			}
 		}
 	}
-	m.initMinHeap()
-	return m.findNextFragmentSet()
-}
+	m.initMaxHeap()
+	if m.err != nil {
+		return nil
+	} else if len(m.heap.items) == 0 {
+		// There are no spans covering any key < `key`. There is no span that
+		// straddles the seek key. Reorient the heap into a min heap and return
+		// the first span we find in the forward direction.
+		m.switchToMinHeap()
+		return m.findNextFragmentSet()
+	}
 
-// SeekLT moves the iterator to the last span with a start key less than key.
-func (m *MergingIter) SeekLT(key []byte) *Span {
-	// TODO(jackson): Evaluate whether there's an implementation of SeekLT
-	// independent of SeekGE that is more efficient. It's tricky, because the
-	// span we should return might straddle `key` itself.
+	// The heap root is now the largest boundary key b such that:
+	//   1. b < k
+	//   2. b = k, and b is an end boundary
+	// There's a third case that we will need to consider later, after we've
+	// switched to a min heap:
+	//   3. there exists a start boundary key b such that b = k.
+	// A start boundary key equal to k would not be surfaced when we seeked all
+	// the levels using SeekLT(k), since no key <k would be covered within a
+	// span within an inclusive `k` start boundary.
 	//
-	// Consider the scenario:
-	//       a----------l      #2
-	//         b-----------m   #1
-	//
-	// The merged, fully-fragmented spans that MergingIter exposes to the caller
-	// have bounds:
-	//        a-b              #2
-	//          b--------l     #2
-	//          b--------l     #1
-	//                   l-m   #1
-	//
-	// A call SeekLT(c) must return the largest of the above spans with a
-	// Start user key < key: [b,l)#1. This requires examining bounds both < 'c'
-	// (the 'b' of [b,m)#1's start key) and bounds ≥ 'c' (the 'l' of ([a,l)#2's
-	// end key).
-	if s := m.SeekGE(key); s == nil && m.err != nil {
+	// Assume that the tightest boundary ≤ k is the current heap root (cases 1 &
+	// 2). After we switch to a min heap, we'll check for the third case and
+	// adjust the start boundary if necessary.
+	m.start = m.heap.items[0].boundKey.key
+
+	// Before switching the direction of the heap, save a copy of the start
+	// boundary if it's the end boundary of some child span. Next-ing the child
+	// iterator might switch files and invalidate the memory of the bound.
+	if m.heap.items[0].boundKey.kind == boundKindFragmentEnd {
+		m.buf = append(m.buf[:0], m.start...)
+		m.start = m.buf
+	}
+
+	// Switch to a min heap. This will move each level to the next bound in
+	// every level, and then establish a min heap. This allows us to obtain the
+	// smallest boundary key > `key`, which will serve as our candidate end
+	// bound.
+	m.switchToMinHeap()
+	if m.err != nil {
+		return nil
+	} else if len(m.heap.items) == 0 {
 		return nil
 	}
-	// Prev to the previous span.
-	return m.Prev()
+
+	// Check for the case 3 described above. It's possible that when we switch
+	// heap directions, we discover a start boundary of some child span that is
+	// equal to the seek key `key`. In this case, we want this key to be our
+	// start boundary.
+	if m.heap.items[0].boundKey.kind == boundKindFragmentStart &&
+		m.cmp(m.heap.items[0].boundKey.key, key) == 0 {
+		// Call findNextFragmentSet, which will set m.start to the heap root and
+		// proceed forward.
+		return m.findNextFragmentSet()
+	}
+
+	m.end = m.heap.items[0].boundKey.key
+	if found, s := m.synthesizeKeys(+1); found && s != nil {
+		return s
+	}
+	return m.findNextFragmentSet()
+
+}
+
+// SeekLT moves the iterator to the last span covering a key less than the
+// given key. This is equivalent to seeking to the last span with a start
+// key less than the given key.
+func (m *MergingIter) SeekLT(key []byte) *Span {
+	m.invalidate() // clear state about current position
+
+	// SeekLT(k) seeks to the last span with a start key less than the given
+	// key. The merged span M that we're searching for might straddle the seek
+	// `key`. In this case, the M.End may be a key ≥ the seek key.
+	//
+	// Consider a SeekLT(dog) in the following example.
+	//
+	//            i0:     b---d e-----h
+	//            i1:   a---c         h-----k
+	//            i2:   a------------------------------p
+	//        merged:   a-b-c-d-e-----h-----k----------p
+	//
+	// The merged span M containing the largest key <'dog' is [d,e). The 'e' of
+	// the merged span comes from i0's [e,h)'s start boundary. The [e,h) span
+	// does not cover any key < dog, so we cannot find the span by positioning
+	// the child iterators using a SeekLT(dog).
+	//
+	// Instead, if we take all the child iterators' spans bounds:
+	//                  a b c d e     h     k          p
+	// We want to partition them into keys < `key` and keys ≥ `key`.
+	//                        dog
+	//                         │
+	//                  a b c d│e     h     k          p
+	//                         │
+	// The largest key on the left of the partition forms the merged span's
+	// start key, and the smallest key on the right of the partition forms the
+	// merged span's end key. Recharacterized:
+	//
+	//   M.Start: the largest boundary < k of any child span
+	//   M.End:   the smallest boundary ≥ k of any child span
+	//
+	// The FragmentIterator interface doesn't implement seeking by all bounds,
+	// it implements seeking by containment. A SeekGE(k) will ensure we observe
+	// all start boundaries ≥ k and all end boundaries > k but does not ensure
+	// we observe end boundaries = k or any boundaries < k.  A SeekLT(k) will
+	// ensure we observe all start boundaries < k and all end boundaries ≤ k but
+	// does not ensure we observe any start boundaries = k or any boundaries >
+	// k. This forces us to seek in one direction and step in the other.
+	//
+	// In a SeekLT, we want to end up oriented in the backward direction when
+	// complete, so we begin with searching for M.End by SeekGE-ing every
+	// child iterator to `k`. For every child span found, we determine the
+	// smallest bound ≥ `k` and use it to initialize our min heap. The resulting
+	// root of the min heap is a preliminary value for `M.End`.
+	for i := range m.levels {
+		l := &m.levels[i]
+		s := l.iter.SeekGE(key)
+		if s == nil {
+			l.heapKey = boundKey{kind: boundKindInvalid}
+		} else if m.cmp(s.Start, key) >= 0 {
+			l.heapKey = boundKey{
+				kind: boundKindFragmentStart,
+				key:  s.Start,
+				span: s,
+			}
+		} else {
+			// s.Start < key
+			// We need to use this span's end bound, since that's the smallest
+			// bound > key.
+			l.heapKey = boundKey{
+				kind: boundKindFragmentEnd,
+				key:  s.End,
+				span: s,
+			}
+		}
+	}
+	m.initMinHeap()
+	if m.err != nil {
+		return nil
+	} else if len(m.heap.items) == 0 {
+		// There are no spans covering any key ≥ `key`. There is no span that
+		// straddles the seek key. Reorient the heap into a max heap and return
+		// the first span we find in the reverse direction.
+		m.switchToMaxHeap()
+		return m.findPrevFragmentSet()
+	}
+
+	// The heap root is now the smallest boundary key b such that:
+	//   1. b > k
+	//   2. b = k, and b is a start boundary
+	// There's a third case that we will need to consider later, after we've
+	// switched to a max heap:
+	//   3. there exists an end boundary key b such that b = k.
+	// An end boundary key equal to k would not be surfaced when we seeked all
+	// the levels using SeekGE(k), since k would not be contained within the
+	// exclusive end boundary.
+	//
+	// Assume that the tightest boundary ≥ k is the current heap root (cases 1 &
+	// 2). After we switch to a max heap, we'll check for the third case and
+	// adjust the end boundary if necessary.
+	m.end = m.heap.items[0].boundKey.key
+
+	// Before switching the direction of the heap, save a copy of the end
+	// boundary if it's the start boundary of some child span. Prev-ing the
+	// child iterator might switch files and invalidate the memory of the bound.
+	if m.heap.items[0].boundKey.kind == boundKindFragmentStart {
+		m.buf = append(m.buf[:0], m.end...)
+		m.end = m.buf
+	}
+
+	// Switch to a max heap. This will move each level to the previous bound in
+	// every level, and then establish a max heap. This allows us to obtain the
+	// largest boundary key < `key`, which will serve as our candidate start
+	// bound.
+	m.switchToMaxHeap()
+	if m.err != nil {
+		return nil
+	} else if len(m.heap.items) == 0 {
+		return nil
+	}
+	// Check for the case 3 described above. It's possible that when we switch
+	// heap directions, we discover an end boundary of some child span that is
+	// equal to the seek key `key`. In this case, we want this key to be our end
+	// boundary.
+	if m.heap.items[0].boundKey.kind == boundKindFragmentEnd &&
+		m.cmp(m.heap.items[0].boundKey.key, key) == 0 {
+		// Call findPrevFragmentSet, which will set m.end to the heap root and
+		// proceed backwards.
+		return m.findPrevFragmentSet()
+	}
+
+	m.start = m.heap.items[0].boundKey.key
+	if found, s := m.synthesizeKeys(-1); found && s != nil {
+		return s
+	}
+	return m.findPrevFragmentSet()
 }
 
 // First seeks the iterator to the first span.
@@ -823,8 +1044,8 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) {
 	// sorted by trailer descending for the range key iteration use case.
 	sort.Sort(&m.keys)
 
-	// Apply the configured transform. See visibleTransform.
-	s := Span{
+	// Apply the configured transform. See VisibleTransform.
+	m.span = Span{
 		Start:     m.start,
 		End:       m.end,
 		Keys:      m.keys,
@@ -832,7 +1053,7 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) {
 	}
 	// NB: m.heap.cmp is a base.Compare, whereas m.cmp is a method on
 	// MergingIter.
-	if err := m.transformer.Transform(m.heap.cmp, s, &m.span); err != nil {
+	if err := m.transformer.Transform(m.heap.cmp, m.span, &m.span); err != nil {
 		m.err = err
 		return false, nil
 	}

vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go

@@ -6,34 +6,6 @@ package keyspan
 
 import "github.com/cockroachdb/pebble/internal/base"
 
-// SeekGE seeks to the span that contains the target key or the first span past
-// the target key.
-func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
-	// NB: We use SeekLT in order to land on the proper span for a search
-	// key that resides in the middle of a span. Consider the scenario:
-	//
-	//     a---e
-	//         e---i
-	//
-	// The spans are indexed by their start keys `a` and `e`. If the
-	// search key is `c` we want to land on the span [a,e). If we were to
-	// use SeekGE then the search key `c` would land on the span [e,i) and
-	// we'd have to backtrack. The one complexity here is what happens for the
-	// search key `e`. In that case SeekLT will land us on the span [a,e)
-	// and we'll have to move forward.
-	iterSpan := iter.SeekLT(key)
-
-	// Invariant: key > iterSpan.Start
-
-	if iterSpan == nil || cmp(key, iterSpan.End) >= 0 {
-		// The current span lies entirely before the search key, or the iterator
-		// is exhausted. Advance the iterator to the next span which is
-		// guaranteed to lie at or past the search key.
-		iterSpan = iter.Next()
-	}
-	return iterSpan
-}
-
 // SeekLE seeks to the span that contains or is before the target key.
 func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
 	// NB: We use SeekLT in order to land on the proper span for a search

vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go

@@ -24,7 +24,8 @@ import (
 // Note that the start user key is inclusive and the end user key is exclusive.
 //
 // Currently the only supported key kinds are:
-//   RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
+//
+//	RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
 type Span struct {
 	// Start and End encode the user key range of all the contained items, with
 	// an inclusive start key and exclusive end key. Both Start and End must be
@@ -75,6 +76,15 @@ func (k Key) SeqNum() uint64 {
 	return k.Trailer >> 8
 }
 
+// VisibleAt returns true if the provided key is visible at the provided
+// snapshot sequence number. It interprets batch sequence numbers as always
+// visible, because non-visible batch span keys are filtered when they're
+// fragmented.
+func (k Key) VisibleAt(snapshot uint64) bool {
+	seq := k.SeqNum()
+	return seq < snapshot || seq&base.InternalKeySeqNumBatch != 0
+}
+
 // Kind returns the kind component of the key.
 func (k Key) Kind() base.InternalKeyKind {
 	return base.InternalKeyKind(k.Trailer & 0xff)
@@ -396,6 +406,17 @@ func SortKeysByTrailer(keys *[]Key) {
 	sort.Sort(sorted)
 }
 
+// KeysBySuffix implements sort.Interface, sorting its member Keys slice to by
+// Suffix in the order dictated by Cmp.
+type KeysBySuffix struct {
+	Cmp  base.Compare
+	Keys []Key
+}
+
+func (s *KeysBySuffix) Len() int           { return len(s.Keys) }
+func (s *KeysBySuffix) Less(i, j int) bool { return s.Cmp(s.Keys[i].Suffix, s.Keys[j].Suffix) < 0 }
+func (s *KeysBySuffix) Swap(i, j int)      { s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] }
+
 // ParseSpan parses the string representation of a Span. It's intended for
 // tests. ParseSpan panics if passed a malformed span representation.
 func ParseSpan(input string) Span {

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base"
 // truncated to be contained within the range [lower, upper). If start and end
 // are specified, filter out any spans that are completely outside those bounds.
 func Truncate(
-	cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
+	cmp base.Compare,
+	iter FragmentIterator,
+	lower, upper []byte,
+	start, end *base.InternalKey,
+	panicOnPartialOverlap bool,
 ) FragmentIterator {
 	return Filter(iter, func(in *Span, out *Span) (keep bool) {
 		out.Start, out.End = in.Start, in.End
 		out.Keys = append(out.Keys[:0], in.Keys...)
 
-		// Ignore this span if it lies completely outside [start, end].
+		// Ignore this span if it lies completely outside start, end. Note that
+		// end endInclusive indicated whether end is inclusive.
 		//
 		// The comparison between s.End and start is by user key only, as
 		// the span is exclusive at s.End, so comparing by user keys
@@ -48,13 +53,22 @@ func Truncate(
 				// Wholly within the end bound. Keep it.
 			}
 		}
+
+		var truncated bool
 		// Truncate the bounds to lower and upper.
 		if cmp(in.Start, lower) < 0 {
+			truncated = true
 			out.Start = lower
 		}
 		if cmp(in.End, upper) > 0 {
+			truncated = true
 			out.End = upper
 		}
+
+		if panicOnPartialOverlap && truncated {
+			panic("pebble: bounds should not be truncated")
+		}
+
 		return !out.Empty() && cmp(out.Start, out.End) < 0
 	})
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go

@@ -12,6 +12,7 @@ import (
 	"unsafe"
 
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // The Annotator type defined below is used by other packages to lazily
@@ -110,7 +111,16 @@ type leafNode struct {
 	ref   int32
 	count int16
 	leaf  bool
-	items [maxItems]*FileMetadata
+	// subtreeCount holds the count of files in the entire subtree formed by
+	// this node. For leaf nodes, subtreeCount is always equal to count. For
+	// non-leaf nodes, it's the sum of count plus all the children's
+	// subtreeCounts.
+	//
+	// NB: We could move this field to the end of the node struct, since leaf =>
+	// count=subtreeCount, however the unsafe casting [leafToNode] performs make
+	// it risky and cumbersome.
+	subtreeCount int
+	items        [maxItems]*FileMetadata
 	// annot contains one annotation per annotator, merged over the entire
 	// node's files (and all descendants for non-leaf nodes).
 	annot []annotation
@@ -167,7 +177,7 @@ func mut(n **node) *node {
 	// reference count to be greater than 1, we might be racing
 	// with another call to decRef on this node.
 	c := (*n).clone()
-	(*n).decRef(true /* recursive */, nil)
+	(*n).decRef(true /* contentsToo */, nil)
 	*n = c
 	// NB: We don't need to clear annotations, because (*node).clone does not
 	// copy them.
@@ -179,19 +189,25 @@ func (n *node) incRef() {
 	atomic.AddInt32(&n.ref, 1)
 }
 
-// decRef releases a reference to the node. If requested, the method
-// will recurse into child nodes and decrease their refcounts as well.
+// decRef releases a reference to the node. If requested, the method will unref
+// its items and recurse into child nodes and decrease their refcounts as well.
+// Some internal codepaths that manually copy the node's items or children to
+// new nodes pass contentsToo=false to preserve existing reference counts during
+// operations that should yield a net-zero change to descendant refcounts.
 // When a node is released, its contained files are dereferenced.
-func (n *node) decRef(recursive bool, obsolete *[]*FileMetadata) {
+func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
 	if atomic.AddInt32(&n.ref, -1) > 0 {
 		// Other references remain. Can't free.
 		return
 	}
 
-	// Dereference the node's metadata and release child references.
-	if recursive {
+	// Dereference the node's metadata and release child references if
+	// requested. Some internal callers may not want to propagate the deref
+	// because they're manually copying the filemetadata and children to other
+	// nodes, and they want to preserve the existing reference count.
+	if contentsToo {
 		for _, f := range n.items[:n.count] {
-			if atomic.AddInt32(&f.refs, -1) == 0 {
+			if f.Unref() == 0 {
 				// There are two sources of node dereferences: tree mutations
 				// and Version dereferences. Files should only be made obsolete
 				// during Version dereferences, during which `obsolete` will be
@@ -204,7 +220,7 @@ func (n *node) decRef(recursive bool, obsolete *[]*FileMetadata) {
 		}
 		if !n.leaf {
 			for i := int16(0); i <= n.count; i++ {
-				n.children[i].decRef(true /* recursive */, obsolete)
+				n.children[i].decRef(true /* contentsToo */, obsolete)
 			}
 		}
 	}
@@ -222,9 +238,10 @@ func (n *node) clone() *node {
 	// triggering the race detector and looking like a data race.
 	c.count = n.count
 	c.items = n.items
+	c.subtreeCount = n.subtreeCount
 	// Increase the refcount of each contained item.
 	for _, f := range n.items[:n.count] {
-		atomic.AddInt32(&f.refs, 1)
+		f.Ref()
 	}
 	if !c.leaf {
 		// Copy children and increase each refcount.
@@ -236,6 +253,9 @@ func (n *node) clone() *node {
 	return c
 }
 
+// insertAt inserts the provided file and node at the provided index. This
+// function is for use only as a helper function for internal B-Tree code.
+// Clients should not invoke it directly.
 func (n *node) insertAt(index int, item *FileMetadata, nd *node) {
 	if index < int(n.count) {
 		copy(n.items[index+1:n.count+1], n.items[index:n.count])
@@ -250,6 +270,9 @@ func (n *node) insertAt(index int, item *FileMetadata, nd *node) {
 	n.count++
 }
 
+// pushBack inserts the provided file and node at the tail of the node's items.
+// This function is for use only as a helper function for internal B-Tree code.
+// Clients should not invoke it directly.
 func (n *node) pushBack(item *FileMetadata, nd *node) {
 	n.items[n.count] = item
 	if !n.leaf {
@@ -258,6 +281,9 @@ func (n *node) pushBack(item *FileMetadata, nd *node) {
 	n.count++
 }
 
+// pushFront inserts the provided file and node at the head of the
+// node's items. This function is for use only as a helper function for internal B-Tree
+// code. Clients should not invoke it directly.
 func (n *node) pushFront(item *FileMetadata, nd *node) {
 	if !n.leaf {
 		copy(n.children[1:n.count+2], n.children[:n.count+1])
@@ -269,7 +295,8 @@ func (n *node) pushFront(item *FileMetadata, nd *node) {
 }
 
 // removeAt removes a value at a given index, pulling all subsequent values
-// back.
+// back. This function is for use only as a helper function for internal B-Tree
+// code. Clients should not invoke it directly.
 func (n *node) removeAt(index int) (*FileMetadata, *node) {
 	var child *node
 	if !n.leaf {
@@ -284,7 +311,9 @@ func (n *node) removeAt(index int) (*FileMetadata, *node) {
 	return out, child
 }
 
-// popBack removes and returns the last element in the list.
+// popBack removes and returns the last element in the list. This function is
+// for use only as a helper function for internal B-Tree code. Clients should
+// not invoke it directly.
 func (n *node) popBack() (*FileMetadata, *node) {
 	n.count--
 	out := n.items[n.count]
@@ -297,7 +326,9 @@ func (n *node) popBack() (*FileMetadata, *node) {
 	return out, child
 }
 
-// popFront removes and returns the first element in the list.
+// popFront removes and returns the first element in the list. This function is
+// for use only as a helper function for internal B-Tree code. Clients should
+// not invoke it directly.
 func (n *node) popFront() (*FileMetadata, *node) {
 	n.count--
 	var child *node
@@ -315,6 +346,9 @@ func (n *node) popFront() (*FileMetadata, *node) {
 // find returns the index where the given item should be inserted into this
 // list. 'found' is true if the item already exists in the list at the given
 // index.
+//
+// This function is for use only as a helper function for internal B-Tree code.
+// Clients should not invoke it directly.
 func (n *node) find(cmp btreeCmp, item *FileMetadata) (index int, found bool) {
 	// Logic copied from sort.Search. Inlining this gave
 	// an 11% speedup on BenchmarkBTreeDeleteInsert.
@@ -338,23 +372,34 @@ func (n *node) find(cmp btreeCmp, item *FileMetadata) (index int, found bool) {
 // and this function returns the item that existed at that index and a new
 // node containing all items/children after it.
 //
-// Before:
+// split is called when we want to perform a transformation like the one
+// depicted in the following diagram.
 //
-//          +-----------+
-//          |   x y z   |
-//          +--/-/-\-\--+
+//	Before:
+//	                       +-----------+
+//	             n *node   |   x y z   |
+//	                       +--/-/-\-\--+
 //
-// After:
+//	After:
+//	                       +-----------+
+//	                       |     y     |  n's parent
+//	                       +----/-\----+
+//	                           /   \
+//	                          v     v
+//	              +-----------+     +-----------+
+//	      n *node |         x |     | z         | next *node
+//	              +-----------+     +-----------+
 //
-//          +-----------+
-//          |     y     |
-//          +----/-\----+
-//              /   \
-//             v     v
-// +-----------+     +-----------+
-// |         x |     | z         |
-// +-----------+     +-----------+
+// split does not perform the complete transformation; the caller is responsible
+// for updating the parent appropriately. split splits `n` into two nodes, `n`
+// and `next`, returning `next` and the file that separates them. In the diagram
+// above, `n.split` removes y and z from `n`, returning y in the first return
+// value and `next` in the second return value. The caller is responsible for
+// updating n's parent to now contain `y` as the separator between nodes `n` and
+// `next`.
 //
+// This function is for use only as a helper function for internal B-Tree code.
+// Clients should not invoke it directly.
 func (n *node) split(i int) (*FileMetadata, *node) {
 	out := n.items[i]
 	var next *node
@@ -370,17 +415,27 @@ func (n *node) split(i int) (*FileMetadata, *node) {
 	}
 	if !n.leaf {
 		copy(next.children[:], n.children[i+1:n.count+1])
+		descendantsMoved := 0
 		for j := int16(i + 1); j <= n.count; j++ {
+			descendantsMoved += n.children[j].subtreeCount
 			n.children[j] = nil
 		}
+		n.subtreeCount -= descendantsMoved
+		next.subtreeCount += descendantsMoved
 	}
 	n.count = int16(i)
+	// NB: We subtract one more than `next.count` from n's subtreeCount because
+	// the item at index `i` was removed from `n.items`. We'll return the item
+	// at index `i`, and the caller is responsible for updating the subtree
+	// count of whichever node adopts it.
+	n.subtreeCount -= int(next.count) + 1
+	next.subtreeCount += int(next.count)
 	return out, next
 }
 
-// insert inserts a item into the subtree rooted at this node, making sure no
+// Insert inserts a item into the subtree rooted at this node, making sure no
 // nodes in the subtree exceed maxItems items.
-func (n *node) insert(cmp btreeCmp, item *FileMetadata) error {
+func (n *node) Insert(cmp btreeCmp, item *FileMetadata) error {
 	i, found := n.find(cmp, item)
 	if found {
 		// cmp provides a total ordering of the files within a level.
@@ -391,6 +446,7 @@ func (n *node) insert(cmp btreeCmp, item *FileMetadata) error {
 	}
 	if n.leaf {
 		n.insertAt(i, item, nil)
+		n.subtreeCount++
 		return nil
 	}
 	if n.children[i].count >= maxItems {
@@ -410,14 +466,21 @@ func (n *node) insert(cmp btreeCmp, item *FileMetadata) error {
 				errors.Safe(item.FileNum), errors.Safe(n.items[i].FileNum))
 		}
 	}
-	return mut(&n.children[i]).insert(cmp, item)
+
+	err := mut(&n.children[i]).Insert(cmp, item)
+	if err == nil {
+		n.subtreeCount++
+	}
+	return err
 }
 
-// removeMax removes and returns the maximum item from the subtree rooted
-// at this node.
+// removeMax removes and returns the maximum item from the subtree rooted at
+// this node. This function is for use only as a helper function for internal
+// B-Tree code. Clients should not invoke it directly.
 func (n *node) removeMax() *FileMetadata {
 	if n.leaf {
 		n.count--
+		n.subtreeCount--
 		out := n.items[n.count]
 		n.items[n.count] = nil
 		return out
@@ -427,16 +490,18 @@ func (n *node) removeMax() *FileMetadata {
 		n.rebalanceOrMerge(int(n.count))
 		return n.removeMax()
 	}
+	n.subtreeCount--
 	return child.removeMax()
 }
 
-// remove removes a item from the subtree rooted at this node. Returns
+// Remove removes a item from the subtree rooted at this node. Returns
 // the item that was removed or nil if no matching item was found.
-func (n *node) remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) {
+func (n *node) Remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) {
 	i, found := n.find(cmp, item)
 	if n.leaf {
 		if found {
 			out, _ = n.removeAt(i)
+			n.subtreeCount--
 			return out
 		}
 		return nil
@@ -444,22 +509,28 @@ func (n *node) remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) {
 	if n.children[i].count <= minItems {
 		// Child not large enough to remove from.
 		n.rebalanceOrMerge(i)
-		return n.remove(cmp, item)
+		return n.Remove(cmp, item)
 	}
 	child := mut(&n.children[i])
 	if found {
 		// Replace the item being removed with the max item in our left child.
 		out = n.items[i]
 		n.items[i] = child.removeMax()
+		n.subtreeCount--
 		return out
 	}
-	// Latch is not in this node and child is large enough to remove from.
-	out = child.remove(cmp, item)
+	// File is not in this node and child is large enough to remove from.
+	out = child.Remove(cmp, item)
+	if out != nil {
+		n.subtreeCount--
+	}
 	return out
 }
 
-// rebalanceOrMerge grows child 'i' to ensure it has sufficient room to remove
-// a item from it while keeping it at or above minItems.
+// rebalanceOrMerge grows child 'i' to ensure it has sufficient room to remove a
+// item from it while keeping it at or above minItems. This function is for use
+// only as a helper function for internal B-Tree code. Clients should not invoke
+// it directly.
 func (n *node) rebalanceOrMerge(i int) {
 	switch {
 	case i > 0 && n.children[i-1].count > minItems:
@@ -497,6 +568,12 @@ func (n *node) rebalanceOrMerge(i int) {
 		yLa := n.items[i-1]
 		child.pushFront(yLa, grandChild)
 		n.items[i-1] = xLa
+		child.subtreeCount++
+		left.subtreeCount--
+		if grandChild != nil {
+			child.subtreeCount += grandChild.subtreeCount
+			left.subtreeCount -= grandChild.subtreeCount
+		}
 
 	case i < int(n.count) && n.children[i+1].count > minItems:
 		// Rebalance from right sibling.
@@ -532,6 +609,12 @@ func (n *node) rebalanceOrMerge(i int) {
 		xLa, grandChild := right.popFront()
 		yLa := n.items[i]
 		child.pushBack(yLa, grandChild)
+		child.subtreeCount++
+		right.subtreeCount--
+		if grandChild != nil {
+			child.subtreeCount += grandChild.subtreeCount
+			right.subtreeCount -= grandChild.subtreeCount
+		}
 		n.items[i] = xLa
 
 	default:
@@ -570,12 +653,15 @@ func (n *node) rebalanceOrMerge(i int) {
 			copy(child.children[child.count+1:], mergeChild.children[:mergeChild.count+1])
 		}
 		child.count += mergeChild.count + 1
+		child.subtreeCount += mergeChild.subtreeCount + 1
 
-		mergeChild.decRef(false /* recursive */, nil)
+		mergeChild.decRef(false /* contentsToo */, nil)
 	}
 }
 
-func (n *node) invalidateAnnotation(a Annotator) {
+// InvalidateAnnotation removes any existing cached annotations for the provided
+// annotator from this node's subtree.
+func (n *node) InvalidateAnnotation(a Annotator) {
 	// Find this annotator's annotation on this node.
 	var annot *annotation
 	for i := range n.annot {
@@ -590,12 +676,17 @@ func (n *node) invalidateAnnotation(a Annotator) {
 	}
 	if !n.leaf {
 		for i := int16(0); i <= n.count; i++ {
-			n.children[i].invalidateAnnotation(a)
+			n.children[i].InvalidateAnnotation(a)
 		}
 	}
 }
 
-func (n *node) annotation(a Annotator) (interface{}, bool) {
+// Annotation retrieves, computing if not already computed, the provided
+// annotator's annotation of this node. The second return value indicates
+// whether the future reads of this annotation may use the first return value
+// as-is. If false, the annotation is not stable and may change on a subsequent
+// computation.
+func (n *node) Annotation(a Annotator) (interface{}, bool) {
 	// Find this annotator's annotation on this node.
 	var annot *annotation
 	for i := range n.annot {
@@ -627,7 +718,7 @@ func (n *node) annotation(a Annotator) (interface{}, bool) {
 	annot.valid = true
 	for i := int16(0); i <= n.count; i++ {
 		if !n.leaf {
-			v, ok := n.children[i].annotation(a)
+			v, ok := n.children[i].Annotation(a)
 			annot.v = a.Merge(v, annot.v)
 			annot.valid = annot.valid && ok
 		}
@@ -640,6 +731,20 @@ func (n *node) annotation(a Annotator) (interface{}, bool) {
 	return annot.v, annot.valid
 }
 
+func (n *node) verifyInvariants() {
+	recomputedSubtreeCount := int(n.count)
+	if !n.leaf {
+		for i := int16(0); i <= n.count; i++ {
+			n.children[i].verifyInvariants()
+			recomputedSubtreeCount += n.children[i].subtreeCount
+		}
+	}
+	if recomputedSubtreeCount != n.subtreeCount {
+		panic(fmt.Sprintf("recomputed subtree count (%d) ≠ n.subtreeCount (%d)",
+			recomputedSubtreeCount, n.subtreeCount))
+	}
+}
+
 // btree is an implementation of a B-Tree.
 //
 // btree stores FileMetadata in an ordered structure, allowing easy insertion,
@@ -650,25 +755,23 @@ func (n *node) annotation(a Annotator) (interface{}, bool) {
 // Write operations are not safe for concurrent mutation by multiple
 // goroutines, but Read operations are.
 type btree struct {
-	root   *node
-	length int
-	cmp    btreeCmp
+	root *node
+	cmp  btreeCmp
 }
 
-// release dereferences and clears the root node of the btree, removing all
+// Release dereferences and clears the root node of the btree, removing all
 // items from the btree. In doing so, it decrements contained file counts.
 // It returns a slice of newly obsolete files, if any.
-func (t *btree) release() (obsolete []*FileMetadata) {
+func (t *btree) Release() (obsolete []*FileMetadata) {
 	if t.root != nil {
-		t.root.decRef(true /* recursive */, &obsolete)
+		t.root.decRef(true /* contentsToo */, &obsolete)
 		t.root = nil
 	}
-	t.length = 0
 	return obsolete
 }
 
-// clone clones the btree, lazily. It does so in constant time.
-func (t *btree) clone() btree {
+// Clone clones the btree, lazily. It does so in constant time.
+func (t *btree) Clone() btree {
 	c := *t
 	if c.root != nil {
 		// Incrementing the reference count on the root node is sufficient to
@@ -690,15 +793,17 @@ func (t *btree) clone() btree {
 	return c
 }
 
-// delete removes the provided file from the tree.
+// Delete removes the provided file from the tree.
 // It returns true if the file now has a zero reference count.
-func (t *btree) delete(item *FileMetadata) (obsolete bool) {
+func (t *btree) Delete(item *FileMetadata) (obsolete bool) {
 	if t.root == nil || t.root.count == 0 {
 		return false
 	}
-	if out := mut(&t.root).remove(t.cmp, item); out != nil {
-		t.length--
-		obsolete = atomic.AddInt32(&out.refs, -1) == 0
+	if out := mut(&t.root).Remove(t.cmp, item); out != nil {
+		obsolete = out.Unref() == 0
+	}
+	if invariants.Enabled {
+		t.root.verifyInvariants()
 	}
 	if t.root.count == 0 {
 		old := t.root
@@ -707,14 +812,14 @@ func (t *btree) delete(item *FileMetadata) (obsolete bool) {
 		} else {
 			t.root = t.root.children[0]
 		}
-		old.decRef(false /* recursive */, nil)
+		old.decRef(false /* contentsToo */, nil)
 	}
 	return obsolete
 }
 
-// insert adds the given item to the tree. If a item in the tree already
-// equals the given one, insert panics.
-func (t *btree) insert(item *FileMetadata) error {
+// Insert adds the given item to the tree. If a item in the tree already
+// equals the given one, Insert panics.
+func (t *btree) Insert(item *FileMetadata) error {
 	if t.root == nil {
 		t.root = newLeafNode()
 	} else if t.root.count >= maxItems {
@@ -724,39 +829,36 @@ func (t *btree) insert(item *FileMetadata) error {
 		newRoot.items[0] = splitLa
 		newRoot.children[0] = t.root
 		newRoot.children[1] = splitNode
+		newRoot.subtreeCount = t.root.subtreeCount + splitNode.subtreeCount + 1
 		t.root = newRoot
 	}
-	atomic.AddInt32(&item.refs, 1)
-	err := mut(&t.root).insert(t.cmp, item)
-	t.length++
+	item.Ref()
+	err := mut(&t.root).Insert(t.cmp, item)
+	if invariants.Enabled {
+		t.root.verifyInvariants()
+	}
 	return err
 }
 
-// iter returns a new iterator object. It is not safe to continue using an
+// Iter returns a new iterator object. It is not safe to continue using an
 // iterator after modifications are made to the tree. If modifications are made,
 // create a new iterator.
-func (t *btree) iter() iterator {
+func (t *btree) Iter() iterator {
 	return iterator{r: t.root, pos: -1, cmp: t.cmp}
 }
 
-// height returns the height of the tree.
-func (t *btree) height() int {
+// Count returns the number of files contained within the B-Tree.
+func (t *btree) Count() int {
 	if t.root == nil {
 		return 0
 	}
-	h := 1
-	n := t.root
-	for !n.leaf {
-		n = n.children[0]
-		h++
-	}
-	return h
+	return t.root.subtreeCount
 }
 
 // String returns a string description of the tree. The format is
 // similar to the https://en.wikipedia.org/wiki/Newick_format.
 func (t *btree) String() string {
-	if t.length == 0 {
+	if t.Count() == 0 {
 		return ";"
 	}
 	var b strings.Builder
@@ -873,6 +975,8 @@ type iterator struct {
 	r *node
 	// n and pos make up the current position of the iterator.
 	// If valid, n.items[pos] is the current value of the iterator.
+	//
+	// n may be nil iff i.r is nil.
 	n   *node
 	pos int16
 	// cmp dictates the ordering of the FileMetadata.
@@ -883,6 +987,59 @@ type iterator struct {
 	s iterStack
 }
 
+// countLeft returns the count of files that are to the left of the current
+// iterator position.
+func (i *iterator) countLeft() int {
+	if i.r == nil {
+		return 0
+	}
+
+	// Each iterator has a stack of frames marking the path from the root node
+	// to the current iterator position. All files (n.items) and all subtrees
+	// (n.children) with indexes less than [pos] are to the left of the current
+	// iterator position.
+	//
+	//     +------------------------+  -
+	//     |  Root            pos:5 |   |
+	//     +------------------------+   | stack
+	//     |  Root/5          pos:3 |   | frames
+	//     +------------------------+   | [i.s]
+	//     |  Root/5/3        pos:9 |   |
+	//     +========================+  -
+	//     |                        |
+	//     | i.n: Root/5/3/9 i.pos:2|
+	//     +------------------------+
+	//
+	var count int
+	// Walk all the ancestors in the iterator stack [i.s], tallying up all the
+	// files and subtrees to the left of the stack frame's position.
+	f, ok := i.s.nth(0)
+	for fi := 0; ok; fi++ {
+		// There are [f.pos] files contained within [f.n.items] that sort to the
+		// left of the subtree the iterator has descended.
+		count += int(f.pos)
+		// Any subtrees that fall before the stack frame's position are entirely
+		// to the left of the iterator's current position.
+		for j := int16(0); j < f.pos; j++ {
+			count += f.n.children[j].subtreeCount
+		}
+		f, ok = i.s.nth(fi + 1)
+	}
+
+	// The bottommost stack frame is inlined within the iterator struct. Again,
+	// [i.pos] files fall to the left of the current iterator position.
+	count += int(i.pos)
+	if !i.n.leaf {
+		// NB: Unlike above, we use a `<= i.pos` comparison. The iterator is
+		// positioned at item `i.n.items[i.pos]`, which sorts after everything
+		// in the subtree at `i.n.children[i.pos]`.
+		for j := int16(0); j <= i.pos; j++ {
+			count += i.n.children[j].subtreeCount
+		}
+	}
+	return count
+}
+
 func (i *iterator) clone() iterator {
 	c := *i
 	c.s = i.s.clone()
@@ -904,7 +1061,7 @@ func (i iterator) String() string {
 		}
 		fmt.Fprintf(&buf, "%p: %02d/%02d\n", f.n, f.pos, f.n.count)
 	}
-	if i.n == nil {
+	if i.r == nil {
 		fmt.Fprintf(&buf, "<nil>: %02d", i.pos)
 	} else {
 		fmt.Fprintf(&buf, "%p: %02d/%02d", i.n, i.pos, i.n.count)
@@ -1034,7 +1191,7 @@ func (i *iterator) ascend() {
 // tree's files, and then true for the (possibly empty) remainder.
 func (i *iterator) seek(fn func(*FileMetadata) bool) {
 	i.reset()
-	if i.n == nil {
+	if i.r == nil {
 		return
 	}
 
@@ -1066,7 +1223,7 @@ func (i *iterator) seek(fn func(*FileMetadata) bool) {
 // first seeks to the first item in the btree.
 func (i *iterator) first() {
 	i.reset()
-	if i.n == nil {
+	if i.r == nil {
 		return
 	}
 	for !i.n.leaf {
@@ -1078,7 +1235,7 @@ func (i *iterator) first() {
 // last seeks to the last item in the btree.
 func (i *iterator) last() {
 	i.reset()
-	if i.n == nil {
+	if i.r == nil {
 		return
 	}
 	for !i.n.leaf {
@@ -1090,7 +1247,7 @@ func (i *iterator) last() {
 // next positions the iterator to the item immediately following
 // its current position.
 func (i *iterator) next() {
-	if i.n == nil {
+	if i.r == nil {
 		return
 	}
 
@@ -1117,7 +1274,7 @@ func (i *iterator) next() {
 // prev positions the iterator to the item immediately preceding
 // its current position.
 func (i *iterator) prev() {
-	if i.n == nil {
+	if i.r == nil {
 		return
 	}
 
@@ -1148,5 +1305,8 @@ func (i *iterator) valid() bool {
 // cur returns the item at the iterator's current position. It is illegal
 // to call cur if the iterator is not valid.
 func (i *iterator) cur() *FileMetadata {
+	if invariants.Enabled && !i.valid() {
+		panic("btree iterator.cur invoked on invalid iterator")
+	}
 	return i.n.items[i.pos]
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go

@@ -112,13 +112,15 @@ func sortAndSweep(keys []intervalKeyTemp, cmp Compare) []intervalKeyTemp {
 	sorter := intervalKeySorter{keys: keys, cmp: cmp}
 	sort.Sort(sorter)
 
-	// intervalKeys are generated using the file bounds. Specifically, there are 2 intervalKeys
-	// for each file, and len(keys) = 2 * number of files. Each intervalKeyTemp stores information
-	// about which file it was generated from, and whether the key represents the end key of the file.
-	// So, as we're deduplicating the `keys` slice, we're guaranteed to iterate over the interval
-	// keys belonging to each of the files. Since the file.{min,max}IntervalIndex points to the position
-	// of the files bounds in the deduplicated `keys` slice, we can determine file.{min,max}IntervalIndex
-	// during the iteration.
+	// intervalKeys are generated using the file bounds. Specifically, there are
+	// 2 intervalKeys for each file, and len(keys) = 2 * number of files. Each
+	// `intervalKeyTemp` stores information about which file it was generated
+	// from, and whether the key represents the end key of the file. So, as
+	// we're deduplicating the `keys` slice, we're guaranteed to iterate over
+	// the interval keys belonging to each of the files. Since the
+	// file.{min,max}IntervalIndex points to the position of the files bounds in
+	// the deduplicated `keys` slice, we can determine
+	// file.{min,max}IntervalIndex during the iteration.
 	i := 0
 	j := 0
 	for i < len(keys) {
@@ -151,24 +153,24 @@ type fileInterval struct {
 	// cannot have any files participate in L0 -> Lbase compactions.
 	isBaseCompacting bool
 
-	// The min and max intervals index across all the files that overlap with this
-	// interval. Inclusive on both sides.
+	// The min and max intervals index across all the files that overlap with
+	// this interval. Inclusive on both sides.
 	filesMinIntervalIndex int
 	filesMaxIntervalIndex int
 
 	// True if another interval that has a file extending into this interval is
-	// undergoing a compaction into Lbase. In other words, this bool is true
-	// if any interval in [filesMinIntervalIndex,
-	// filesMaxIntervalIndex] has isBaseCompacting set to true. This
-	// lets the compaction picker de-prioritize this interval for picking
-	// compactions, since there's a high chance that a base compaction with a
-	// sufficient height of sublevels rooted at this interval could not be
-	// chosen due to the ongoing base compaction in the
-	// other interval. If the file straddling the two intervals is at a
-	// sufficiently high sublevel (with enough compactible files below it to
-	// satisfy minCompactionDepth), this is not an issue, but to optimize for
-	// quickly picking base compactions far away from other base compactions,
-	// this bool is used as a heuristic (but not as a complete disqualifier).
+	// undergoing a compaction into Lbase. In other words, this bool is true if
+	// any interval in [filesMinIntervalIndex, filesMaxIntervalIndex] has
+	// isBaseCompacting set to true. This lets the compaction picker
+	// de-prioritize this interval for picking compactions, since there's a high
+	// chance that a base compaction with a sufficient height of sublevels
+	// rooted at this interval could not be chosen due to the ongoing base
+	// compaction in the other interval. If the file straddling the two
+	// intervals is at a sufficiently high sublevel (with enough compactible
+	// files below it to satisfy minCompactionDepth), this is not an issue, but
+	// to optimize for quickly picking base compactions far away from other base
+	// compactions, this bool is used as a heuristic (but not as a complete
+	// disqualifier).
 	intervalRangeIsBaseCompacting bool
 
 	// All files in this interval, in increasing sublevel order.
@@ -178,9 +180,8 @@ type fileInterval struct {
 	// starting new compactions. This metric is not precise since the
 	// compactingFileCount can include files that are part of N (where N > 1)
 	// intra-L0 compactions, so the stack depth after those complete will be
-	// len(files) - compactingFileCount + N. We ignore this imprecision since
-	// we don't want to track which files are part of which intra-L0
-	// compaction.
+	// len(files) - compactingFileCount + N. We ignore this imprecision since we
+	// don't want to track which files are part of which intra-L0 compaction.
 	compactingFileCount int
 
 	// Interpolated from files in this interval. For files spanning multiple
@@ -332,17 +333,17 @@ func NewL0Sublevels(
 	for _, sublevelFiles := range s.levelFiles {
 		tr, ls := makeBTree(btreeCmpSmallestKey(cmp), sublevelFiles)
 		s.Levels = append(s.Levels, ls)
-		tr.release()
+		tr.Release()
 	}
 
 	s.calculateFlushSplitKeys(flushSplitMaxBytes)
 	return s, nil
 }
 
-// Helper function to merge new intervalKeys into an existing slice
-// of old fileIntervals, into result. Returns the new result and a slice of ints
-// mapping old interval indices to new ones. The added intervalKeys do not
-// need to be sorted; they get sorted and deduped in this function.
+// Helper function to merge new intervalKeys into an existing slice of old
+// fileIntervals, into result. Returns the new result and a slice of ints
+// mapping old interval indices to new ones. The added intervalKeys do not need
+// to be sorted; they get sorted and deduped in this function.
 func mergeIntervals(
 	old, result []fileInterval, added []intervalKeyTemp, compare Compare,
 ) ([]fileInterval, []int) {
@@ -413,35 +414,36 @@ func mergeIntervals(
 	return result, oldToNewMap
 }
 
-// AddL0Files incrementally builds a new L0Sublevels for when the only
-// change since the receiver L0Sublevels was an addition of the specified files,
-// with no L0 deletions. The common case of this is an ingestion or a flush.
-// These files can "sit on top" of existing sublevels, creating at most one
-// new sublevel for a flush (and possibly multiple for an ingestion), and at
-// most 2*len(files) additions to s.orderedIntervals. No files must have been
-// deleted from L0, and the added files must all be newer in sequence numbers
-// than existing files in L0Sublevels. The files parameter must be sorted in
-// seqnum order. The levelMetadata parameter corresponds to the new L0 post
-// addition of files. This method is meant to be significantly more performant
-// than NewL0Sublevels.
+// AddL0Files incrementally builds a new L0Sublevels for when the only change
+// since the receiver L0Sublevels was an addition of the specified files, with
+// no L0 deletions. The common case of this is an ingestion or a flush. These
+// files can "sit on top" of existing sublevels, creating at most one new
+// sublevel for a flush (and possibly multiple for an ingestion), and at most
+// 2*len(files) additions to s.orderedIntervals. No files must have been deleted
+// from L0, and the added files must all be newer in sequence numbers than
+// existing files in L0Sublevels. The files parameter must be sorted in seqnum
+// order. The levelMetadata parameter corresponds to the new L0 post addition of
+// files. This method is meant to be significantly more performant than
+// NewL0Sublevels.
 //
 // Note that this function can only be called once on a given receiver; it
 // appends to some slices in s which is only safe when done once. This is okay,
 // as the common case (generating a new L0Sublevels after a flush/ingestion) is
 // only going to necessitate one call of this method on a given receiver. The
-// returned value, if non-nil, can then have AddL0Files called on it again, and
-// so on. If errInvalidL0SublevelsOpt is returned as an error, it likely means
-// the optimization could not be applied (i.e. files added were older than files
-// already in the sublevels, which is possible around ingestions and in tests).
-// Eg. it can happen when an ingested file was ingested without queueing a flush
-// since it did not actually overlap with any keys in the memtable. Later on the
-// memtable was flushed, and the memtable had keys spanning around the ingested
-// file, producing a flushed file that overlapped with the ingested file in file
-// bounds but not in keys. It's possible for that flushed file to have a lower
-// LargestSeqNum than the ingested file if all the additions after the ingestion
-// were to another flushed file that was split into a separate sstable during
-// flush. Any other non-nil error means L0Sublevels generation failed in the same
-// way as NewL0Sublevels would likely fail.
+// returned value, if non-nil, can then have [*L0Sublevels.AddL0Files] called on
+// it again, and so on. If [errInvalidL0SublevelsOpt] is returned as an error,
+// it likely means the optimization could not be applied (i.e. files added were
+// older than files already in the sublevels, which is possible around
+// ingestions and in tests). Eg. it can happen when an ingested file was
+// ingested without queueing a flush since it did not actually overlap with any
+// keys in the memtable. Later on the memtable was flushed, and the memtable had
+// keys spanning around the ingested file, producing a flushed file that
+// overlapped with the ingested file in file bounds but not in keys. It's
+// possible for that flushed file to have a lower LargestSeqNum than the
+// ingested file if all the additions after the ingestion were to another
+// flushed file that was split into a separate sstable during flush. Any other
+// non-nil error means [L0Sublevels] generation failed in the same way as
+// [NewL0Sublevels] would likely fail.
 func (s *L0Sublevels) AddL0Files(
 	files []*FileMetadata, flushSplitMaxBytes int64, levelMetadata *LevelMetadata,
 ) (*L0Sublevels, error) {
@@ -485,8 +487,8 @@ func (s *L0Sublevels) AddL0Files(
 	keys := make([]fileInterval, 0, 2*levelMetadata.Len())
 	var oldToNewMap []int
 	// We can avoid the sortAndSweep step on the combined length of
-	// s.orderedIntervals and fileKeys by treating this as a merge of two
-	// sorted runs, fileKeys and s.orderedIntervals, into `keys` which will form
+	// s.orderedIntervals and fileKeys by treating this as a merge of two sorted
+	// runs, fileKeys and s.orderedIntervals, into `keys` which will form
 	// newVal.orderedIntervals.
 	keys, oldToNewMap = mergeIntervals(s.orderedIntervals, keys, fileKeys, s.cmp)
 	if invariants.Enabled {
@@ -502,20 +504,21 @@ func (s *L0Sublevels) AddL0Files(
 		newInterval := &keys[newIdx]
 		newInterval.index = newIdx
 		// This code, and related code in the for loop below, adjusts
-		// files{Min,Max}IntervalIndex just for interval indices shifting due to new
-		// intervals, and not for any of the new files being added to the same
-		// intervals. The goal is to produce a state of the system that's accurate
-		// for all existing files, and has all the new intervals to support new
-		// files. Once that's done, we can just call addFileToSublevel to adjust
-		// all relevant intervals for new files.
+		// files{Min,Max}IntervalIndex just for interval indices shifting due to
+		// new intervals, and not for any of the new files being added to the
+		// same intervals. The goal is to produce a state of the system that's
+		// accurate for all existing files, and has all the new intervals to
+		// support new files. Once that's done, we can just call
+		// addFileToSublevel to adjust all relevant intervals for new files.
 		newInterval.filesMinIntervalIndex = oldToNewMap[newInterval.filesMinIntervalIndex]
 		// maxIntervalIndexes are special. Since it's an inclusive end bound, we
 		// actually have to map it to the _next_ old interval's new previous
 		// interval. This logic is easier to understand if you see
 		// [f.minIntervalIndex, f.maxIntervalIndex] as [f.minIntervalIndex,
-		// f.maxIntervalIndex+1). The other case to remember is when the interval is
-		// completely empty (i.e. len(newInterval.files) == 0); in that case we want
-		// to refer back to ourselves regardless of additions to the right of us.
+		// f.maxIntervalIndex+1). The other case to remember is when the
+		// interval is completely empty (i.e. len(newInterval.files) == 0); in
+		// that case we want to refer back to ourselves regardless of additions
+		// to the right of us.
 		if newInterval.filesMaxIntervalIndex < len(oldToNewMap)-1 && len(newInterval.files) > 0 {
 			newInterval.filesMaxIntervalIndex = oldToNewMap[newInterval.filesMaxIntervalIndex+1] - 1
 		} else {
@@ -523,16 +526,16 @@ func (s *L0Sublevels) AddL0Files(
 			newInterval.filesMaxIntervalIndex = oldToNewMap[newInterval.filesMaxIntervalIndex]
 		}
 	}
-	// Loop through all instances of new intervals added between two old intervals
-	// and expand [filesMinIntervalIndex, filesMaxIntervalIndex] of new intervals
-	// to reflect that of adjacent old intervals.
+	// Loop through all instances of new intervals added between two old
+	// intervals and expand [filesMinIntervalIndex, filesMaxIntervalIndex] of
+	// new intervals to reflect that of adjacent old intervals.
 	{
 		// We can skip cases where new intervals were added to the left of all
 		// existing intervals (eg. if the first entry in oldToNewMap is
-		// oldToNewMap[0] >= 1). Those intervals will only contain newly added files
-		// and will have their parameters adjusted down in addFileToSublevels. The
-		// same can also be said about new intervals that are to the right of all
-		// existing intervals.
+		// oldToNewMap[0] >= 1). Those intervals will only contain newly added
+		// files and will have their parameters adjusted down in
+		// addFileToSublevels. The same can also be said about new intervals
+		// that are to the right of all existing intervals.
 		lastIdx := 0
 		for _, newIdx := range oldToNewMap {
 			for i := lastIdx + 1; i < newIdx; i++ {
@@ -555,28 +558,29 @@ func (s *L0Sublevels) AddL0Files(
 	// TODO(bilal): This is the only place in this method where we loop through
 	// all existing files, which could be much more in number than newly added
 	// files. See if we can avoid the need for this, either by getting rid of
-	// f.minIntervalIndex and f.maxIntervalIndex and calculating them on the
-	// fly with a binary search, or by only looping through files to the right
-	// of the first interval touched by this method.
+	// f.minIntervalIndex and f.maxIntervalIndex and calculating them on the fly
+	// with a binary search, or by only looping through files to the right of
+	// the first interval touched by this method.
 	for sublevel := range s.Levels {
 		s.Levels[sublevel].Each(func(f *FileMetadata) {
 			oldIntervalDelta := f.maxIntervalIndex - f.minIntervalIndex + 1
 			oldMinIntervalIndex := f.minIntervalIndex
 			f.minIntervalIndex = oldToNewMap[f.minIntervalIndex]
-			// maxIntervalIndex is special. Since it's an inclusive end bound, we
-			// actually have to map it to the _next_ old interval's new previous
-			// interval. This logic is easier to understand if you see
-			// [f.minIntervalIndex, f.maxIntervalIndex] as
-			// [f.minIntervalIndex, f.maxIntervalIndex+1).
+			// maxIntervalIndex is special. Since it's an inclusive end bound,
+			// we actually have to map it to the _next_ old interval's new
+			// previous interval. This logic is easier to understand if you see
+			// [f.minIntervalIndex, f.maxIntervalIndex] as [f.minIntervalIndex,
+			// f.maxIntervalIndex+1).
 			f.maxIntervalIndex = oldToNewMap[f.maxIntervalIndex+1] - 1
 			newIntervalDelta := f.maxIntervalIndex - f.minIntervalIndex + 1
-			// Recalculate estimatedBytes for all old files across new intervals, but
-			// only if new intervals were added in between.
+			// Recalculate estimatedBytes for all old files across new
+			// intervals, but only if new intervals were added in between.
 			if oldIntervalDelta != newIntervalDelta {
-				// j is incremented so that oldToNewMap[j] points to the next old
-				// interval. This is used to distinguish between old intervals (i.e.
-				// ones where we need to subtract f.Size/oldIntervalDelta) from new
-				// ones (where we don't need to subtract). In both cases we need to add
+				// j is incremented so that oldToNewMap[j] points to the next
+				// old interval. This is used to distinguish between old
+				// intervals (i.e. ones where we need to subtract
+				// f.Size/oldIntervalDelta) from new ones (where we don't need
+				// to subtract). In both cases we need to add
 				// f.Size/newIntervalDelta.
 				j := oldMinIntervalIndex
 				for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
@@ -628,7 +632,7 @@ func (s *L0Sublevels) AddL0Files(
 			// populated correctly.
 			newVal.Levels[sublevel] = ls
 		}
-		tr.release()
+		tr.Release()
 	}
 
 	newVal.flushSplitUserKeys = nil
@@ -636,12 +640,12 @@ func (s *L0Sublevels) AddL0Files(
 	return newVal, nil
 }
 
-// addFileToSublevels is called during L0Sublevels generation, and adds f to
-// the correct sublevel's levelFiles, the relevant intervals' files slices, and
-// sets interval indices on f. This method, if called successively on multiple
-// files, _must_ be called on successively newer files (by seqnum). If
-// checkInvariant is true, it could check for this in some cases and return
-// errInvalidL0SublevelsOpt if that invariant isn't held.
+// addFileToSublevels is called during L0Sublevels generation, and adds f to the
+// correct sublevel's levelFiles, the relevant intervals' files slices, and sets
+// interval indices on f. This method, if called successively on multiple files,
+// _must_ be called on successively newer files (by seqnum). If checkInvariant
+// is true, it could check for this in some cases and return
+// [errInvalidL0SublevelsOpt] if that invariant isn't held.
 func (s *L0Sublevels) addFileToSublevels(f *FileMetadata, checkInvariant bool) error {
 	// This is a simple and not very accurate estimate of the number of
 	// bytes this SSTable contributes to the intervals it is a part of.
@@ -654,14 +658,18 @@ func (s *L0Sublevels) addFileToSublevels(f *FileMetadata, checkInvariant bool) e
 	// Update state in every fileInterval for this file.
 	for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
 		interval := &s.orderedIntervals[i]
-		if len(interval.files) > 0 &&
-			subLevel <= interval.files[len(interval.files)-1].SubLevel {
+		if len(interval.files) > 0 {
 			if checkInvariant && interval.files[len(interval.files)-1].LargestSeqNum > f.LargestSeqNum {
 				// We are sliding this file "underneath" an existing file. Throw away
 				// and start over in NewL0Sublevels.
 				return errInvalidL0SublevelsOpt
 			}
-			subLevel = interval.files[len(interval.files)-1].SubLevel + 1
+			// interval.files is sorted by sublevels, from lowest to highest.
+			// AddL0Files can only add files at sublevels higher than existing files
+			// in the same key intervals.
+			if maxSublevel := interval.files[len(interval.files)-1].SubLevel; subLevel <= maxSublevel {
+				subLevel = maxSublevel + 1
+			}
 		}
 		interval.estimatedBytes += interpolatedBytes
 		if f.minIntervalIndex < interval.filesMinIntervalIndex {
@@ -738,11 +746,10 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
 		}
 	}
 
-	// Some intervals may be base compacting without the files contained
-	// within those intervals being marked as compacting. This is possible if
-	// the files were added after the compaction initiated, and the active
-	// compaction files straddle the input file. Mark these intervals as base
-	// compacting.
+	// Some intervals may be base compacting without the files contained within
+	// those intervals being marked as compacting. This is possible if the files
+	// were added after the compaction initiated, and the active compaction
+	// files straddle the input file. Mark these intervals as base compacting.
 	for _, c := range inProgress {
 		startIK := intervalKey{key: c.Smallest.UserKey, isLargest: false}
 		endIK := intervalKey{key: c.Largest.UserKey, isLargest: !c.Largest.IsExclusiveSentinel()}
@@ -924,35 +931,35 @@ func (s *L0Sublevels) InUseKeyRanges(smallest, largest []byte) []UserKeyRange {
 			curr = &keyRanges[len(keyRanges)-1]
 		}
 
-		// If the filesMaxIntervalIndex is not the current index, we can jump
-		// to the max index, knowing that all intermediary intervals are
-		// overlapped by some file.
+		// If the filesMaxIntervalIndex is not the current index, we can jump to
+		// the max index, knowing that all intermediary intervals are overlapped
+		// by some file.
 		if maxIdx := s.orderedIntervals[i].filesMaxIntervalIndex; maxIdx != i {
 			// Note that end may be less than or equal to maxIdx if we're
 			// concerned with a key range that ends before the interval at
-			// maxIdx starts. We must set curr.End now, before making that
-			// leap, because this iteration may be the last.
+			// maxIdx starts. We must set curr.End now, before making that leap,
+			// because this iteration may be the last.
 			i = maxIdx
 			curr.End = s.orderedIntervals[i+1].startKey.key
 			continue
 		}
 
 		// No files overlapping with this interval overlap with the next
-		// interval. Update the current end to be the next interval's start
-		// key. Note that curr is not necessarily finished, because there may
-		// be an abutting non-empty interval.
+		// interval. Update the current end to be the next interval's start key.
+		// Note that curr is not necessarily finished, because there may be an
+		// abutting non-empty interval.
 		curr.End = s.orderedIntervals[i+1].startKey.key
 		i++
 	}
 	return keyRanges
 }
 
-// FlushSplitKeys returns a slice of user keys to split flushes at.
-// Used by flushes to avoid writing sstables that straddle these split keys.
-// These should be interpreted as the keys to start the next sstable (not the
-// last key to include in the prev sstable). These are user keys so that
-// range tombstones can be properly truncated (untruncated range tombstones
-// are not permitted for L0 files).
+// FlushSplitKeys returns a slice of user keys to split flushes at. Used by
+// flushes to avoid writing sstables that straddle these split keys. These
+// should be interpreted as the keys to start the next sstable (not the last key
+// to include in the prev sstable). These are user keys so that range tombstones
+// can be properly truncated (untruncated range tombstones are not permitted for
+// L0 files).
 func (s *L0Sublevels) FlushSplitKeys() [][]byte {
 	return s.flushSplitUserKeys
 }
@@ -978,6 +985,7 @@ func (s *L0Sublevels) MaxDepthAfterOngoingCompactions() int {
 //
 // TODO(bilal): Simplify away the debugging statements in this method, and make
 // this a pure sanity checker.
+//
 //lint:ignore U1000 - useful for debugging
 func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error {
 	includedFiles := newBitSet(s.levelMetadata.Len())
@@ -1074,7 +1082,7 @@ func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error {
 // recently started compaction. isBase specifies if this is a base compaction;
 // if false, this is assumed to be an intra-L0 compaction. The specified
 // compaction must be involving L0 SSTables. It's assumed that the Compacting
-// and IsIntraL0Compacting fields are already set on all FileMetadatas passed
+// and IsIntraL0Compacting fields are already set on all [FileMetadata]s passed
 // in.
 func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs []LevelSlice, isBase bool) error {
 	minIntervalIndex := -1
@@ -1106,11 +1114,10 @@ func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs []LevelSlice, isBas
 	return nil
 }
 
-// L0CompactionFiles represents a candidate set of L0 files for compaction.
-// Also referred to as "lcf". Contains state information useful
-// for generating the compaction (such as Files), as well as for picking
-// between candidate compactions (eg. fileBytes and
-// seedIntervalStackDepthReduction).
+// L0CompactionFiles represents a candidate set of L0 files for compaction. Also
+// referred to as "lcf". Contains state information useful for generating the
+// compaction (such as Files), as well as for picking between candidate
+// compactions (eg. fileBytes and seedIntervalStackDepthReduction).
 type L0CompactionFiles struct {
 	Files []*FileMetadata
 
@@ -1146,6 +1153,23 @@ type L0CompactionFiles struct {
 	filesAdded              []*FileMetadata
 }
 
+// Clone allocates a new L0CompactionFiles, with the same underlying data. Note
+// that the two fileMetadata slices contain values that point to the same
+// underlying fileMetadata object. This is safe because these objects are read
+// only.
+func (l *L0CompactionFiles) Clone() *L0CompactionFiles {
+	oldLcf := *l
+	return &oldLcf
+}
+
+// String merely prints the starting address of the first file, if it exists.
+func (l *L0CompactionFiles) String() string {
+	if len(l.Files) > 0 {
+		return fmt.Sprintf("First File Address: %p", &l.Files[0])
+	}
+	return ""
+}
+
 // addFile adds the specified file to the LCF.
 func (l *L0CompactionFiles) addFile(f *FileMetadata) {
 	if l.FilesIncluded[f.L0Index] {
@@ -1180,12 +1204,11 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
 
 // Compactions:
 //
-// The sub-levels and intervals can be visualized in 2 dimensions as the X
-// axis containing intervals in increasing order and the Y axis containing
-// sub-levels (older to younger). The intervals can be sparse wrt sub-levels.
-// We observe that the system is typically under severe pressure in L0 during
-// large numbers of ingestions where most files added to L0 are narrow and
-// non-overlapping.
+// The sub-levels and intervals can be visualized in 2 dimensions as the X axis
+// containing intervals in increasing order and the Y axis containing sub-levels
+// (older to younger). The intervals can be sparse wrt sub-levels. We observe
+// that the system is typically under severe pressure in L0 during large numbers
+// of ingestions where most files added to L0 are narrow and non-overlapping.
 //
 //    L0.1    d---g
 //    L0.0  c--e  g--j o--s u--x
@@ -1203,23 +1226,23 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
 // it amenable to concurrent L0 -> Lbase compactions.
 //
 // L0 -> Lbase: The high-level goal of a L0 -> Lbase compaction is to reduce
-// stack depth, by compacting files in the intervals with the highest
-// (fileCount - compactingCount). Additionally, we would like compactions to
-// not involve a huge number of files, so that they finish quickly, and to
-// allow for concurrent L0 -> Lbase compactions when needed. In order to
-// achieve these goals we would like compactions to visualize as capturing
-// thin and tall rectangles. The approach below is to consider intervals in
-// some order and then try to construct a compaction using the interval. The
-// first interval we can construct a compaction for is the compaction that is
-// started. There can be multiple heuristics in choosing the ordering of the
-// intervals -- the code uses one heuristic that worked well for a large
-// ingestion stemming from a cockroachdb import, but additional experimentation
-// is necessary to pick a general heuristic. Additionally, the compaction that
-// gets picked may be not as desirable as one that could be constructed later
-// in terms of reducing stack depth (since adding more files to the compaction
-// can get blocked by needing to encompass files that are already being
-// compacted). So an alternative would be to try to construct more than one
-// compaction and pick the best one.
+// stack depth, by compacting files in the intervals with the highest (fileCount
+// - compactingCount). Additionally, we would like compactions to not involve a
+// huge number of files, so that they finish quickly, and to allow for
+// concurrent L0 -> Lbase compactions when needed. In order to achieve these
+// goals we would like compactions to visualize as capturing thin and tall
+// rectangles. The approach below is to consider intervals in some order and
+// then try to construct a compaction using the interval. The first interval we
+// can construct a compaction for is the compaction that is started. There can
+// be multiple heuristics in choosing the ordering of the intervals -- the code
+// uses one heuristic that worked well for a large ingestion stemming from a
+// cockroachdb import, but additional experimentation is necessary to pick a
+// general heuristic. Additionally, the compaction that gets picked may be not
+// as desirable as one that could be constructed later in terms of reducing
+// stack depth (since adding more files to the compaction can get blocked by
+// needing to encompass files that are already being compacted). So an
+// alternative would be to try to construct more than one compaction and pick
+// the best one.
 //
 // Here's a visualization of an ideal L0->LBase compaction selection:
 //
@@ -1243,9 +1266,9 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
 //    Lbase a--------i    m---------w
 //
 // Note that running this compaction will mark the a--i file in Lbase as
-// compacting, and when ExtendL0ForBaseCompactionTo is called with the bounds
-// of that base file, it'll expand the compaction to also include all L0 files
-// in the a-d interval. The resultant compaction would then be:
+// compacting, and when ExtendL0ForBaseCompactionTo is called with the bounds of
+// that base file, it'll expand the compaction to also include all L0 files in
+// the a-d interval. The resultant compaction would then be:
 //
 //         _____________
 //    L0.3 |a--d    g-j|
@@ -1255,11 +1278,10 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
 //
 //    Lbase a--------i    m---------w
 //
-// The next best interval for base compaction would therefore
-// be the one including r--t in L0.2 and p--x in L0.0, and both this compaction
-// and the one picked earlier can run in parallel. This is assuming
-// minCompactionDepth >= 2, otherwise the second compaction has too little
-// depth to pick.
+// The next best interval for base compaction would therefore be the one
+// including r--t in L0.2 and p--x in L0.0, and both this compaction and the one
+// picked earlier can run in parallel. This is assuming minCompactionDepth >= 2,
+// otherwise the second compaction has too little depth to pick.
 //
 //         _____________
 //    L0.3 |a--d    g-j|      _________
@@ -1292,24 +1314,23 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
 //
 //    Lbase a--------ij--lm---------w
 //
-// Intra-L0: If the L0 score is high, but PickBaseCompaction() is unable to
-// pick a compaction, PickIntraL0Compaction will be used to pick an intra-L0
-// compaction. Similar to L0 -> Lbase compactions, we want to allow for
-// multiple intra-L0 compactions and not generate wide output files that
-// hinder later concurrency of L0 -> Lbase compactions. Also compactions
-// that produce wide files don't reduce stack depth -- they represent wide
-// rectangles in our visualization, which means many intervals have their
-// depth reduced by a small amount. Typically, L0 files have non-overlapping
-// sequence numbers, and sticking to that invariant would require us to
-// consider intra-L0 compactions that proceed from youngest to oldest files,
-// which could result in the aforementioned undesirable wide rectangle
-// shape. But this non-overlapping sequence number is already relaxed in
-// RocksDB -- sstables are primarily ordered by their largest sequence
-// number. So we can arrange for intra-L0 compactions to capture thin and
-// tall rectangles starting with the top of the stack (youngest files).
-// Like the L0 -> Lbase case we order the intervals using a heuristic and
-// consider each in turn. The same comment about better L0 -> Lbase heuristics
-// and not being greedy applies here.
+// Intra-L0: If the L0 score is high, but PickBaseCompaction() is unable to pick
+// a compaction, PickIntraL0Compaction will be used to pick an intra-L0
+// compaction. Similar to L0 -> Lbase compactions, we want to allow for multiple
+// intra-L0 compactions and not generate wide output files that hinder later
+// concurrency of L0 -> Lbase compactions. Also compactions that produce wide
+// files don't reduce stack depth -- they represent wide rectangles in our
+// visualization, which means many intervals have their depth reduced by a small
+// amount. Typically, L0 files have non-overlapping sequence numbers, and
+// sticking to that invariant would require us to consider intra-L0 compactions
+// that proceed from youngest to oldest files, which could result in the
+// aforementioned undesirable wide rectangle shape. But this non-overlapping
+// sequence number is already relaxed in RocksDB -- sstables are primarily
+// ordered by their largest sequence number. So we can arrange for intra-L0
+// compactions to capture thin and tall rectangles starting with the top of the
+// stack (youngest files). Like the L0 -> Lbase case we order the intervals
+// using a heuristic and consider each in turn. The same comment about better L0
+// -> Lbase heuristics and not being greedy applies here.
 //
 // Going back to a modified version of our example from earlier, let's say these
 // are the base compactions in progress:
@@ -1387,10 +1408,9 @@ func (s *L0Sublevels) PickBaseCompaction(
 		// Pick the seed file for the interval as the file
 		// in the lowest sub-level.
 		f := interval.files[0]
-		// Don't bother considering the intervals that are
-		// covered by the seed file since they are likely
-		// nearby. Note that it is possible that those intervals
-		// have seed files at lower sub-levels so could be
+		// Don't bother considering the intervals that are covered by the seed
+		// file since they are likely nearby. Note that it is possible that
+		// those intervals have seed files at lower sub-levels so could be
 		// viable for compaction.
 		if f == nil {
 			return nil, errors.New("no seed file found in sublevel intervals")
@@ -1398,23 +1418,22 @@ func (s *L0Sublevels) PickBaseCompaction(
 		consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1)
 		if f.IsCompacting() {
 			if f.IsIntraL0Compacting {
-				// If we're picking a base compaction and we came across a
-				// seed file candidate that's being intra-L0 compacted, skip
-				// the interval instead of erroring out.
+				// If we're picking a base compaction and we came across a seed
+				// file candidate that's being intra-L0 compacted, skip the
+				// interval instead of erroring out.
 				continue
 			}
-			// We chose a compaction seed file that should not be
-			// compacting. Usually means the score is not accurately
-			// accounting for files already compacting, or internal state is
-			// inconsistent.
+			// We chose a compaction seed file that should not be compacting.
+			// Usually means the score is not accurately accounting for files
+			// already compacting, or internal state is inconsistent.
 			return nil, errors.Errorf("file %s chosen as seed file for compaction should not be compacting", f.FileNum)
 		}
 
 		c := s.baseCompactionUsingSeed(f, interval.index, minCompactionDepth)
 		if c != nil {
-			// Check if the chosen compaction overlaps with any files
-			// in Lbase that have Compacting = true. If that's the case,
-			// this compaction cannot be chosen.
+			// Check if the chosen compaction overlaps with any files in Lbase
+			// that have Compacting = true. If that's the case, this compaction
+			// cannot be chosen.
 			baseIter := baseFiles.Iter()
 			// An interval starting at ImmediateSuccessor(key) can never be the
 			// first interval of a compaction since no file can start at that
@@ -1454,10 +1473,9 @@ func (s *L0Sublevels) baseCompactionUsingSeed(
 	c.addFile(f)
 
 	// The first iteration of this loop builds the compaction at the seed file's
-	// sublevel. Future iterations expand on this compaction by stacking
-	// more files from intervalIndex and repeating. This is an
-	// optional activity so when it fails we can fallback to the last
-	// successful candidate.
+	// sublevel. Future iterations expand on this compaction by stacking more
+	// files from intervalIndex and repeating. This is an optional activity so
+	// when it fails we can fallback to the last successful candidate.
 	var lastCandidate *L0CompactionFiles
 	interval := &s.orderedIntervals[intervalIndex]
 
@@ -1467,29 +1485,28 @@ func (s *L0Sublevels) baseCompactionUsingSeed(
 		c.seedIntervalStackDepthReduction++
 		c.seedIntervalMaxLevel = sl
 		c.addFile(f2)
-		// The seed file is in the lowest sublevel in the seed interval, but it may
-		// overlap with other files in even lower sublevels. For
-		// correctness we need to grow our interval to include those files, and
-		// capture all files in the next level that fall in this extended interval
-		// and so on. This can result in a triangular shape like the following
-		// where again the X axis is the key intervals and the Y axis
-		// is oldest to youngest. Note that it is not necessary for
-		// correctness to fill out the shape at the higher sub-levels
-		// to make it more rectangular since the invariant only requires
-		// that younger versions of a key not be moved to Lbase while
-		// leaving behind older versions.
+		// The seed file is in the lowest sublevel in the seed interval, but it
+		// may overlap with other files in even lower sublevels. For correctness
+		// we need to grow our interval to include those files, and capture all
+		// files in the next level that fall in this extended interval and so
+		// on. This can result in a triangular shape like the following where
+		// again the X axis is the key intervals and the Y axis is oldest to
+		// youngest. Note that it is not necessary for correctness to fill out
+		// the shape at the higher sub-levels to make it more rectangular since
+		// the invariant only requires that younger versions of a key not be
+		// moved to Lbase while leaving behind older versions.
 		//                     -
 		//                    ---
 		//                   -----
-		// It may be better for performance to have a more rectangular
-		// shape since the files being left behind will overlap with the
-		// same Lbase key range as that of this compaction. But there is
-		// also the danger that in trying to construct a more rectangular
-		// shape we will be forced to pull in a file that is already
-		// compacting. We expect extendCandidateToRectangle to eventually be called
-		// on this compaction if it's chosen, at which point we would iterate
-		// backward and choose those files. This logic is similar to compaction.grow
-		// for non-L0 compactions.
+		// It may be better for performance to have a more rectangular shape
+		// since the files being left behind will overlap with the same Lbase
+		// key range as that of this compaction. But there is also the danger
+		// that in trying to construct a more rectangular shape we will be
+		// forced to pull in a file that is already compacting. We expect
+		// extendCandidateToRectangle to eventually be called on this compaction
+		// if it's chosen, at which point we would iterate backward and choose
+		// those files. This logic is similar to compaction.grow for non-L0
+		// compactions.
 		done := false
 		for currLevel := sl - 1; currLevel >= 0; currLevel-- {
 			if !s.extendFiles(currLevel, math.MaxUint64, c) {
@@ -1509,8 +1526,8 @@ func (s *L0Sublevels) baseCompactionUsingSeed(
 		// depth, but long running compactions reduce flexibility in what can
 		// run concurrently in L0 and even Lbase -> Lbase+1. An increase more
 		// than 150% in bytes since the last candidate compaction (along with a
-		// total compaction size in excess of 100mb), or a total compaction
-		// size beyond a hard limit of 500mb, is criteria for rejecting this
+		// total compaction size in excess of 100mb), or a total compaction size
+		// beyond a hard limit of 500mb, is criteria for rejecting this
 		// candidate. This lets us prefer slow growths as we add files, while
 		// still having a hard limit. Note that if this is the first compaction
 		// candidate to reach a stack depth reduction of minCompactionDepth or
@@ -1568,7 +1585,7 @@ func (s *L0Sublevels) extendFiles(
 
 // PickIntraL0Compaction picks an intra-L0 compaction for files in this
 // sublevel. This method is only called when a base compaction cannot be chosen.
-// See comment above PickBaseCompaction for heuristics involved in this
+// See comment above [PickBaseCompaction] for heuristics involved in this
 // selection.
 func (s *L0Sublevels) PickIntraL0Compaction(
 	earliestUnflushedSeqNum uint64, minCompactionDepth int,
@@ -1584,9 +1601,8 @@ func (s *L0Sublevels) PickIntraL0Compaction(
 	}
 	sort.Sort(intervalSorterByDecreasingScore(scoredIntervals))
 
-	// Optimization to avoid considering different intervals that
-	// are likely to choose the same seed file. Again this is just
-	// to reduce wasted work.
+	// Optimization to avoid considering different intervals that are likely to
+	// choose the same seed file. Again this is just to reduce wasted work.
 	consideredIntervals := newBitSet(len(s.orderedIntervals))
 	for _, scoredInterval := range scoredIntervals {
 		interval := &s.orderedIntervals[scoredInterval.interval]
@@ -1595,8 +1611,8 @@ func (s *L0Sublevels) PickIntraL0Compaction(
 		}
 
 		var f *FileMetadata
-		// Pick the seed file for the interval as the file
-		// in the highest sub-level.
+		// Pick the seed file for the interval as the file in the highest
+		// sub-level.
 		stackDepthReduction := scoredInterval.score
 		for i := len(interval.files) - 1; i >= 0; i-- {
 			f = interval.files[i]
@@ -1604,9 +1620,8 @@ func (s *L0Sublevels) PickIntraL0Compaction(
 				break
 			}
 			consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1)
-			// Can this be the seed file? Files with newer sequence
-			// numbers than earliestUnflushedSeqNum cannot be in
-			// the compaction.
+			// Can this be the seed file? Files with newer sequence numbers than
+			// earliestUnflushedSeqNum cannot be in the compaction.
 			if f.LargestSeqNum >= earliestUnflushedSeqNum {
 				stackDepthReduction--
 				if stackDepthReduction == 0 {
@@ -1669,10 +1684,10 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed(
 	}
 	// The first iteration of this loop produces an intra-L0 compaction at the
 	// seed level. Iterations after that optionally add to the compaction by
-	// stacking more files from intervalIndex and repeating. This is an
-	// optional activity so when it fails we can fallback to the last
-	// successful candidate. The code stops adding when it can't add more, or
-	// when fileBytes grows too large.
+	// stacking more files from intervalIndex and repeating. This is an optional
+	// activity so when it fails we can fallback to the last successful
+	// candidate. The code stops adding when it can't add more, or when
+	// fileBytes grows too large.
 	for ; slIndex >= 0; slIndex-- {
 		f2 := interval.files[slIndex]
 		sl := f2.SubLevel
@@ -1684,23 +1699,24 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed(
 		c.addFile(f2)
 		// The seed file captures all files in the higher level that fall in the
 		// range of intervals. That may extend the range of intervals so for
-		// correctness we need to capture all files in the next higher level that
-		// fall in this extended interval and so on. This can result in an
-		// inverted triangular shape like the following where again the X axis is the
-		// key intervals and the Y axis is oldest to youngest. Note that it is not
-		// necessary for correctness to fill out the shape at lower sub-levels to
-		// make it more rectangular since the invariant only requires that if we
-		// move an older seqnum for key k into a file that has a higher seqnum, we
-		// also move all younger seqnums for that key k into that file.
+		// correctness we need to capture all files in the next higher level
+		// that fall in this extended interval and so on. This can result in an
+		// inverted triangular shape like the following where again the X axis
+		// is the key intervals and the Y axis is oldest to youngest. Note that
+		// it is not necessary for correctness to fill out the shape at lower
+		// sub-levels to make it more rectangular since the invariant only
+		// requires that if we move an older seqnum for key k into a file that
+		// has a higher seqnum, we also move all younger seqnums for that key k
+		// into that file.
 		//                  -----
 		//                   ---
 		//                    -
-		//
-		// It may be better for performance to have a more rectangular shape since
-		// it will reduce the stack depth for more intervals. But there is also
-		// the danger that in explicitly trying to construct a more rectangular
-		// shape we will be forced to pull in a file that is already compacting.
-		// We assume that the performance concern is not a practical issue.
+		// It may be better for performance to have a more rectangular shape
+		// since it will reduce the stack depth for more intervals. But there is
+		// also the danger that in explicitly trying to construct a more
+		// rectangular shape we will be forced to pull in a file that is already
+		// compacting. We assume that the performance concern is not a practical
+		// issue.
 		done := false
 		for currLevel := sl + 1; currLevel < len(s.levelFiles); currLevel++ {
 			if !s.extendFiles(currLevel, earliestUnflushedSeqNum, c) {
@@ -1734,11 +1750,11 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed(
 }
 
 // ExtendL0ForBaseCompactionTo extends the specified base compaction candidate
-// L0CompactionFiles to optionally cover more files in L0 without "touching"
-// any of the passed-in keys (i.e. the smallest/largest bounds are exclusive),
-// as including any user keys for those internal keys
-// could require choosing more files in LBase which is undesirable. Unbounded
-// start/end keys are indicated by passing in the InvalidInternalKey.
+// L0CompactionFiles to optionally cover more files in L0 without "touching" any
+// of the passed-in keys (i.e. the smallest/largest bounds are exclusive), as
+// including any user keys for those internal keys could require choosing more
+// files in LBase which is undesirable. Unbounded start/end keys are indicated
+// by passing in the InvalidInternalKey.
 func (s *L0Sublevels) ExtendL0ForBaseCompactionTo(
 	smallest, largest InternalKey, candidate *L0CompactionFiles,
 ) bool {
@@ -1794,65 +1810,66 @@ func (s *L0Sublevels) ExtendL0ForBaseCompactionTo(
 // Consider this scenario (original candidate is inside the rectangle), with
 // isBase = true and interval bounds a-j (from the union of base file bounds and
 // that of compaction candidate):
-//               _______
-//    L0.3  a--d |  g-j|
-//    L0.2       | f--j|         r-t
-//    L0.1   b-d |e---j|
-//    L0.0  a--d | f--j| l--o  p-----x
 //
-//    Lbase a--------i    m---------w
+//	           _______
+//	L0.3  a--d |  g-j|
+//	L0.2       | f--j|         r-t
+//	L0.1   b-d |e---j|
+//	L0.0  a--d | f--j| l--o  p-----x
+//
+//	Lbase a--------i    m---------w
 //
 // This method will iterate from the bottom up. At L0.0, it will add a--d since
 // it's in the bounds, then add b-d, then a--d, and so on, to produce this:
 //
-//         _____________
-//    L0.3 |a--d    g-j|
-//    L0.2 |       f--j|         r-t
-//    L0.1 | b-d  e---j|
-//    L0.0 |a--d   f--j| l--o  p-----x
+//	     _____________
+//	L0.3 |a--d    g-j|
+//	L0.2 |       f--j|         r-t
+//	L0.1 | b-d  e---j|
+//	L0.0 |a--d   f--j| l--o  p-----x
 //
-//    Lbase a-------i     m---------w
+//	Lbase a-------i     m---------w
 //
 // Let's assume that, instead of a--d in the top sublevel, we had 3 files, a-b,
 // bb-c, and cc-d, of which bb-c is compacting. Let's also add another sublevel
 // L0.4 with some files, all of which aren't compacting:
 //
-//    L0.4  a------c ca--d _______
-//    L0.3  a-b bb-c  cc-d |  g-j|
-//    L0.2                 | f--j|         r-t
-//    L0.1    b----------d |e---j|
-//    L0.0  a------------d | f--j| l--o  p-----x
+//	L0.4  a------c ca--d _______
+//	L0.3  a-b bb-c  cc-d |  g-j|
+//	L0.2                 | f--j|         r-t
+//	L0.1    b----------d |e---j|
+//	L0.0  a------------d | f--j| l--o  p-----x
 //
-//    Lbase a------------------i    m---------w
+//	Lbase a------------------i    m---------w
 //
-// This method then needs to choose between the left side of L0.3 bb-c
-// (i.e. a-b), or the right side (i.e. cc-d and g-j) for inclusion in this
-// compaction. Since the right side has more files as well as one file that has
-// already been picked, it gets chosen at that sublevel, resulting in this
-// intermediate compaction:
+// This method then needs to choose between the left side of L0.3 bb-c (i.e.
+// a-b), or the right side (i.e. cc-d and g-j) for inclusion in this compaction.
+// Since the right side has more files as well as one file that has already been
+// picked, it gets chosen at that sublevel, resulting in this intermediate
+// compaction:
 //
-//    L0.4  a------c ca--d
-//                  ______________
-//    L0.3  a-b bb-c| cc-d    g-j|
-//    L0.2 _________|        f--j|         r-t
-//    L0.1 |  b----------d  e---j|
-//    L0.0 |a------------d   f--j| l--o  p-----x
+//	L0.4  a------c ca--d
+//	              ______________
+//	L0.3  a-b bb-c| cc-d    g-j|
+//	L0.2 _________|        f--j|         r-t
+//	L0.1 |  b----------d  e---j|
+//	L0.0 |a------------d   f--j| l--o  p-----x
 //
-//    Lbase a------------------i    m---------w
+//	Lbase a------------------i    m---------w
 //
 // Since bb-c had to be excluded at L0.3, the interval bounds for L0.4 are
 // actually ca-j, since ca is the next interval start key after the end interval
 // of bb-c. This would result in only ca-d being chosen at that sublevel, even
 // though a--c is also not compacting. This is the final result:
 //
-//                  ______________
-//    L0.4  a------c|ca--d       |
-//    L0.3  a-b bb-c| cc-d    g-j|
-//    L0.2 _________|        f--j|         r-t
-//    L0.1 |  b----------d  e---j|
-//    L0.0 |a------------d   f--j| l--o  p-----x
+//	              ______________
+//	L0.4  a------c|ca--d       |
+//	L0.3  a-b bb-c| cc-d    g-j|
+//	L0.2 _________|        f--j|         r-t
+//	L0.1 |  b----------d  e---j|
+//	L0.0 |a------------d   f--j| l--o  p-----x
 //
-//    Lbase a------------------i    m---------w
+//	Lbase a------------------i    m---------w
 //
 // TODO(bilal): Add more targeted tests for this method, through
 // ExtendL0ForBaseCompactionTo and intraL0CompactionUsingSeed.
@@ -1883,19 +1900,19 @@ func (s *L0Sublevels) extendCandidateToRectangle(
 	}
 	// Stats for files.
 	addedCount := 0
-	// Iterate from the oldest sub-level for L0 -> Lbase and youngest
-	// sub-level for intra-L0. The idea here is that anything that can't
-	// be included from that level constrains what can be included from
-	// the next level. This change in constraint is directly incorporated
-	// into minIntervalIndex, maxIntervalIndex.
+	// Iterate from the oldest sub-level for L0 -> Lbase and youngest sub-level
+	// for intra-L0. The idea here is that anything that can't be included from
+	// that level constrains what can be included from the next level. This
+	// change in constraint is directly incorporated into minIntervalIndex,
+	// maxIntervalIndex.
 	for sl := startLevel; sl != endLevel; sl += increment {
 		files := s.levelFiles[sl]
 		// Find the first file that overlaps with minIntervalIndex.
 		index := sort.Search(len(files), func(i int) bool {
 			return minIntervalIndex <= files[i].maxIntervalIndex
 		})
-		// Track the files that are fully within the current constraint
-		// of [minIntervalIndex, maxIntervalIndex].
+		// Track the files that are fully within the current constraint of
+		// [minIntervalIndex, maxIntervalIndex].
 		firstIndex := -1
 		lastIndex := -1
 		for ; index < len(files); index++ {
@@ -1904,8 +1921,8 @@ func (s *L0Sublevels) extendCandidateToRectangle(
 				break
 			}
 			include := true
-			// Extends out on the left so can't be included. This narrows
-			// what we can included in the next level.
+			// Extends out on the left so can't be included. This narrows what
+			// we can included in the next level.
 			if f.minIntervalIndex < minIntervalIndex {
 				include = false
 				minIntervalIndex = f.maxIntervalIndex + 1
@@ -1932,13 +1949,13 @@ func (s *L0Sublevels) extendCandidateToRectangle(
 			continue
 		}
 		// We have the files in [firstIndex, lastIndex] as potential for
-		// inclusion. Some of these may already have been picked. Some
-		// of them may be already compacting. The latter is tricky since
-		// we have to decide whether to contract minIntervalIndex or
-		// maxIntervalIndex when we encounter an already compacting file.
-		// We pick the longest sequence between firstIndex
-		// and lastIndex of non-compacting files -- this is represented by
-		// [candidateNonCompactingFirst, candidateNonCompactingLast].
+		// inclusion. Some of these may already have been picked. Some of them
+		// may be already compacting. The latter is tricky since we have to
+		// decide whether to contract minIntervalIndex or maxIntervalIndex when
+		// we encounter an already compacting file. We pick the longest sequence
+		// between firstIndex and lastIndex of non-compacting files -- this is
+		// represented by [candidateNonCompactingFirst,
+		// candidateNonCompactingLast].
 		nonCompactingFirst := -1
 		currentRunHasAlreadyPickedFiles := false
 		candidateNonCompactingFirst := -1
@@ -1985,8 +2002,8 @@ func (s *L0Sublevels) extendCandidateToRectangle(
 			}
 		}
 		if candidateNonCompactingFirst == -1 {
-			// All files are compacting. There will be gaps that we could exploit
-			// to continue, but don't bother.
+			// All files are compacting. There will be gaps that we could
+			// exploit to continue, but don't bother.
 			break
 		}
 		// May need to shrink [minIntervalIndex, maxIntervalIndex] for the next level.

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // LevelMetadata contains metadata for all of the files within
@@ -23,12 +24,12 @@ type LevelMetadata struct {
 func (lm *LevelMetadata) clone() LevelMetadata {
 	return LevelMetadata{
 		level: lm.level,
-		tree:  lm.tree.clone(),
+		tree:  lm.tree.Clone(),
 	}
 }
 
 func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
-	return lm.tree.release()
+	return lm.tree.Release()
 }
 
 func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetadata {
@@ -46,29 +47,29 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
 	var t btree
 	t.cmp = cmp
 	for _, f := range files {
-		t.insert(f)
+		t.Insert(f)
 	}
-	return t, LevelSlice{iter: t.iter(), length: t.length}
+	return t, newLevelSlice(t.Iter())
 }
 
 // Empty indicates whether there are any files in the level.
 func (lm *LevelMetadata) Empty() bool {
-	return lm.tree.length == 0
+	return lm.tree.Count() == 0
 }
 
 // Len returns the number of files within the level.
 func (lm *LevelMetadata) Len() int {
-	return lm.tree.length
+	return lm.tree.Count()
 }
 
 // Iter constructs a LevelIterator over the entire level.
 func (lm *LevelMetadata) Iter() LevelIterator {
-	return LevelIterator{iter: lm.tree.iter()}
+	return LevelIterator{iter: lm.tree.Iter()}
 }
 
 // Slice constructs a slice containing the entire level.
 func (lm *LevelMetadata) Slice() LevelSlice {
-	return LevelSlice{iter: lm.tree.iter(), length: lm.tree.length}
+	return newLevelSlice(lm.tree.Iter())
 }
 
 // Find finds the provided file in the level if it exists.
@@ -99,7 +100,7 @@ func (lm *LevelMetadata) Annotation(annotator Annotator) interface{} {
 	if lm.Empty() {
 		return annotator.Zero(nil)
 	}
-	v, _ := lm.tree.root.annotation(annotator)
+	v, _ := lm.tree.root.Annotation(annotator)
 	return v
 }
 
@@ -112,7 +113,7 @@ func (lm *LevelMetadata) InvalidateAnnotation(annotator Annotator) {
 	if lm.Empty() {
 		return
 	}
-	lm.tree.root.invalidateAnnotation(annotator)
+	lm.tree.root.InvalidateAnnotation(annotator)
 }
 
 // LevelFile holds a file's metadata along with its position
@@ -133,7 +134,8 @@ func (lf LevelFile) Slice() LevelSlice {
 // a slice constructor like this?
 func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
 	tr, slice := makeBTree(btreeCmpSeqNum, files)
-	tr.release()
+	tr.Release()
+	slice.verifyInvariants()
 	return slice
 }
 
@@ -143,7 +145,8 @@ func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
 // a slice constructor like this?
 func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice {
 	tr, slice := makeBTree(btreeCmpSmallestKey(cmp), files)
-	tr.release()
+	tr.Release()
+	slice.verifyInvariants()
 	return slice
 }
 
@@ -153,13 +156,56 @@ func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice
 // TODO(jackson): Update tests to avoid requiring this and remove it.
 func NewLevelSliceSpecificOrder(files []*FileMetadata) LevelSlice {
 	tr, slice := makeBTree(btreeCmpSpecificOrder(files), files)
-	tr.release()
+	tr.Release()
+	slice.verifyInvariants()
 	return slice
 }
 
+// newLevelSlice constructs a new LevelSlice backed by iter.
+func newLevelSlice(iter iterator) LevelSlice {
+	s := LevelSlice{iter: iter}
+	if iter.r != nil {
+		s.length = iter.r.subtreeCount
+	}
+	s.verifyInvariants()
+	return s
+}
+
+// newBoundedLevelSlice constructs a new LevelSlice backed by iter and bounded
+// by the provided start and end bounds. The provided startBound and endBound
+// iterators must be iterators over the same B-Tree. Both start and end bounds
+// are inclusive.
+func newBoundedLevelSlice(iter iterator, startBound, endBound *iterator) LevelSlice {
+	s := LevelSlice{
+		iter:  iter,
+		start: startBound,
+		end:   endBound,
+	}
+	if iter.valid() {
+		s.length = endBound.countLeft() - startBound.countLeft()
+		// NB: The +1 is a consequence of the end bound being inclusive.
+		if endBound.valid() {
+			s.length++
+		}
+		// NB: A slice that's empty due to its bounds may have an endBound
+		// positioned before the startBound due to the inclusive bounds.
+		// TODO(jackson): Consider refactoring the end boundary to be exclusive;
+		// it would simplify some areas (eg, here) and complicate others (eg,
+		// Reslice-ing to grow compactions).
+		if s.length < 0 {
+			s.length = 0
+		}
+	}
+	s.verifyInvariants()
+	return s
+}
+
 // LevelSlice contains a slice of the files within a level of the LSM.
 // A LevelSlice is immutable once created, but may be used to construct a
 // mutable LevelIterator over the slice's files.
+//
+// LevelSlices should be constructed through one of the existing constructors,
+// not manually initialized.
 type LevelSlice struct {
 	iter   iterator
 	length int
@@ -170,6 +216,19 @@ type LevelSlice struct {
 	end   *iterator
 }
 
+func (ls LevelSlice) verifyInvariants() {
+	if invariants.Enabled {
+		i := ls.Iter()
+		var length int
+		for f := i.First(); f != nil; f = i.Next() {
+			length++
+		}
+		if ls.length != length {
+			panic(fmt.Sprintf("LevelSlice %s has length %d value; actual length is %d", ls, ls.length, length))
+		}
+	}
+}
+
 // Each invokes fn for each element in the slice.
 func (ls LevelSlice) Each(fn func(*FileMetadata)) {
 	iter := ls.Iter()
@@ -181,6 +240,7 @@ func (ls LevelSlice) Each(fn func(*FileMetadata)) {
 // String implements fmt.Stringer.
 func (ls LevelSlice) String() string {
 	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "%d files: ", ls.length)
 	ls.Each(func(f *FileMetadata) {
 		if buf.Len() > 0 {
 			fmt.Fprintf(&buf, " ")
@@ -246,18 +306,7 @@ func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelS
 		end.iter = ls.end.clone()
 	}
 	resliceFunc(&start, &end)
-
-	s := LevelSlice{
-		iter:  start.iter.clone(),
-		start: &start.iter,
-		end:   &end.iter,
-	}
-	// Calculate the new slice's length.
-	iter := s.Iter()
-	for f := iter.First(); f != nil; f = iter.Next() {
-		s.length++
-	}
-	return s
+	return newBoundedLevelSlice(start.iter.clone(), &start.iter, &end.iter)
 }
 
 // KeyType is used to specify the type of keys we're looking for in
@@ -387,8 +436,13 @@ func (i *LevelIterator) Clone() LevelIterator {
 }
 
 // Current returns the item at the current iterator position.
+//
+// Current is deprecated. Callers should instead use the return value of a
+// positioning operation.
 func (i *LevelIterator) Current() *FileMetadata {
-	if !i.iter.valid() {
+	if !i.iter.valid() ||
+		(i.end != nil && cmpIter(i.iter, *i.end) > 0) ||
+		(i.start != nil && cmpIter(i.iter, *i.start) < 0) {
 		return nil
 	}
 	return i.iter.cur()
@@ -426,7 +480,7 @@ func (i *LevelIterator) First() *FileMetadata {
 	if !i.iter.valid() {
 		return nil
 	}
-	return i.filteredNextFile(i.iter.cur())
+	return i.skipFilteredForward(i.iter.cur())
 }
 
 // Last seeks to the last file in the iterator and returns it.
@@ -442,31 +496,37 @@ func (i *LevelIterator) Last() *FileMetadata {
 	if !i.iter.valid() {
 		return nil
 	}
-	return i.filteredPrevFile(i.iter.cur())
+	return i.skipFilteredBackward(i.iter.cur())
 }
 
 // Next advances the iterator to the next file and returns it.
 func (i *LevelIterator) Next() *FileMetadata {
+	if i.iter.r == nil {
+		return nil
+	}
+	if invariants.Enabled && (i.iter.pos >= i.iter.n.count || (i.end != nil && cmpIter(i.iter, *i.end) > 0)) {
+		panic("pebble: cannot next forward-exhausted iterator")
+	}
 	i.iter.next()
 	if !i.iter.valid() {
 		return nil
 	}
-	if i.end != nil && cmpIter(i.iter, *i.end) > 0 {
-		return nil
-	}
-	return i.filteredNextFile(i.iter.cur())
+	return i.skipFilteredForward(i.iter.cur())
 }
 
 // Prev moves the iterator the previous file and returns it.
 func (i *LevelIterator) Prev() *FileMetadata {
+	if i.iter.r == nil {
+		return nil
+	}
+	if invariants.Enabled && (i.iter.pos < 0 || (i.start != nil && cmpIter(i.iter, *i.start) < 0)) {
+		panic("pebble: cannot prev backward-exhausted iterator")
+	}
 	i.iter.prev()
 	if !i.iter.valid() {
 		return nil
 	}
-	if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
-		return nil
-	}
-	return i.filteredPrevFile(i.iter.cur())
+	return i.skipFilteredBackward(i.iter.cur())
 }
 
 // SeekGE seeks to the first file in the iterator's file set with a largest
@@ -475,28 +535,23 @@ func (i *LevelIterator) Prev() *FileMetadata {
 // be sorted by user keys and non-overlapping.
 func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
 	// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
-	if i.empty() {
+	if i.iter.r == nil {
 		return nil
 	}
-	meta := i.seek(func(m *FileMetadata) bool {
+	m := i.seek(func(m *FileMetadata) bool {
 		return cmp(m.Largest.UserKey, userKey) >= 0
 	})
-	for meta != nil {
-		switch i.filter {
-		case KeyTypePointAndRange:
-			return meta
-		case KeyTypePoint:
-			if meta.HasPointKeys && cmp(meta.LargestPointKey.UserKey, userKey) >= 0 {
-				return meta
-			}
-		case KeyTypeRange:
-			if meta.HasRangeKeys && cmp(meta.LargestRangeKey.UserKey, userKey) >= 0 {
-				return meta
-			}
+	if i.filter != KeyTypePointAndRange && m != nil {
+		b, ok := m.LargestBound(i.filter)
+		if !ok {
+			m = i.Next()
+		} else if c := cmp(b.UserKey, userKey); c < 0 || c == 0 && b.IsExclusiveSentinel() {
+			// This file does not contain any keys of the type ≥ lower. It
+			// should be filtered, even though it does contain point keys.
+			m = i.Next()
 		}
-		meta = i.Next()
 	}
-	return i.filteredNextFile(meta)
+	return i.skipFilteredForward(m)
 }
 
 // SeekLT seeks to the last file in the iterator's file set with a smallest
@@ -505,71 +560,76 @@ func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
 // by user keys and non-overlapping.
 func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata {
 	// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
-	if i.empty() {
+	if i.iter.r == nil {
 		return nil
 	}
 	i.seek(func(m *FileMetadata) bool {
 		return cmp(m.Smallest.UserKey, userKey) >= 0
 	})
-	meta := i.Prev()
-	for meta != nil {
-		switch i.filter {
-		case KeyTypePointAndRange:
-			return meta
-		case KeyTypePoint:
-			if meta.HasPointKeys && cmp(meta.SmallestPointKey.UserKey, userKey) < 0 {
-				return meta
-			}
-		case KeyTypeRange:
-			if meta.HasRangeKeys && cmp(meta.SmallestRangeKey.UserKey, userKey) < 0 {
-				return meta
-			}
+	m := i.Prev()
+	// Although i.Prev() guarantees that the current file contains keys of the
+	// relevant type, it doesn't guarantee that the keys of the relevant type
+	// are < userKey.
+	if i.filter != KeyTypePointAndRange && m != nil {
+		b, ok := m.SmallestBound(i.filter)
+		if !ok {
+			panic("unreachable")
+		}
+		if c := cmp(b.UserKey, userKey); c >= 0 {
+			// This file does not contain any keys of the type ≥ lower. It
+			// should be filtered, even though it does contain point keys.
+			m = i.Prev()
 		}
-		meta = i.Prev()
 	}
-	return i.filteredPrevFile(meta)
+	return i.skipFilteredBackward(m)
 }
 
-func (i *LevelIterator) filteredNextFile(meta *FileMetadata) *FileMetadata {
-	switch i.filter {
-	case KeyTypePoint:
-		for meta != nil && !meta.HasPointKeys {
-			meta = i.Next()
+// skipFilteredForward takes the file metadata at the iterator's current
+// position, and skips forward if the current key-type filter (i.filter)
+// excludes the file. It skips until it finds an unfiltered file or exhausts the
+// level. If lower is != nil, skipFilteredForward skips any files that do not
+// contain keys with the provided key-type ≥ lower.
+//
+// skipFilteredForward also enforces the upper bound, returning nil if at any
+// point the upper bound is exceeded.
+func (i *LevelIterator) skipFilteredForward(meta *FileMetadata) *FileMetadata {
+	for meta != nil && !meta.ContainsKeyType(i.filter) {
+		i.iter.next()
+		if !i.iter.valid() {
+			meta = nil
+		} else {
+			meta = i.iter.cur()
 		}
-		return meta
-	case KeyTypeRange:
-		// TODO(bilal): Range keys are expected to be rare and sparse. Add an
-		// optimization to annotate the tree and efficiently skip over files that
-		// do not contain range keys right at the seek step, to reduce iterations
-		// here.
-		for meta != nil && !meta.HasRangeKeys {
-			meta = i.Next()
-		}
-		return meta
-	default:
-		return meta
 	}
+	if meta != nil && i.end != nil && cmpIter(i.iter, *i.end) > 0 {
+		// Exceeded upper bound.
+		meta = nil
+	}
+	return meta
 }
 
-func (i *LevelIterator) filteredPrevFile(meta *FileMetadata) *FileMetadata {
-	switch i.filter {
-	case KeyTypePoint:
-		for meta != nil && !meta.HasPointKeys {
-			meta = i.Prev()
+// skipFilteredBackward takes the file metadata at the iterator's current
+// position, and skips backward if the current key-type filter (i.filter)
+// excludes the file. It skips until it finds an unfiltered file or exhausts the
+// level. If upper is != nil, skipFilteredBackward skips any files that do not
+// contain keys with the provided key-type < upper.
+//
+// skipFilteredBackward also enforces the lower bound, returning nil if at any
+// point the lower bound is exceeded.
+func (i *LevelIterator) skipFilteredBackward(meta *FileMetadata) *FileMetadata {
+	for meta != nil && !meta.ContainsKeyType(i.filter) {
+		i.iter.prev()
+		if !i.iter.valid() {
+			meta = nil
+		} else {
+			meta = i.iter.cur()
 		}
-		return meta
-	case KeyTypeRange:
-		// TODO(bilal): Range keys are expected to be rare and sparse. Add an
-		// optimization to annotate the tree and efficiently skip over files that
-		// do not contain range keys right at the seek step, to reduce iterations
-		// here.
-		for meta != nil && !meta.HasRangeKeys {
-			meta = i.Prev()
-		}
-		return meta
-	default:
-		return meta
 	}
+	if meta != nil && i.start != nil && cmpIter(i.iter, *i.start) < 0 {
+		// Exceeded lower bound.
+		meta = nil
+	}
+	return meta
 }
 
 func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
@@ -594,7 +654,6 @@ func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
 		return nil
 	} else if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
 		i.iter = i.start.clone()
-		return i.iter.cur()
 	}
 	if !i.iter.valid() {
 		return nil
@@ -614,13 +673,9 @@ func (i *LevelIterator) Take() LevelFile {
 	// the same position for a LevelFile because they're inclusive, so we can
 	// share one iterator stack between the two bounds.
 	boundsIter := i.iter.clone()
+	s := newBoundedLevelSlice(i.iter.clone(), &boundsIter, &boundsIter)
 	return LevelFile{
 		FileMetadata: m,
-		slice: LevelSlice{
-			iter:   i.iter.clone(),
-			start:  &boundsIter,
-			end:    &boundsIter,
-			length: 1,
-		},
+		slice:        s,
 	}
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -17,7 +17,6 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
-	"github.com/cockroachdb/pebble/vfs"
 )
 
 // Compare exports the base.Compare type.
@@ -42,7 +41,8 @@ type TableInfo struct {
 	LargestSeqNum uint64
 }
 
-// TableStats contains statistics on a table used for compaction heuristics.
+// TableStats contains statistics on a table used for compaction heuristics,
+// and export via Metrics.
 type TableStats struct {
 	// The total number of entries in the table.
 	NumEntries uint64
@@ -60,10 +60,14 @@ type TableStats struct {
 	// overlapping data in L0 and ignores L0 sublevels, but the error that
 	// introduces is expected to be small.
 	//
-	// Tables in the bottommost level of the LSM may have a nonzero estimate
-	// if snapshots or move compactions prevented the elision of their range
-	// tombstones.
+	// Tables in the bottommost level of the LSM may have a nonzero estimate if
+	// snapshots or move compactions prevented the elision of their range
+	// tombstones. A table in the bottommost level that was ingested into L6
+	// will have a zero estimate, because the file's sequence numbers indicate
+	// that the tombstone cannot drop any data contained within the file itself.
 	RangeDeletionsBytesEstimate uint64
+	// Total size of value blocks and value index block.
+	ValueBlocksSize uint64
 }
 
 // boundType represents the type of key (point or range) present as the smallest
@@ -79,10 +83,10 @@ const (
 //
 // The following shows the valid state transitions:
 //
-//    NotCompacting --> Compacting --> Compacted
-//          ^               |
-//          |               |
-//          +-------<-------+
+//	NotCompacting --> Compacting --> Compacted
+//	      ^               |
+//	      |               |
+//	      +-------<-------+
 //
 // Input files to a compaction transition to Compacting when a compaction is
 // picked. A file that has finished compacting typically transitions into the
@@ -117,43 +121,81 @@ func (s CompactionState) String() string {
 	}
 }
 
-// FileMetadata holds the metadata for an on-disk table.
+// FileMetadata is maintained for leveled-ssts, i.e., they belong to a level of
+// some version. FileMetadata does not contain the actual level of the sst,
+// since such leveled-ssts can move across levels in different versions, while
+// sharing the same FileMetadata. There are two kinds of leveled-ssts, physical
+// and virtual. Underlying both leveled-ssts is a backing-sst, for which the
+// only state is FileBacking. A backing-sst is level-less. It is possible for a
+// backing-sst to be referred to by a physical sst in one version and by one or
+// more virtual ssts in one or more versions. A backing-sst becomes obsolete
+// and can be deleted once it is no longer required by any physical or virtual
+// sst in any version.
+//
+// We maintain some invariants:
+//
+//  1. Each physical and virtual sst will have a unique FileMetadata.FileNum,
+//     and there will be exactly one FileMetadata associated with the FileNum.
+//
+//  2. Within a version, a backing-sst is either only referred to by one
+//     physical sst or one or more virtual ssts.
+//
+//  3. Once a backing-sst is referred to by a virtual sst in the latest version,
+//     it cannot go back to being referred to by a physical sst in any future
+//     version.
+//
+// Once a physical sst is no longer needed by any version, we will no longer
+// maintain the file metadata associated with it. We will still maintain the
+// FileBacking associated with the physical sst if the backing sst is required
+// by any virtual ssts in any version.
 type FileMetadata struct {
-	// Atomic contains fields which are accessed atomically. Go allocations
-	// are guaranteed to be 64-bit aligned which we take advantage of by
-	// placing the 64-bit fields which we access atomically at the beginning
-	// of the FileMetadata struct. For more information, see
-	// https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
-	Atomic struct {
-		// AllowedSeeks is used to determine if a file should be picked for
-		// a read triggered compaction. It is decremented when read sampling
-		// in pebble.Iterator after every after every positioning operation
-		// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
-		AllowedSeeks int64
+	// AllowedSeeks is used to determine if a file should be picked for
+	// a read triggered compaction. It is decremented when read sampling
+	// in pebble.Iterator after every after every positioning operation
+	// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
+	AllowedSeeks atomic.Int64
 
-		// statsValid is 1 if stats have been loaded for the table. The
-		// TableStats structure is populated only if valid is 1.
-		statsValid uint32
-	}
+	// statsValid indicates if stats have been loaded for the table. The
+	// TableStats structure is populated only if valid is true.
+	statsValid atomic.Bool
+
+	// FileBacking is the state which backs either a physical or virtual
+	// sstables.
+	FileBacking *FileBacking
 
 	// InitAllowedSeeks is the inital value of allowed seeks. This is used
 	// to re-set allowed seeks on a file once it hits 0.
 	InitAllowedSeeks int64
-
-	// Reference count for the file: incremented when a file is added to a
-	// version and decremented when the version is unreferenced. The file is
-	// obsolete when the reference count falls to zero.
-	refs int32
 	// FileNum is the file number.
+	//
+	// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
+	//
+	// TODO(bananabrick): Consider creating separate types for
+	// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
+	// an indentifier for the FileMetadata in Pebble, and also as a handle to
+	// perform reads and writes. We should ensure through types that
+	// FileMetadata.FileNum isn't used to perform reads, and that
+	// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
 	FileNum base.FileNum
-	// Size is the size of the file, in bytes.
+	// Size is the size of the file, in bytes. Size is an approximate value for
+	// virtual sstables.
+	//
+	// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
+	//
+	// TODO(bananabrick): Size is currently used in metrics, and for many key
+	// Pebble level heuristics. Make sure that the heuristics will still work
+	// appropriately with an approximate value of size.
 	Size uint64
 	// File creation time in seconds since the epoch (1970-01-01 00:00:00
 	// UTC). For ingested sstables, this corresponds to the time the file was
-	// ingested.
+	// ingested. For virtual sstables, this corresponds to the wall clock time
+	// when the FileMetadata for the virtual sstable was first created.
 	CreationTime int64
-	// Smallest and largest sequence numbers in the table, across both point and
-	// range keys.
+	// Lower and upper bounds for the smallest and largest sequence numbers in
+	// the table, across both point and range keys. For physical sstables, these
+	// values are tight bounds. For virtual sstables, there is no guarantee that
+	// there will be keys with SmallestSeqNum or LargestSeqNum within virtual
+	// sstable bounds.
 	SmallestSeqNum uint64
 	LargestSeqNum  uint64
 	// SmallestPointKey and LargestPointKey are the inclusive bounds for the
@@ -176,6 +218,13 @@ type FileMetadata struct {
 	Smallest InternalKey
 	Largest  InternalKey
 	// Stats describe table statistics. Protected by DB.mu.
+	//
+	// For virtual sstables, set stats upon virtual sstable creation as
+	// asynchronous computation of stats is not currently supported.
+	//
+	// TODO(bananabrick): To support manifest replay for virtual sstables, we
+	// probably need to compute virtual sstable stats asynchronously. Otherwise,
+	// we'd have to write virtual sstable stats to the version edit.
 	Stats TableStats
 
 	SubLevel         int
@@ -224,6 +273,173 @@ type FileMetadata struct {
 	// key type (point or range) corresponds to the smallest and largest overall
 	// table bounds.
 	boundTypeSmallest, boundTypeLargest boundType
+	// Virtual is true if the FileMetadata belongs to a virtual sstable.
+	Virtual bool
+}
+
+// PhysicalFileMeta is used by functions which want a guarantee that their input
+// belongs to a physical sst and not a virtual sst.
+//
+// NB: This type should only be constructed by calling
+// FileMetadata.PhysicalMeta.
+type PhysicalFileMeta struct {
+	*FileMetadata
+}
+
+// VirtualFileMeta is used by functions which want a guarantee that their input
+// belongs to a virtual sst and not a physical sst.
+//
+// NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
+type VirtualFileMeta struct {
+	*FileMetadata
+}
+
+// PhysicalMeta should be the only source of creating the PhysicalFileMeta
+// wrapper type.
+func (m *FileMetadata) PhysicalMeta() PhysicalFileMeta {
+	if m.Virtual {
+		panic("pebble: file metadata does not belong to a physical sstable")
+	}
+	return PhysicalFileMeta{
+		m,
+	}
+}
+
+// VirtualMeta should be the only source of creating the VirtualFileMeta wrapper
+// type.
+func (m *FileMetadata) VirtualMeta() VirtualFileMeta {
+	if !m.Virtual {
+		panic("pebble: file metadata does not belong to a virtual sstable")
+	}
+	return VirtualFileMeta{
+		m,
+	}
+}
+
+// FileBacking either backs a single physical sstable, or one or more virtual
+// sstables.
+//
+// See the comment above the FileMetadata type for sstable terminology.
+type FileBacking struct {
+	// Reference count for the backing file on disk: incremented when a
+	// physical or virtual sstable which is backed by the FileBacking is
+	// added to a version and decremented when the version is unreferenced.
+	// We ref count in order to determine when it is safe to delete a
+	// backing sst file from disk. The backing file is obsolete when the
+	// reference count falls to zero.
+	refs atomic.Int32
+	// latestVersionRefs are the references to the FileBacking in the
+	// latest version. This reference can be through a single physical
+	// sstable in the latest version, or one or more virtual sstables in the
+	// latest version.
+	//
+	// INVARIANT: latestVersionRefs <= refs.
+	latestVersionRefs atomic.Int32
+	// VirtualizedSize is set iff the backing sst is only referred to by
+	// virtual ssts in the latest version. VirtualizedSize is the sum of the
+	// virtual sstable sizes of all of the virtual sstables in the latest
+	// version which are backed by the physical sstable. When a virtual
+	// sstable is removed from the latest version, we will decrement the
+	// VirtualizedSize. During compaction picking, we'll compensate a
+	// virtual sstable file size by
+	// (FileBacking.Size - FileBacking.VirtualizedSize) / latestVersionRefs.
+	// The intuition is that if FileBacking.Size - FileBacking.VirtualizedSize
+	// is high, then the space amplification due to virtual sstables is
+	// high, and we should pick the virtual sstable with a higher priority.
+	//
+	// TODO(bananabrick): Compensate the virtual sstable file size using
+	// the VirtualizedSize during compaction picking and test.
+	VirtualizedSize atomic.Uint64
+	DiskFileNum     base.DiskFileNum
+	Size            uint64
+}
+
+// InitPhysicalBacking allocates and sets the FileBacking which is required by a
+// physical sstable FileMetadata.
+//
+// Ensure that the state required by FileBacking, such as the FileNum, is
+// already set on the FileMetadata before InitPhysicalBacking is called.
+// Calling InitPhysicalBacking only after the relevant state has been set in the
+// FileMetadata is not necessary in tests which don't rely on FileBacking.
+func (m *FileMetadata) InitPhysicalBacking() {
+	if m.Virtual {
+		panic("pebble: virtual sstables should use a pre-existing FileBacking")
+	}
+	if m.FileBacking == nil {
+		m.FileBacking = &FileBacking{Size: m.Size, DiskFileNum: m.FileNum.DiskFileNum()}
+	}
+}
+
+// ValidateVirtual should be called once the FileMetadata for a virtual sstable
+// is created to verify that the fields of the virtual sstable are sound.
+func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {
+	if !m.Virtual {
+		panic("pebble: invalid virtual sstable")
+	}
+
+	if createdFrom.SmallestSeqNum != m.SmallestSeqNum {
+		panic("pebble: invalid smallest sequence number for virtual sstable")
+	}
+
+	if createdFrom.LargestSeqNum != m.LargestSeqNum {
+		panic("pebble: invalid largest sequence number for virtual sstable")
+	}
+
+	if createdFrom.FileBacking != nil && createdFrom.FileBacking != m.FileBacking {
+		panic("pebble: invalid physical sstable state for virtual sstable")
+	}
+
+	if m.Size == 0 {
+		panic("pebble: virtual sstable size must be set upon creation")
+	}
+}
+
+// Refs returns the refcount of backing sstable.
+func (m *FileMetadata) Refs() int32 {
+	return m.FileBacking.refs.Load()
+}
+
+// Ref increments the ref count associated with the backing sstable.
+func (m *FileMetadata) Ref() {
+	m.FileBacking.refs.Add(1)
+}
+
+// Unref decrements the ref count associated with the backing sstable.
+func (m *FileMetadata) Unref() int32 {
+	v := m.FileBacking.refs.Add(-1)
+	if invariants.Enabled && v < 0 {
+		panic("pebble: invalid FileMetadata refcounting")
+	}
+	return v
+}
+
+// LatestRef increments the latest ref count associated with the backing
+// sstable.
+func (m *FileMetadata) LatestRef() {
+	m.FileBacking.latestVersionRefs.Add(1)
+
+	if m.Virtual {
+		m.FileBacking.VirtualizedSize.Add(m.Size)
+	}
+}
+
+// LatestUnref decrements the latest ref count associated with the backing
+// sstable.
+func (m *FileMetadata) LatestUnref() int32 {
+	if m.Virtual {
+		m.FileBacking.VirtualizedSize.Add(-m.Size)
+	}
+
+	v := m.FileBacking.latestVersionRefs.Add(-1)
+	if invariants.Enabled && v < 0 {
+		panic("pebble: invalid FileMetadata latest refcounting")
+	}
+	return v
+}
+
+// LatestRefs returns the latest ref count associated with the backing sstable.
+func (m *FileMetadata) LatestRefs() int32 {
+	return m.FileBacking.latestVersionRefs.Load()
 }
 
 // SetCompactionState transitions this file's compaction state to the given
@@ -261,22 +477,14 @@ func (m *FileMetadata) IsCompacting() bool {
 // returns true, the Stats field may be read (with or without holding the
 // database mutex).
 func (m *FileMetadata) StatsValid() bool {
-	return atomic.LoadUint32(&m.Atomic.statsValid) == 1
-}
-
-// StatsValidLocked returns true if the table stats have been populated.
-// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the
-// overhead of an atomic load. This is possible because table stats validity is
-// only set while DB.mu is held.
-func (m *FileMetadata) StatsValidLocked() bool {
-	return m.Atomic.statsValid == 1
+	return m.statsValid.Load()
 }
 
 // StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu
 // while populating TableStats and calling StatsMarkValud. Once stats are
 // populated, they must not be mutated.
 func (m *FileMetadata) StatsMarkValid() {
-	atomic.StoreUint32(&m.Atomic.statsValid, 1)
+	m.statsValid.Store(true)
 }
 
 // ExtendPointKeyBounds attempts to extend the lower and upper point key bounds
@@ -354,6 +562,66 @@ func (m *FileMetadata) extendOverallBounds(
 	}
 }
 
+// Overlaps returns true if the file key range overlaps with the given range.
+func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusiveEnd bool) bool {
+	if c := cmp(m.Largest.UserKey, start); c < 0 || (c == 0 && m.Largest.IsExclusiveSentinel()) {
+		// f is completely before the specified range; no overlap.
+		return false
+	}
+	if c := cmp(m.Smallest.UserKey, end); c > 0 || (c == 0 && exclusiveEnd) {
+		// f is completely after the specified range; no overlap.
+		return false
+	}
+	return true
+}
+
+// ContainsKeyType returns whether or not the file contains keys of the provided
+// type.
+func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
+	switch kt {
+	case KeyTypePointAndRange:
+		return true
+	case KeyTypePoint:
+		return m.HasPointKeys
+	case KeyTypeRange:
+		return m.HasRangeKeys
+	default:
+		panic("unrecognized key type")
+	}
+}
+
+// SmallestBound returns the file's smallest bound of the key type. It returns a
+// false second return value if the file does not contain any keys of the key
+// type.
+func (m *FileMetadata) SmallestBound(kt KeyType) (*InternalKey, bool) {
+	switch kt {
+	case KeyTypePointAndRange:
+		return &m.Smallest, true
+	case KeyTypePoint:
+		return &m.SmallestPointKey, m.HasPointKeys
+	case KeyTypeRange:
+		return &m.SmallestRangeKey, m.HasRangeKeys
+	default:
+		panic("unrecognized key type")
+	}
+}
+
+// LargestBound returns the file's largest bound of the key type. It returns a
+// false second return value if the file does not contain any keys of the key
+// type.
+func (m *FileMetadata) LargestBound(kt KeyType) (*InternalKey, bool) {
+	switch kt {
+	case KeyTypePointAndRange:
+		return &m.Largest, true
+	case KeyTypePoint:
+		return &m.LargestPointKey, m.HasPointKeys
+	case KeyTypeRange:
+		return &m.LargestRangeKey, m.HasRangeKeys
+	default:
+		panic("unrecognized key type")
+	}
+}
+
 const (
 	maskContainsPointKeys = 1 << 0
 	maskSmallest          = 1 << 1
@@ -417,7 +685,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 
 // ParseFileMetadataDebug parses a FileMetadata from its DebugString
 // representation.
-func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
+func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
 	// Split lines of the form:
 	//  000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
 	fields := strings.FieldsFunc(s, func(c rune) bool {
@@ -429,8 +697,9 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
 		}
 	})
 	if len(fields)%3 != 0 {
-		return m, errors.Newf("malformed input: %s", s)
+		return nil, errors.Newf("malformed input: %s", s)
 	}
+	m := &FileMetadata{}
 	for len(fields) > 0 {
 		prefix := fields[0]
 		smallest := base.ParsePrettyInternalKey(fields[1])
@@ -460,7 +729,8 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
 		m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
 		m.HasPointKeys = true
 	}
-	return
+	m.InitPhysicalBacking()
+	return m, nil
 }
 
 // Validate validates the metadata for consistency with itself, returning an
@@ -522,6 +792,11 @@ func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error {
 		}
 	}
 
+	// Ensure that FileMetadata.Init was called.
+	if m.FileBacking == nil {
+		return base.CorruptionErrorf("file metadata FileBacking not set")
+	}
+
 	return nil
 }
 
@@ -626,51 +901,50 @@ func SortBySmallest(files []*FileMetadata, cmp Compare) {
 
 func overlaps(iter LevelIterator, cmp Compare, start, end []byte, exclusiveEnd bool) LevelSlice {
 	startIter := iter.Clone()
-	startIter.SeekGE(cmp, start)
-
-	// SeekGE compares user keys. The user key `start` may be equal to the
-	// f.Largest because f.Largest is a range deletion sentinel, indicating that
-	// the user key `start` is NOT contained within the file f. If that's the
-	// case, we can narrow the overlapping bounds to exclude the file with the
-	// sentinel.
-	if f := startIter.Current(); f != nil && f.Largest.IsExclusiveSentinel() &&
-		cmp(f.Largest.UserKey, start) == 0 {
-		startIter.Next()
+	{
+		startIterFile := startIter.SeekGE(cmp, start)
+		// SeekGE compares user keys. The user key `start` may be equal to the
+		// f.Largest because f.Largest is a range deletion sentinel, indicating
+		// that the user key `start` is NOT contained within the file f. If
+		// that's the case, we can narrow the overlapping bounds to exclude the
+		// file with the sentinel.
+		if startIterFile != nil && startIterFile.Largest.IsExclusiveSentinel() &&
+			cmp(startIterFile.Largest.UserKey, start) == 0 {
+			startIterFile = startIter.Next()
+		}
+		_ = startIterFile // Ignore unused assignment.
 	}
 
 	endIter := iter.Clone()
-	endIter.SeekGE(cmp, end)
+	{
+		endIterFile := endIter.SeekGE(cmp, end)
 
-	if !exclusiveEnd {
-		// endIter is now pointing at the *first* file with a largest key >= end.
-		// If there are multiple files including the user key `end`, we want all
-		// of them, so move forward.
-		for f := endIter.Current(); f != nil && cmp(f.Largest.UserKey, end) == 0; {
-			f = endIter.Next()
+		if !exclusiveEnd {
+			// endIter is now pointing at the *first* file with a largest key >= end.
+			// If there are multiple files including the user key `end`, we want all
+			// of them, so move forward.
+			for endIterFile != nil && cmp(endIterFile.Largest.UserKey, end) == 0 {
+				endIterFile = endIter.Next()
+			}
 		}
-	}
 
-	// LevelSlice uses inclusive bounds, so if we seeked to the end sentinel
-	// or nexted too far because Largest.UserKey equaled `end`, go back.
-	//
-	// Consider !exclusiveEnd and end = 'f', with the following file bounds:
-	//
-	//     [b,d] [e, f] [f, f] [g, h]
-	//
-	// the above for loop will Next until it arrives at [g, h]. We need to
-	// observe that g > f, and Prev to the file with bounds [f, f].
-	if !endIter.iter.valid() {
-		endIter.Prev()
-	} else if c := cmp(endIter.Current().Smallest.UserKey, end); c > 0 || c == 0 && exclusiveEnd {
-		endIter.Prev()
-	}
-
-	iter = startIter.Clone()
-	return LevelSlice{
-		iter:  iter.iter,
-		start: &startIter.iter,
-		end:   &endIter.iter,
+		// LevelSlice uses inclusive bounds, so if we seeked to the end sentinel
+		// or nexted too far because Largest.UserKey equaled `end`, go back.
+		//
+		// Consider !exclusiveEnd and end = 'f', with the following file bounds:
+		//
+		//     [b,d] [e, f] [f, f] [g, h]
+		//
+		// the above for loop will Next until it arrives at [g, h]. We need to
+		// observe that g > f, and Prev to the file with bounds [f, f].
+		if endIterFile == nil {
+			endIterFile = endIter.Prev()
+		} else if c := cmp(endIterFile.Smallest.UserKey, end); c > 0 || c == 0 && exclusiveEnd {
+			endIterFile = endIter.Prev()
+		}
+		_ = endIterFile // Ignore unused assignment.
 	}
+	return newBoundedLevelSlice(startIter.Clone().iter, &startIter.iter, &endIter.iter)
 }
 
 // NumLevels is the number of levels a Version contains.
@@ -744,7 +1018,7 @@ type Version struct {
 	// usage.
 	//
 	// L0Sublevels.Levels contains L0 files ordered by sublevels. All the files
-	// in Files[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
+	// in Levels[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
 	// a reference to that slice, as that slice is necessary for iterator
 	// creation and needs to outlast L0Sublevels.
 	L0Sublevels     *L0Sublevels
@@ -759,7 +1033,7 @@ type Version struct {
 
 	// The callback to invoke when the last reference to a version is
 	// removed. Will be called with list.mu held.
-	Deleted func(obsolete []*FileMetadata)
+	Deleted func(obsolete []*FileBacking)
 
 	// Stats holds aggregated stats about the version maintained from
 	// version to version.
@@ -788,15 +1062,21 @@ func (v *Version) DebugString(format base.FormatKey) string {
 	return v.string(format, true)
 }
 
+func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string {
+	var buf bytes.Buffer
+	for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- {
+		fmt.Fprintf(&buf, "0.%d:\n", sublevel)
+		sublevels[sublevel].Each(func(f *FileMetadata) {
+			fmt.Fprintf(&buf, "  %s\n", f.DebugString(format, verbose))
+		})
+	}
+	return buf.String()
+}
+
 func (v *Version) string(format base.FormatKey, verbose bool) string {
 	var buf bytes.Buffer
 	if len(v.L0SublevelFiles) > 0 {
-		for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
-			fmt.Fprintf(&buf, "0.%d:\n", sublevel)
-			v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) {
-				fmt.Fprintf(&buf, "  %s\n", f.DebugString(format, verbose))
-			})
-		}
+		fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles))
 	}
 	for level := 1; level < NumLevels; level++ {
 		if v.Levels[level].Empty() {
@@ -837,7 +1117,7 @@ func ParseVersionDebug(
 				m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
 				m.HasPointKeys = true
 			}
-			files[level] = append(files[level], &m)
+			files[level] = append(files[level], m)
 		}
 	}
 	// Reverse the order of L0 files. This ensures we construct the same
@@ -865,11 +1145,15 @@ func (v *Version) Ref() {
 // locked.
 func (v *Version) Unref() {
 	if atomic.AddInt32(&v.refs, -1) == 0 {
-		obsolete := v.unrefFiles()
 		l := v.list
 		l.mu.Lock()
 		l.Remove(v)
-		v.Deleted(obsolete)
+		obsolete := v.unrefFiles()
+		fileBacking := make([]*FileBacking, len(obsolete))
+		for i, f := range obsolete {
+			fileBacking[i] = f.FileBacking
+		}
+		v.Deleted(fileBacking)
 		l.mu.Unlock()
 	}
 }
@@ -881,7 +1165,12 @@ func (v *Version) Unref() {
 func (v *Version) UnrefLocked() {
 	if atomic.AddInt32(&v.refs, -1) == 0 {
 		v.list.Remove(v)
-		v.Deleted(v.unrefFiles())
+		obsolete := v.unrefFiles()
+		fileBacking := make([]*FileBacking, len(obsolete))
+		for i, f := range obsolete {
+			fileBacking[i] = f.FileBacking
+		}
+		v.Deleted(fileBacking)
 	}
 }
 
@@ -933,11 +1222,11 @@ func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool {
 }
 
 // Overlaps returns all elements of v.files[level] whose user key range
-// intersects the inclusive range [start, end]. If level is non-zero then the
-// user key ranges of v.files[level] are assumed to not overlap (although they
-// may touch). If level is zero then that assumption cannot be made, and the
-// [start, end] range is expanded to the union of those matching ranges so far
-// and the computation is repeated until [start, end] stabilizes.
+// intersects the given range. If level is non-zero then the user key ranges of
+// v.files[level] are assumed to not overlap (although they may touch). If level
+// is zero then that assumption cannot be made, and the [start, end] range is
+// expanded to the union of those matching ranges so far and the computation is
+// repeated until [start, end] stabilizes.
 // The returned files are a subsequence of the input files, i.e., the ordering
 // is not changed.
 func (v *Version) Overlaps(
@@ -957,20 +1246,16 @@ func (v *Version) Overlaps(
 				if selected {
 					continue
 				}
-				smallest := meta.Smallest.UserKey
-				largest := meta.Largest.UserKey
-				if c := cmp(largest, start); c < 0 || c == 0 && meta.Largest.IsExclusiveSentinel() {
-					// meta is completely before the specified range; skip it.
-					continue
-				}
-				if c := cmp(smallest, end); c > 0 || c == 0 && exclusiveEnd {
-					// meta is completely after the specified range; skip it.
+				if !meta.Overlaps(cmp, start, end, exclusiveEnd) {
+					// meta is completely outside the specified range; skip it.
 					continue
 				}
 				// Overlaps.
 				selectedIndices[i] = true
 				numSelected++
 
+				smallest := meta.Smallest.UserKey
+				largest := meta.Largest.UserKey
 				// Since level == 0, check if the newly added fileMetadata has
 				// expanded the range. We expand the range immediately for files
 				// we have remaining to check in this loop. All already checked
@@ -998,17 +1283,17 @@ func (v *Version) Overlaps(
 				tr.cmp = v.Levels[level].tree.cmp
 				for i, meta := 0, l0Iter.First(); meta != nil; i, meta = i+1, l0Iter.Next() {
 					if selectedIndices[i] {
-						err := tr.insert(meta)
+						err := tr.Insert(meta)
 						if err != nil {
 							panic(err)
 						}
 					}
 				}
-				slice = LevelSlice{iter: tr.iter(), length: tr.length}
+				slice = newLevelSlice(tr.Iter())
 				// TODO(jackson): Avoid the oddity of constructing and
 				// immediately releasing a B-Tree. Make LevelSlice an
 				// interface?
-				tr.release()
+				tr.Release()
 				break
 			}
 			// Continue looping to retry the files that were not selected.
@@ -1038,37 +1323,6 @@ func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
 	return nil
 }
 
-// CheckConsistency checks that all of the files listed in the version exist
-// and their on-disk sizes match the sizes listed in the version.
-func (v *Version) CheckConsistency(dirname string, fs vfs.FS) error {
-	var buf bytes.Buffer
-	var args []interface{}
-
-	for level, files := range v.Levels {
-		iter := files.Iter()
-		for f := iter.First(); f != nil; f = iter.Next() {
-			path := base.MakeFilepath(fs, dirname, base.FileTypeTable, f.FileNum)
-			info, err := fs.Stat(path)
-			if err != nil {
-				buf.WriteString("L%d: %s: %v\n")
-				args = append(args, errors.Safe(level), errors.Safe(f.FileNum), err)
-				continue
-			}
-			if info.Size() != int64(f.Size) {
-				buf.WriteString("L%d: %s: file size mismatch (%s): %d (disk) != %d (MANIFEST)\n")
-				args = append(args, errors.Safe(level), errors.Safe(f.FileNum), path,
-					errors.Safe(info.Size()), errors.Safe(f.Size))
-				continue
-			}
-		}
-	}
-
-	if buf.Len() == 0 {
-		return nil
-	}
-	return errors.Errorf(buf.String(), args...)
-}
-
 // VersionList holds a list of versions. The versions are ordered from oldest
 // to newest.
 type VersionList struct {

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -8,11 +8,14 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"io"
-	"sync/atomic"
+	"sort"
+	"time"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // TODO(peter): describe the MANIFEST file format, independently of the C++
@@ -107,9 +110,40 @@ type VersionEdit struct {
 	// found that there was no overlapping file at the higher level).
 	DeletedFiles map[DeletedFileEntry]*FileMetadata
 	NewFiles     []NewFileEntry
+	// CreatedBackingTables can be used to preserve the FileBacking associated
+	// with a physical sstable. This is useful when virtual sstables in the
+	// latest version are reconstructed during manifest replay, and we also need
+	// to reconstruct the FileBacking which is required by these virtual
+	// sstables.
+	//
+	// INVARIANT: The FileBacking associated with a physical sstable must only
+	// be added as a backing file in the same version edit where the physical
+	// sstable is first virtualized. This means that the physical sstable must
+	// be present in DeletedFiles and that there must be at least one virtual
+	// sstable with the same FileBacking as the physical sstable in NewFiles. A
+	// file must be present in CreatedBackingTables in exactly one version edit.
+	// The physical sstable associated with the FileBacking must also not be
+	// present in NewFiles.
+	CreatedBackingTables []*FileBacking
+	// RemovedBackingTables is used to remove the FileBacking associated with a
+	// virtual sstable. Note that a backing sstable can be removed as soon as
+	// there are no virtual sstables in the latest version which are using the
+	// backing sstable, but the backing sstable doesn't necessarily have to be
+	// removed atomically with the version edit which removes the last virtual
+	// sstable associated with the backing sstable. The removal can happen in a
+	// future version edit.
+	//
+	// INVARIANT: A file must only be added to RemovedBackingTables if it was
+	// added to CreateBackingTables in a prior version edit. The same version
+	// edit also cannot have the same file present in both CreateBackingTables
+	// and RemovedBackingTables. A file must be present in RemovedBackingTables
+	// in exactly one version edit.
+	RemovedBackingTables []base.DiskFileNum
 }
 
 // Decode decodes an edit from the specified reader.
+//
+// TODO(bananabrick): Support decoding of virtual sstable state.
 func (v *VersionEdit) Decode(r io.Reader) error {
 	br, ok := r.(byteReader)
 	if !ok {
@@ -342,6 +376,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				}
 			}
 			m.boundsSet = true
+			m.InitPhysicalBacking()
 			v.NewFiles = append(v.NewFiles, NewFileEntry{
 				Level: level,
 				Meta:  m,
@@ -364,7 +399,51 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 	return nil
 }
 
+// String implements fmt.Stringer for a VersionEdit.
+func (v *VersionEdit) String() string {
+	var buf bytes.Buffer
+	if v.ComparerName != "" {
+		fmt.Fprintf(&buf, "  comparer:     %s", v.ComparerName)
+	}
+	if v.MinUnflushedLogNum != 0 {
+		fmt.Fprintf(&buf, "  log-num:       %d\n", v.MinUnflushedLogNum)
+	}
+	if v.ObsoletePrevLogNum != 0 {
+		fmt.Fprintf(&buf, "  prev-log-num:  %d\n", v.ObsoletePrevLogNum)
+	}
+	if v.NextFileNum != 0 {
+		fmt.Fprintf(&buf, "  next-file-num: %d\n", v.NextFileNum)
+	}
+	if v.LastSeqNum != 0 {
+		fmt.Fprintf(&buf, "  last-seq-num:  %d\n", v.LastSeqNum)
+	}
+	entries := make([]DeletedFileEntry, 0, len(v.DeletedFiles))
+	for df := range v.DeletedFiles {
+		entries = append(entries, df)
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		if entries[i].Level != entries[j].Level {
+			return entries[i].Level < entries[j].Level
+		}
+		return entries[i].FileNum < entries[j].FileNum
+	})
+	for _, df := range entries {
+		fmt.Fprintf(&buf, "  deleted:       L%d %s\n", df.Level, df.FileNum)
+	}
+	for _, nf := range v.NewFiles {
+		fmt.Fprintf(&buf, "  added:         L%d %s", nf.Level, nf.Meta.String())
+		if nf.Meta.CreationTime != 0 {
+			fmt.Fprintf(&buf, " (%s)",
+				time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
+		}
+		fmt.Fprintln(&buf)
+	}
+	return buf.String()
+}
+
 // Encode encodes an edit to the specified writer.
+//
+// TODO(bananabrick): Support encoding of virtual sstable state.
 func (v *VersionEdit) Encode(w io.Writer) error {
 	e := versionEditEncoder{new(bytes.Buffer)}
 
@@ -536,10 +615,28 @@ func (e versionEditEncoder) writeUvarint(u uint64) {
 
 // BulkVersionEdit summarizes the files added and deleted from a set of version
 // edits.
+//
+// INVARIANTS:
+// No file can be added to a level more than once. This is true globally, and
+// also true for all of the calls to Accumulate for a single bulk version edit.
+//
+// No file can be removed from a level more than once. This is true globally,
+// and also true for all of the calls to Accumulate for a single bulk version
+// edit.
+//
+// A file must not be added and removed from a given level in the same version
+// edit.
+//
+// A file that is being removed from a level must have been added to that level
+// before (in a prior version edit). Note that a given file can be deleted from
+// a level and added to another level in a single version edit
 type BulkVersionEdit struct {
-	Added   [NumLevels][]*FileMetadata
+	Added   [NumLevels]map[base.FileNum]*FileMetadata
 	Deleted [NumLevels]map[base.FileNum]*FileMetadata
 
+	AddedFileBacking   []*FileBacking
+	RemovedFileBacking []base.DiskFileNum
+
 	// AddedByFileNum maps file number to file metadata for all added files
 	// from accumulated version edits. AddedByFileNum is only populated if set
 	// to non-nil by a caller. It must be set to non-nil when replaying
@@ -558,6 +655,19 @@ type BulkVersionEdit struct {
 
 // Accumulate adds the file addition and deletions in the specified version
 // edit to the bulk edit's internal state.
+//
+// INVARIANTS:
+// If a file is added to a given level in a call to Accumulate and then removed
+// from that level in a subsequent call, the file will not be present in the
+// resulting BulkVersionEdit.Deleted for that level.
+//
+// After accumulation of version edits, the bulk version edit may have
+// information about a file which has been deleted from a level, but it may
+// not have information about the same file added to the same level. The add
+// could've occurred as part of a previous bulk version edit. In this case,
+// the deleted file must be present in BulkVersionEdit.Deleted, at the end
+// of the accumulation, because we need to decrease the refcount of the
+// deleted file in Apply.
 func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 	for df, m := range ve.DeletedFiles {
 		dmap := b.Deleted[df.Level]
@@ -579,7 +689,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		if m.MarkedForCompaction {
 			b.MarkedForCompactionCountDiff--
 		}
-		dmap[df.FileNum] = m
+		if _, ok := b.Added[df.Level][df.FileNum]; !ok {
+			dmap[df.FileNum] = m
+		} else {
+			// Present in b.Added for the same level.
+			delete(b.Added[df.Level], df.FileNum)
+		}
 	}
 
 	for _, nf := range ve.NewFiles {
@@ -590,7 +705,10 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 				return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
 			}
 		}
-		b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta)
+		if b.Added[nf.Level] == nil {
+			b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
+		}
+		b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta
 		if b.AddedByFileNum != nil {
 			b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta
 		}
@@ -598,36 +716,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 			b.MarkedForCompactionCountDiff++
 		}
 	}
+
+	// Generate state for the backing files.
+	b.AddedFileBacking = append(b.AddedFileBacking, ve.CreatedBackingTables...)
+
+	// Since a file can be removed from backing files in exactly one version
+	// edit it is safe to just append without any de-duplication.
+	b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)
+
 	return nil
 }
 
+// AccumulateIncompleteAndApplySingleVE should be called if a single version edit
+// is to be applied to the provided curr Version and if the caller needs to
+// update the versionSet.zombieTables map. This function exists separately from
+// BulkVersionEdit.Apply because it is easier to reason about properties
+// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we
+// know that exactly one version edit is being accumulated.
+//
+// Note that the version edit passed into this function may be incomplete
+// because compactions don't have the ref counting information necessary to
+// populate VersionEdit.RemovedBackingTables. This function will complete such a
+// version edit by populating RemovedBackingTables.
+//
+// Invariant: Any file being deleted through ve must belong to the curr Version.
+// We can't have a delete for some arbitrary file which does not exist in curr.
+func AccumulateIncompleteAndApplySingleVE(
+	ve *VersionEdit,
+	curr *Version,
+	cmp Compare,
+	formatKey base.FormatKey,
+	flushSplitBytes int64,
+	readCompactionRate int64,
+	backingStateMap map[base.DiskFileNum]*FileBacking,
+) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
+	if len(ve.RemovedBackingTables) != 0 {
+		panic("pebble: invalid incomplete version edit")
+	}
+	var b BulkVersionEdit
+	err := b.Accumulate(ve)
+	if err != nil {
+		return nil, nil, err
+	}
+	zombies = make(map[base.DiskFileNum]uint64)
+	v, err := b.Apply(
+		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, s := range b.AddedFileBacking {
+		backingStateMap[s.DiskFileNum] = s
+	}
+
+	for fileNum := range zombies {
+		if _, ok := backingStateMap[fileNum]; ok {
+			// This table was backing some virtual sstable in the latest version,
+			// but is now a zombie. We add RemovedBackingTables entries for
+			// these, before the version edit is written to disk.
+			ve.RemovedBackingTables = append(
+				ve.RemovedBackingTables, fileNum,
+			)
+			delete(backingStateMap, fileNum)
+		}
+	}
+
+	return v, zombies, nil
+}
+
 // Apply applies the delta b to the current version to produce a new
 // version. The new version is consistent with respect to the comparer cmp.
 //
 // curr may be nil, which is equivalent to a pointer to a zero version.
 //
-// On success, a map of zombie files containing the file numbers and sizes of
-// deleted files is returned. These files are considered zombies because they
-// are no longer referenced by the returned Version, but cannot be deleted from
-// disk as they are still in use by the incoming Version.
+// On success, if a non-nil zombies map is provided to Apply, the map is updated
+// with file numbers and files sizes of deleted files. These files are
+// considered zombies because they are no longer referenced by the returned
+// Version, but cannot be deleted from disk as they are still in use by the
+// incoming Version.
 func (b *BulkVersionEdit) Apply(
 	curr *Version,
 	cmp Compare,
 	formatKey base.FormatKey,
 	flushSplitBytes int64,
 	readCompactionRate int64,
-) (_ *Version, zombies map[base.FileNum]uint64, _ error) {
-	addZombie := func(fileNum base.FileNum, size uint64) {
-		if zombies == nil {
-			zombies = make(map[base.FileNum]uint64)
-		}
-		zombies[fileNum] = size
-	}
-	// The remove zombie function is used to handle tables that are moved from
-	// one level to another during a version edit (i.e. a "move" compaction).
-	removeZombie := func(fileNum base.FileNum) {
+	zombies map[base.DiskFileNum]uint64,
+) (*Version, error) {
+	addZombie := func(state *FileBacking) {
 		if zombies != nil {
-			delete(zombies, fileNum)
+			zombies[state.DiskFileNum] = state.Size
+		}
+	}
+	removeZombie := func(state *FileBacking) {
+		if zombies != nil {
+			delete(zombies, state.DiskFileNum)
 		}
 	}
 
@@ -639,7 +822,7 @@ func (b *BulkVersionEdit) Apply(
 	}
 	v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff
 	if v.Stats.MarkedForCompaction < 0 {
-		return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
+		return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
 	}
 
 	for level := range v.Levels {
@@ -660,7 +843,7 @@ func (b *BulkVersionEdit) Apply(
 				// Initialize L0Sublevels.
 				if curr == nil || curr.L0Sublevels == nil {
 					if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
-						return nil, nil, errors.Wrap(err, "pebble: internal error")
+						return nil, errors.Wrap(err, "pebble: internal error")
 					}
 				} else {
 					v.L0Sublevels = curr.L0Sublevels
@@ -673,47 +856,70 @@ func (b *BulkVersionEdit) Apply(
 		// Some edits on this level.
 		lm := &v.Levels[level]
 		lmRange := &v.RangeKeyLevels[level]
-		addedFiles := b.Added[level]
-		deletedMap := b.Deleted[level]
-		if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
-			return nil, nil, base.CorruptionErrorf(
+
+		addedFilesMap := b.Added[level]
+		deletedFilesMap := b.Deleted[level]
+		if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 {
+			return nil, base.CorruptionErrorf(
 				"pebble: internal error: No current or added files but have deleted files: %d",
-				errors.Safe(len(deletedMap)))
+				errors.Safe(len(deletedFilesMap)))
 		}
 
-		// NB: addedFiles may be empty and it also is not necessarily
-		// internally consistent: it does not reflect deletions in deletedMap.
+		// NB: addedFilesMap may be empty. If a file is present in addedFilesMap
+		// for a level, it won't be present in deletedFilesMap for the same
+		// level.
 
-		for _, f := range deletedMap {
-			addZombie(f.FileNum, f.Size)
-			if obsolete := v.Levels[level].tree.delete(f); obsolete {
+		for _, f := range deletedFilesMap {
+			if obsolete := v.Levels[level].tree.Delete(f); obsolete {
 				// Deleting a file from the B-Tree may decrement its
 				// reference count. However, because we cloned the
 				// previous level's B-Tree, this should never result in a
 				// file's reference count dropping to zero.
 				err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
-				return nil, nil, err
+				return nil, err
 			}
 			if f.HasRangeKeys {
-				if obsolete := v.RangeKeyLevels[level].tree.delete(f); obsolete {
+				if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
 					// Deleting a file from the B-Tree may decrement its
 					// reference count. However, because we cloned the
 					// previous level's B-Tree, this should never result in a
 					// file's reference count dropping to zero.
 					err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum)
-					return nil, nil, err
+					return nil, err
 				}
 			}
+
+			// Note that a backing sst will only become a zombie if the
+			// references to it in the latest version is 0. We will remove the
+			// backing sst from the zombie list in the next loop if one of the
+			// addedFiles in any of the levels is referencing the backing sst.
+			// This is possible if a physical sstable is virtualized, or if it
+			// is moved.
+			latestRefCount := f.LatestRefs()
+			if latestRefCount <= 0 {
+				// If a file is present in deletedFilesMap for a level, then it
+				// must have already been added to the level previously, which
+				// means that its latest ref count cannot be 0.
+				err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum)
+				return nil, err
+			} else if f.LatestUnref() == 0 {
+				addZombie(f.FileBacking)
+			}
 		}
 
+		addedFiles := make([]*FileMetadata, 0, len(addedFilesMap))
+		for _, f := range addedFilesMap {
+			addedFiles = append(addedFiles, f)
+		}
+		// Sort addedFiles by file number. This isn't necessary, but tests which
+		// replay invalid manifests check the error output, and the error output
+		// depends on the order in which files are added to the btree.
+		sort.Slice(addedFiles, func(i, j int) bool {
+			return addedFiles[i].FileNum < addedFiles[j].FileNum
+		})
+
 		var sm, la *FileMetadata
 		for _, f := range addedFiles {
-			if _, ok := deletedMap[f.FileNum]; ok {
-				// Already called addZombie on this file in the preceding
-				// loop, so we don't need to do it here.
-				continue
-			}
-
 			// NB: allowedSeeks is used for read triggered compactions. It is set using
 			// Options.Experimental.ReadCompactionRate which defaults to 32KB.
 			var allowedSeeks int64
@@ -723,20 +929,23 @@ func (b *BulkVersionEdit) Apply(
 			if allowedSeeks < 100 {
 				allowedSeeks = 100
 			}
-			atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks)
+			f.AllowedSeeks.Store(allowedSeeks)
 			f.InitAllowedSeeks = allowedSeeks
 
-			err := lm.tree.insert(f)
+			err := lm.tree.Insert(f)
+			// We're adding this file to the new version, so increment the
+			// latest refs count.
+			f.LatestRef()
 			if err != nil {
-				return nil, nil, errors.Wrap(err, "pebble")
+				return nil, errors.Wrap(err, "pebble")
 			}
 			if f.HasRangeKeys {
-				err = lmRange.tree.insert(f)
+				err = lmRange.tree.Insert(f)
 				if err != nil {
-					return nil, nil, errors.Wrap(err, "pebble")
+					return nil, errors.Wrap(err, "pebble")
 				}
 			}
-			removeZombie(f.FileNum)
+			removeZombie(f.FileBacking)
 			// Track the keys with the smallest and largest keys, so that we can
 			// check consistency of the modified span.
 			if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 {
@@ -748,27 +957,36 @@ func (b *BulkVersionEdit) Apply(
 		}
 
 		if level == 0 {
-			if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 {
+			if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 {
 				// Flushes and ingestions that do not delete any L0 files do not require
 				// a regeneration of L0Sublevels from scratch. We can instead generate
 				// it incrementally.
 				var err error
 				// AddL0Files requires addedFiles to be sorted in seqnum order.
-				addedFiles = append([]*FileMetadata(nil), addedFiles...)
 				SortBySeqNum(addedFiles)
 				v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0])
 				if errors.Is(err, errInvalidL0SublevelsOpt) {
 					err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes)
+				} else if invariants.Enabled && err == nil {
+					copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes)
+					if err != nil {
+						panic(fmt.Sprintf("error when regenerating sublevels: %s", err))
+					}
+					s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels)
+					s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels)
+					if s1 != s2 {
+						panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2))
+					}
 				}
 				if err != nil {
-					return nil, nil, errors.Wrap(err, "pebble: internal error")
+					return nil, errors.Wrap(err, "pebble: internal error")
 				}
 				v.L0SublevelFiles = v.L0Sublevels.Levels
 			} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			continue
 		}
@@ -789,9 +1007,9 @@ func (b *BulkVersionEdit) Apply(
 				}
 			})
 			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 		}
 	}
-	return v, zombies, nil
+	return v, nil
 }

vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go

@@ -2,8 +2,8 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64
-// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64
+//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64
+// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64
 
 package manual
 

vendor/github.com/cockroachdb/pebble/internal/private/flush_external.go

@@ -1,31 +0,0 @@
-// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package private
-
-import "github.com/cockroachdb/pebble/internal/manifest"
-
-// FlushExternalTable is a hook for linking files into L0 without assigning a
-// global sequence number, mimicking a flush. FlushExternalTable takes a
-// *pebble.DB, the path and the metadata of a sstable to flush directly into
-// L0.
-//
-// Calls to flush a sstable may fail if the file's sequence numbers are not
-// greater than the current commit pipeline's sequence number. On success the
-// commit pipeline's published sequence number will be moved to the file's
-// highest sequence number.
-//
-// This function is wrapped in a safer, more ergonomic API in the
-// internal/replay package. Clients should use the replay package rather than
-// calling this private hook directly.
-var FlushExternalTable func(interface{}, string, *manifest.FileMetadata) error
-
-// RatchetSeqNum is a hook for allocating and publishing sequence numbers up
-// to a specific absolute value. Its first parameter is a *pebble.DB and its
-// second is the new next sequence number. RatchetSeqNum does nothing if the
-// next sequence is already greater than or equal to nextSeqNum.
-//
-// This function is used by the internal/replay package to ensure replayed
-// operations receive the same absolute sequence number.
-var RatchetSeqNum func(interface{}, uint64)

vendor/github.com/cockroachdb/pebble/internal/private/sstable.go

@@ -8,7 +8,7 @@ import "github.com/cockroachdb/pebble/internal/base"
 
 // SSTableCacheOpts is a hook for specifying cache options to
 // sstable.NewReader.
-var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{}
+var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{}
 
 // SSTableRawTombstonesOpt is a sstable.Reader option for disabling
 // fragmentation of the range tombstones returned by
@@ -22,7 +22,8 @@ var SSTableRawTombstonesOpt interface{}
 // tool/make_test_sstables.go.
 var SSTableWriterDisableKeyOrderChecks func(interface{})
 
-// SSTableInternalTableOpt is an sstable.Writer option that sets properties for
-// sstables being created by the db itself (i.e. through flushes and
-// compactions), as opposed to those meant for ingestion.
-var SSTableInternalTableOpt interface{}
+// SSTableInternalProperties is a func(*sstable.Writer) *sstable.Properties
+// function that allows Pebble-internal code to mutate properties that external
+// sstable writers are not permitted to edit. It's an untyped interface{} to
+// avoid a cyclic dependency.
+var SSTableInternalProperties interface{}

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -6,6 +6,7 @@ package rangekey
 
 import (
 	"bytes"
+	"math"
 	"sort"
 
 	"github.com/cockroachdb/pebble/internal/base"
@@ -15,7 +16,41 @@ import (
 )
 
 // UserIteratorConfig holds state for constructing the range key iterator stack
-// for user iteration.
+// for user iteration. The range key iterator must merge range key spans across
+// the levels of the LSM. This merging is performed by a keyspan.MergingIter
+// on-the-fly. The UserIteratorConfig implements keyspan.Transformer, evaluating
+// range-key semantics and shadowing, so the spans returned by a MergingIter are
+// fully resolved.
+//
+// The MergingIter is wrapped by a BoundedIter, which elides spans that are
+// outside the iterator bounds (or the current prefix's bounds, during prefix
+// iteration mode).
+//
+// To provide determinisim during iteration, the BoundedIter is wrapped by a
+// DefragmentingIter that defragments abutting spans with identical
+// user-observable state.
+//
+// At the top-level an InterleavingIter interleaves range keys with point keys
+// and performs truncation to iterator bounds.
+//
+// Below is an abbreviated diagram illustrating the mechanics of a SeekGE.
+//
+//	               InterleavingIter.SeekGE
+//	                       │
+//	            DefragmentingIter.SeekGE
+//	                       │
+//	               BoundedIter.SeekGE
+//	                       │
+//	      ╭────────────────┴───────────────╮
+//	      │                                ├── defragmentBwd*
+//	MergingIter.SeekGE                     │
+//	      │                                ╰── defragmentFwd
+//	      ╰─╶╶ per level╶╶ ─╮
+//	                        │
+//	                        │
+//	                        ├── <?>.SeekLT
+//	                        │
+//	                        ╰── <?>.Next
 type UserIteratorConfig struct {
 	snapshot   uint64
 	comparer   *base.Comparer
@@ -24,14 +59,29 @@ type UserIteratorConfig struct {
 	diter      keyspan.DefragmentingIter
 	liters     [manifest.NumLevels]keyspan.LevelIter
 	litersUsed int
-	sortBuf    keysBySuffix
+	onlySets   bool
+	bufs       *Buffers
+}
+
+// Buffers holds various buffers used for range key iteration. They're exposed
+// so that they may be pooled and reused between iterators.
+type Buffers struct {
+	merging       keyspan.MergingBuffers
+	defragmenting keyspan.DefragmentingBuffers
+	sortBuf       keyspan.KeysBySuffix
+}
+
+// PrepareForReuse discards any excessively large buffers.
+func (bufs *Buffers) PrepareForReuse() {
+	bufs.merging.PrepareForReuse()
+	bufs.defragmenting.PrepareForReuse()
 }
 
 // Init initializes the range key iterator stack for user iteration. The
 // resulting fragment iterator applies range key semantics, defragments spans
-// according to their user-observable state and removes all Keys other than
-// RangeKeySets describing the current state of range keys. The resulting spans
-// contain Keys sorted by Suffix.
+// according to their user-observable state and, if onlySets = true, removes all
+// Keys other than RangeKeySets describing the current state of range keys. The
+// resulting spans contain Keys sorted by Suffix.
 //
 // The snapshot sequence number parameter determines which keys are visible. Any
 // keys not visible at the provided snapshot are ignored.
@@ -41,14 +91,18 @@ func (ui *UserIteratorConfig) Init(
 	lower, upper []byte,
 	hasPrefix *bool,
 	prefix *[]byte,
+	onlySets bool,
+	bufs *Buffers,
 	iters ...keyspan.FragmentIterator,
 ) keyspan.FragmentIterator {
 	ui.snapshot = snapshot
 	ui.comparer = comparer
-	ui.miter.Init(comparer.Compare, ui, iters...)
+	ui.onlySets = onlySets
+	ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
 	ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
-	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer)
+	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
 	ui.litersUsed = 0
+	ui.bufs = bufs
 	return &ui.diter
 }
 
@@ -87,17 +141,17 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 	// Apply shadowing of keys.
 	dst.Start = s.Start
 	dst.End = s.End
-	ui.sortBuf = keysBySuffix{
-		cmp:  cmp,
-		keys: dst.Keys[:0],
+	ui.bufs.sortBuf = keyspan.KeysBySuffix{
+		Cmp:  cmp,
+		Keys: ui.bufs.sortBuf.Keys[:0],
 	}
-	if err := coalesce(&ui.sortBuf, s.Visible(ui.snapshot).Keys, &dst.Keys); err != nil {
+	if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
 		return err
 	}
-	// During user iteration over range keys, unsets and deletes don't
-	// matter. Remove them. This step helps logical defragmentation during
-	// iteration.
-	keys := dst.Keys
+	// During user iteration over range keys, unsets and deletes don't matter.
+	// Remove them if onlySets = true. This step helps logical defragmentation
+	// during iteration.
+	keys := ui.bufs.sortBuf.Keys
 	dst.Keys = dst.Keys[:0]
 	for i := range keys {
 		switch keys[i].Kind() {
@@ -110,11 +164,17 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
 				panic("pebble: keys unexpectedly not in ascending suffix order")
 			}
-			// Skip.
-			continue
+			if ui.onlySets {
+				// Skip.
+				continue
+			}
+			dst.Keys = append(dst.Keys, keys[i])
 		case base.InternalKeyKindRangeKeyDelete:
-			// Skip.
-			continue
+			if ui.onlySets {
+				// Skip.
+				continue
+			}
+			dst.Keys = append(dst.Keys, keys[i])
 		default:
 			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
 		}
@@ -148,8 +208,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 	ret := true
 	for i := range a.Keys {
 		if invariants.Enabled {
-			if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
-				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
+			if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
+				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
 				panic("pebble: unexpected non-RangeKeySet during defragmentation")
 			}
 			if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||
@@ -193,8 +253,9 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 //
 // Additionally, internal range keys at the same sequence number have subtle
 // mechanics:
-//   * RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix.
-//   * RANGEKEYDELs only apply to keys at lower sequence numbers.
+//   - RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix.
+//   - RANGEKEYDELs only apply to keys at lower sequence numbers.
+//
 // This is required for ingestion. Ingested sstables are assigned a single
 // sequence number for the file, at which all of the file's keys are visible.
 // The RANGEKEYSET, RANGEKEYUNSET and RANGEKEYDEL key kinds are ordered such
@@ -204,98 +265,105 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 // set or unset but not both.
 //
 // The resulting dst Keys slice is sorted by Trailer.
-func Coalesce(cmp base.Compare, keys []keyspan.Key, dst *[]keyspan.Key) error {
+func Coalesce(cmp base.Compare, eq base.Equal, keys []keyspan.Key, dst *[]keyspan.Key) error {
 	// TODO(jackson): Currently, Coalesce doesn't actually perform the sequence
 	// number promotion described in the comment above.
-	keysBySuffix := keysBySuffix{
-		cmp:  cmp,
-		keys: (*dst)[:0],
+	keysBySuffix := keyspan.KeysBySuffix{
+		Cmp:  cmp,
+		Keys: (*dst)[:0],
 	}
-	if err := coalesce(&keysBySuffix, keys, dst); err != nil {
+	if err := coalesce(eq, &keysBySuffix, math.MaxUint64, keys); err != nil {
 		return err
 	}
-	// coalesce left the keys in *dst sorted by suffix. Re-sort them by trailer.
+	// Update the span with the (potentially reduced) keys slice. coalesce left
+	// the keys in *dst sorted by suffix. Re-sort them by trailer.
+	*dst = keysBySuffix.Keys
 	keyspan.SortKeysByTrailer(dst)
 	return nil
 }
 
-func coalesce(keysBySuffix *keysBySuffix, keys []keyspan.Key, dst *[]keyspan.Key) error {
-	var deleted bool
-	for i := 0; i < len(keys) && !deleted; i++ {
-		k := keys[i]
-		if invariants.Enabled && i > 0 && k.Trailer > keys[i-1].Trailer {
+func coalesce(
+	equal base.Equal, keysBySuffix *keyspan.KeysBySuffix, snapshot uint64, keys []keyspan.Key,
+) error {
+	// First, enforce visibility and RangeKeyDelete mechanics. We only need to
+	// consider the prefix of keys before and including the first
+	// RangeKeyDelete. We also must skip any keys that aren't visible at the
+	// provided snapshot sequence number.
+	//
+	// NB: Within a given sequence number, keys are ordered as:
+	//   RangeKeySet > RangeKeyUnset > RangeKeyDelete
+	// This is significant, because this ensures that a Set or Unset sharing a
+	// sequence number with a Delete do not shadow each other.
+	deleteIdx := -1
+	for i := range keys {
+		if invariants.Enabled && i > 0 && keys[i].Trailer > keys[i-1].Trailer {
 			panic("pebble: invariant violation: span keys unordered")
 		}
+		if !keys[i].VisibleAt(snapshot) {
+			continue
+		}
+		// Once a RangeKeyDelete is observed, we know it shadows all subsequent
+		// keys and we can break early. We don't add the RangeKeyDelete key to
+		// keysBySuffix.keys yet, because we don't want a suffix-less key
+		// that appeared earlier in the slice to elide it. It'll be added back
+		// in at the end.
+		if keys[i].Kind() == base.InternalKeyKindRangeKeyDelete {
+			deleteIdx = i
+			break
+		}
+		keysBySuffix.Keys = append(keysBySuffix.Keys, keys[i])
+	}
 
-		// NB: Within a given sequence number, keys are ordered as:
-		//   RangeKeySet > RangeKeyUnset > RangeKeyDelete
-		// This is significant, because this ensures that none of the range keys
-		// sharing a sequence number shadow each other.
-		switch k.Kind() {
-		case base.InternalKeyKindRangeKeySet:
-			n := len(keysBySuffix.keys)
+	// Sort the accumulated keys by suffix. There may be duplicates within a
+	// suffix, in which case the one with a larger trailer survives.
+	//
+	// We use a stable sort so that the first key with a given suffix is the one
+	// that with the highest Trailer (because the input `keys` was sorted by
+	// trailer descending).
+	sort.Stable(keysBySuffix)
 
-			if keysBySuffix.get(n, k.Suffix) < n {
-				// This suffix is already set or unset at a higher sequence
-				// number. Skip.
-				continue
+	// Grab a handle of the full sorted slice, before reslicing
+	// keysBySuffix.keys to accumulate the final coalesced keys.
+	sorted := keysBySuffix.Keys
+	keysBySuffix.Keys = keysBySuffix.Keys[:0]
+
+	var (
+		// prevSuffix is updated on each iteration of the below loop, and
+		// compared by the subsequent iteration to determine whether adjacent
+		// keys are defined at the same suffix.
+		prevSuffix []byte
+		// shadowing is set to true once any Key is shadowed by another key.
+		// When it's set to true—or after the loop if no keys are shadowed—the
+		// keysBySuffix.keys slice is resliced to contain the prefix of
+		// unshadowed keys. This avoids copying them incrementally in the common
+		// case of no shadowing.
+		shadowing bool
+	)
+	for i := range sorted {
+		if i > 0 && equal(prevSuffix, sorted[i].Suffix) {
+			// Skip; this key is shadowed by the predecessor that had a larger
+			// Trailer. If this is the first shadowed key, set shadowing=true
+			// and reslice keysBySuffix.keys to hold the entire unshadowed
+			// prefix.
+			if !shadowing {
+				keysBySuffix.Keys = keysBySuffix.Keys[:i]
+				shadowing = true
 			}
-			keysBySuffix.keys = append(keysBySuffix.keys, k)
-			sort.Sort(keysBySuffix)
-		case base.InternalKeyKindRangeKeyUnset:
-			n := len(keysBySuffix.keys)
-
-			if keysBySuffix.get(n, k.Suffix) < n {
-				// This suffix is already set or unset at a higher sequence
-				// number. Skip.
-				continue
-			}
-			keysBySuffix.keys = append(keysBySuffix.keys, k)
-			sort.Sort(keysBySuffix)
-		case base.InternalKeyKindRangeKeyDelete:
-			// All remaining range keys in this span have been deleted by this
-			// RangeKeyDelete. There's no need to continue looping, because all
-			// the remaining keys are shadowed by this one. The for loop
-			// condition will terminate when it sees the last key is a
-			// range key deletion.
-			keysBySuffix.keys = append(keysBySuffix.keys, k)
-			deleted = true
-		default:
-			return base.CorruptionErrorf("pebble: unexpected range key kind %s", k.Kind())
+			continue
+		}
+		prevSuffix = sorted[i].Suffix
+		if shadowing {
+			keysBySuffix.Keys = append(keysBySuffix.Keys, sorted[i])
 		}
 	}
-
-	// Update the span with the (potentially reduced) keys slice.
-	// NB: We don't re-sort by Trailer. The exported Coalesce function however
-	// will.
-	*dst = keysBySuffix.keys
+	// If there was no shadowing, keysBySuffix.keys is untouched. We can simply
+	// set it to the existing `sorted` slice (also backed by keysBySuffix.keys).
+	if !shadowing {
+		keysBySuffix.Keys = sorted
+	}
+	// If the original input `keys` slice contained a RangeKeyDelete, add it.
+	if deleteIdx >= 0 {
+		keysBySuffix.Keys = append(keysBySuffix.Keys, keys[deleteIdx])
+	}
 	return nil
 }
-
-type keysBySuffix struct {
-	cmp  base.Compare
-	keys []keyspan.Key
-}
-
-// get searches for suffix among the first n keys in keys. If the suffix is
-// found, it returns the index of the item with the suffix. If the suffix is not
-// found, it returns n.
-func (s *keysBySuffix) get(n int, suffix []byte) (i int) {
-	// Binary search for the suffix to see if there's an existing key with the
-	// suffix. Only binary search among the first n items. get is called while
-	// appending new keys with suffixes that may sort before existing keys.
-	// The n parameter indicates what portion of the keys slice is sorted and
-	// may contain relevant keys.
-
-	i = sort.Search(n, func(i int) bool {
-		return s.cmp(s.keys[i].Suffix, suffix) >= 0
-	})
-	if i < n && s.cmp(s.keys[i].Suffix, suffix) == 0 {
-		return i
-	}
-	return n
-}
-
-func (s *keysBySuffix) Len() int           { return len(s.keys) }
-func (s *keysBySuffix) Less(i, j int) bool { return s.cmp(s.keys[i].Suffix, s.keys[j].Suffix) < 0 }
-func (s *keysBySuffix) Swap(i, j int)      { s.keys[i], s.keys[j] = s.keys[j], s.keys[i] }

vendor/github.com/cockroachdb/pebble/internal/rate/rate.go

@@ -196,13 +196,15 @@ func (lim *Limiter) Reserve() Reservation {
 // The Limiter takes this Reservation into account when allowing future events.
 // ReserveN returns false if n exceeds the Limiter's burst size.
 // Usage example:
-//   r := lim.ReserveN(time.Now(), 1)
-//   if !r.OK() {
-//     // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
-//     return
-//   }
-//   time.Sleep(r.Delay())
-//   Act()
+//
+//	r := lim.ReserveN(time.Now(), 1)
+//	if !r.OK() {
+//	  // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
+//	  return
+//	}
+//	time.Sleep(r.Delay())
+//	Act()
+//
 // Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
 // If you need to respect a deadline or cancel the delay, use Wait instead.
 // To drop or skip events exceeding rate limit, use Allow instead.

vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_64bit.go

@@ -12,8 +12,8 @@
 // implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64
-// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64
+//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64
+// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64
 
 package rawalloc