Update deps

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -20,6 +20,11 @@ type FileNum uint64
 // String returns a string representation of the file number.
 func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
 
+// SafeFormat implements redact.SafeFormatter.
+func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%06d", redact.SafeUint(fn))
+}
+
 // DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
 // called if the caller can ensure that the FileNum belongs to a physical file
 // on disk. These could be manifests, log files, physical sstables on disk, the
@@ -40,6 +45,11 @@ type DiskFileNum struct {
 
 func (dfn DiskFileNum) String() string { return dfn.fn.String() }
 
+// SafeFormat implements redact.SafeFormatter.
+func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
+	dfn.fn.SafeFormat(w, verb)
+}
+
 // FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
 func (dfn DiskFileNum) FileNum() FileNum {
 	return dfn.fn

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/cockroachdb/redact"
 )
 
 const (
@@ -161,6 +163,11 @@ func (k InternalKeyKind) String() string {
 	return fmt.Sprintf("UNKNOWN:%d", k)
 }
 
+// SafeFormat implements redact.SafeFormatter.
+func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Print(redact.SafeString(k.String()))
+}
+
 // InternalKey is a key used for the in-memory and on-disk partial DBs that
 // make up a pebble DB.
 //
@@ -374,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
 	return k.Trailer >> 8
 }
 
+// SeqNumFromTrailer returns the sequence number component of a trailer.
+func SeqNumFromTrailer(t uint64) uint64 {
+	return t >> 8
+}
+
 // Visible returns true if the key is visible at the specified snapshot
 // sequence number.
 func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {

vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package invalidating
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/fastrand"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
+// MaybeWrapIfInvariants does nothing in non-invariant builds.
+func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
+	if invariants.Enabled {
+		if fastrand.Uint32n(10) == 1 {
+			return NewIter(iter)
+		}
+	}
+	return iter
+}
+
+// iter tests unsafe key/value slice reuse by modifying the last
+// returned key/value to all 1s.
+type iter struct {
+	iter        base.InternalIterator
+	lastKey     *base.InternalKey
+	lastValue   base.LazyValue
+	ignoreKinds [base.InternalKeyKindMax + 1]bool
+	err         error
+}
+
+// Option configures the behavior of an invalidating iterator.
+type Option interface {
+	apply(*iter)
+}
+
+type funcOpt func(*iter)
+
+func (f funcOpt) apply(i *iter) { f(i) }
+
+// IgnoreKinds constructs an Option that configures an invalidating iterator to
+// skip trashing k/v pairs with the provided key kinds. Some iterators provided
+// key stability guarantees for specific key kinds.
+func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
+	return funcOpt(func(i *iter) {
+		for _, kind := range kinds {
+			i.ignoreKinds[kind] = true
+		}
+	})
+}
+
+// NewIter constructs a new invalidating iterator that wraps the provided
+// iterator, trashing buffers for previously returned keys.
+func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
+	i := &iter{iter: originalIterator}
+	for _, opt := range opts {
+		opt.apply(i)
+	}
+	return i
+}
+
+func (i *iter) update(
+	key *base.InternalKey, value base.LazyValue,
+) (*base.InternalKey, base.LazyValue) {
+	i.trashLastKV()
+	if key == nil {
+		i.lastKey = nil
+		i.lastValue = base.LazyValue{}
+		return nil, base.LazyValue{}
+	}
+
+	i.lastKey = &base.InternalKey{}
+	*i.lastKey = key.Clone()
+	i.lastValue = base.LazyValue{
+		ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
+	}
+	if value.Fetcher != nil {
+		fetcher := new(base.LazyFetcher)
+		*fetcher = *value.Fetcher
+		i.lastValue.Fetcher = fetcher
+	}
+	return i.lastKey, i.lastValue
+}
+
+func (i *iter) trashLastKV() {
+	if i.lastKey == nil {
+		return
+	}
+	if i.ignoreKinds[i.lastKey.Kind()] {
+		return
+	}
+
+	if i.lastKey != nil {
+		for j := range i.lastKey.UserKey {
+			i.lastKey.UserKey[j] = 0xff
+		}
+		i.lastKey.Trailer = 0xffffffffffffffff
+	}
+	for j := range i.lastValue.ValueOrHandle {
+		i.lastValue.ValueOrHandle[j] = 0xff
+	}
+	if i.lastValue.Fetcher != nil {
+		// Not all the LazyFetcher fields are visible, so we zero out the last
+		// value's Fetcher struct entirely.
+		*i.lastValue.Fetcher = base.LazyFetcher{}
+	}
+}
+
+func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekGE(key, flags))
+}
+
+func (i *iter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
+}
+
+func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekLT(key, flags))
+}
+
+func (i *iter) First() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.First())
+}
+
+func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Last())
+}
+
+func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Next())
+}
+
+func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Prev())
+}
+
+func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.NextPrefix(succKey))
+}
+
+func (i *iter) Error() error {
+	if err := i.iter.Error(); err != nil {
+		return err
+	}
+	return i.err
+}
+
+func (i *iter) Close() error {
+	return i.iter.Close()
+}
+
+func (i *iter) SetBounds(lower, upper []byte) {
+	i.iter.SetBounds(lower, upper)
+}
+
+func (i *iter) String() string {
+	return i.iter.String()
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 	// fragments before this span. Defragment backward to find the start of the
 	// defragmented span.
 	i.defragmentBackward()
+
+	// Defragmenting backward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosPrev {
 		// Next once back onto the span.
 		i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 	// fragments after this span. Defragment forward to find the end of the
 	// defragmented span.
 	i.defragmentForward()
+
+	// Defragmenting forward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosNext {
 		// Prev once back onto the span.
 		i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
 		// Next once to move onto y, defragment forward to land on the first z
 		// position.
 		i.iterSpan = i.iter.Next()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no next span while switching directions")
 		}
 		// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
 		// Prev once to move onto y, defragment backward to land on the last x
 		// position.
 		i.iterSpan = i.iter.Prev()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no previous span while switching directions")
 		}
 		// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Next()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Prev()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go

@@ -4,6 +4,8 @@
 
 package keyspan
 
+import "github.com/cockroachdb/pebble/internal/base"
+
 // FilterFunc defines a transform from the input Span into the output Span. The
 // function returns true if the Span should be returned by the iterator, and
 // false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
 type filteringIter struct {
 	iter     FragmentIterator
 	filterFn FilterFunc
+	cmp      base.Compare
 
 	// span is a mutable Span passed to the filterFn. The filterFn is free to
 	// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
 
 // Filter returns a new filteringIter that will filter the Spans from the
 // provided child iterator using the provided FilterFunc.
-func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
-	return &filteringIter{iter: iter, filterFn: filter}
+func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
+	return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
 }
 
 // SeekGE implements FragmentIterator.
 func (i *filteringIter) SeekGE(key []byte) *Span {
-	return i.filter(i.iter.SeekGE(key), +1)
+	span := i.filter(i.iter.SeekGE(key), +1)
+	// i.filter could return a span that's less than key, _if_ the filterFunc
+	// (which has no knowledge of the seek key) mutated the span to end at a key
+	// less than or equal to `key`. Detect this case and next/invalidate the iter.
+	if span != nil && i.cmp(span.End, key) <= 0 {
+		return i.Next()
+	}
+	return span
 }
 
 // SeekLT implements FragmentIterator.
 func (i *filteringIter) SeekLT(key []byte) *Span {
-	return i.filter(i.iter.SeekLT(key), -1)
+	span := i.filter(i.iter.SeekLT(key), -1)
+	// i.filter could return a span that's >= key, _if_ the filterFunc (which has
+	// no knowledge of the seek key) mutated the span to start at a key greater
+	// than or equal to `key`. Detect this case and prev/invalidate the iter.
+	if span != nil && i.cmp(span.Start, key) >= 0 {
+		return i.Prev()
+	}
+	return span
 }
 
 // First implements FragmentIterator.

vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go

@@ -7,6 +7,7 @@ package keyspan
 import (
 	"fmt"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 )
@@ -111,6 +112,22 @@ type InterleavingIter struct {
 	nextPrefixBuf []byte
 	pointKey      *base.InternalKey
 	pointVal      base.LazyValue
+	// err holds an iterator error from either pointIter or keyspanIter. It's
+	// reset to nil on seeks. An overview of error-handling mechanics:
+	//
+	// Whenever either pointIter or keyspanIter is respositioned and a nil
+	// key/span is returned, the code performing the positioning is responsible
+	// for checking the iterator's Error() value. This happens in savePoint and
+	// saveSpan[Forward,Backward].
+	//
+	// Once i.err is non-nil, the computation of i.pos must set i.pos =
+	// posExhausted. This happens in compute[Smallest|Largest]Pos and
+	// [next|prev]Pos. Setting i.pos to posExhausted ensures we'll yield nil to
+	// the caller, which they'll interpret as a signal they must check Error().
+	//
+	// INVARIANTS:
+	// i.err != nil => i.pos = posExhausted
+	err error
 	// prefix records the iterator's current prefix if the iterator is in prefix
 	// mode. During prefix mode, Pebble will truncate spans to the next prefix.
 	// If the iterator subsequently leaves prefix mode, the existing span cached
@@ -216,7 +233,7 @@ func (i *InterleavingIter) InitSeekGE(
 	i.dir = +1
 	i.clearMask()
 	i.prefix = prefix
-	i.pointKey, i.pointVal = pointKey, pointValue
+	i.savePoint(pointKey, pointValue)
 	// NB: This keyspanSeekGE call will truncate the span to the seek key if
 	// necessary. This truncation is important for cases where a switch to
 	// combined iteration is made during a user-initiated SeekGE.
@@ -240,7 +257,7 @@ func (i *InterleavingIter) InitSeekLT(
 ) (*base.InternalKey, base.LazyValue) {
 	i.dir = -1
 	i.clearMask()
-	i.pointKey, i.pointVal = pointKey, pointValue
+	i.savePoint(pointKey, pointValue)
 	i.keyspanSeekLT(key)
 	i.computeLargestPos()
 	return i.yieldPosition(i.lower, i.prevPos)
@@ -260,9 +277,10 @@ func (i *InterleavingIter) InitSeekLT(
 func (i *InterleavingIter) SeekGE(
 	key []byte, flags base.SeekGEFlags,
 ) (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.SeekGE(key, flags)
+	i.savePoint(i.pointIter.SeekGE(key, flags))
 
 	// We need to seek the keyspan iterator too. If the keyspan iterator was
 	// already positioned at a span, we might be able to avoid the seek if the
@@ -270,7 +288,7 @@ func (i *InterleavingIter) SeekGE(
 	if i.span != nil && i.cmp(key, i.span.End) < 0 && i.cmp(key, i.span.Start) >= 0 {
 		// We're seeking within the existing span's bounds. We still might need
 		// truncate the span to the iterator's bounds.
-		i.checkForwardBound()
+		i.saveSpanForward(i.span)
 		i.savedKeyspan()
 	} else {
 		i.keyspanSeekGE(key, nil /* prefix */)
@@ -295,9 +313,10 @@ func (i *InterleavingIter) SeekGE(
 func (i *InterleavingIter) SeekPrefixGE(
 	prefix, key []byte, flags base.SeekGEFlags,
 ) (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.prefix = prefix
-	i.pointKey, i.pointVal = i.pointIter.SeekPrefixGE(prefix, key, flags)
+	i.savePoint(i.pointIter.SeekPrefixGE(prefix, key, flags))
 
 	// We need to seek the keyspan iterator too. If the keyspan iterator was
 	// already positioned at a span, we might be able to avoid the seek if the
@@ -326,7 +345,7 @@ func (i *InterleavingIter) SeekPrefixGE(
 		if ei := i.comparer.Split(i.span.End); i.cmp(prefix, i.span.End[:ei]) < 0 {
 			// We're seeking within the existing span's bounds. We still might need
 			// truncate the span to the iterator's bounds.
-			i.checkForwardBound()
+			i.saveSpanForward(i.span)
 			i.savedKeyspan()
 			seekKeyspanIter = false
 		}
@@ -344,9 +363,10 @@ func (i *InterleavingIter) SeekPrefixGE(
 func (i *InterleavingIter) SeekLT(
 	key []byte, flags base.SeekLTFlags,
 ) (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.SeekLT(key, flags)
+	i.savePoint(i.pointIter.SeekLT(key, flags))
 
 	// We need to seek the keyspan iterator too. If the keyspan iterator was
 	// already positioned at a span, we might be able to avoid the seek if the
@@ -354,13 +374,13 @@ func (i *InterleavingIter) SeekLT(
 	if i.span != nil && i.cmp(key, i.span.Start) > 0 && i.cmp(key, i.span.End) < 0 {
 		// We're seeking within the existing span's bounds. We still might need
 		// truncate the span to the iterator's bounds.
-		i.checkBackwardBound()
+		i.saveSpanBackward(i.span)
 		// The span's start key is still not guaranteed to be less than key,
 		// because of the bounds enforcement. Consider the following example:
 		//
 		// Bounds are set to [d,e). The user performs a SeekLT(d). The
 		// FragmentIterator.SeekLT lands on a span [b,f). This span has a start
-		// key less than d, as expected. Above, checkBackwardBound truncates the
+		// key less than d, as expected. Above, saveSpanBackward truncates the
 		// span to match the iterator's current bounds, modifying the span to
 		// [d,e), which does not overlap the search space of [-∞, d).
 		//
@@ -382,11 +402,11 @@ func (i *InterleavingIter) SeekLT(
 
 // First implements (base.InternalIterator).First.
 func (i *InterleavingIter) First() (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.First()
-	i.span = i.keyspanIter.First()
-	i.checkForwardBound()
+	i.savePoint(i.pointIter.First())
+	i.saveSpanForward(i.keyspanIter.First())
 	i.savedKeyspan()
 	i.dir = +1
 	i.computeSmallestPos()
@@ -395,11 +415,11 @@ func (i *InterleavingIter) First() (*base.InternalKey, base.LazyValue) {
 
 // Last implements (base.InternalIterator).Last.
 func (i *InterleavingIter) Last() (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.Last()
-	i.span = i.keyspanIter.Last()
-	i.checkBackwardBound()
+	i.savePoint(i.pointIter.Last())
+	i.saveSpanBackward(i.keyspanIter.Last())
 	i.savedKeyspan()
 	i.dir = -1
 	i.computeLargestPos()
@@ -436,8 +456,7 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) {
 			// entirely behind the current key (!i.withinSpan), then we
 			// need to move it to the first span in the forward direction.
 			if !i.withinSpan {
-				i.span = i.keyspanIter.Next()
-				i.checkForwardBound()
+				i.saveSpanForward(i.keyspanIter.Next())
 				i.savedKeyspan()
 			}
 		case posKeyspanStart:
@@ -445,12 +464,12 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) {
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely behind the current iterator position. Reposition it
 			// ahead of the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Next()
+			i.savePoint(i.pointIter.Next())
 		case posKeyspanEnd:
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely behind of the current iterator position. Reposition it
 			// ahead the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Next()
+			i.savePoint(i.pointIter.Next())
 		}
 		// Fallthrough to calling i.nextPos.
 	}
@@ -468,7 +487,7 @@ func (i *InterleavingIter) NextPrefix(succKey []byte) (*base.InternalKey, base.L
 	case posExhausted:
 		return nil, base.LazyValue{}
 	case posPointKey:
-		i.pointKey, i.pointVal = i.pointIter.NextPrefix(succKey)
+		i.savePoint(i.pointIter.NextPrefix(succKey))
 		if i.withinSpan {
 			if i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) <= 0 {
 				i.pos = posKeyspanEnd
@@ -514,15 +533,14 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
 			// current key (!i.withinSpan), then we need to move it to the first
 			// span in the reverse direction.
 			if !i.withinSpan {
-				i.span = i.keyspanIter.Prev()
-				i.checkBackwardBound()
+				i.saveSpanBackward(i.keyspanIter.Prev())
 				i.savedKeyspan()
 			}
 		case posKeyspanStart:
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely ahead of the current iterator position. Reposition it
 			// behind the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Prev()
+			i.savePoint(i.pointIter.Prev())
 			// Without considering truncation of spans to seek keys, the keyspan
 			// iterator is already in the right place. But consider span [a, z)
 			// and this sequence of iterator calls:
@@ -544,7 +562,7 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely ahead of the current iterator position. Reposition it
 			// behind the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Prev()
+			i.savePoint(i.pointIter.Prev())
 		}
 
 		if i.spanMarkerTruncated {
@@ -561,15 +579,17 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
 //
 //	MIN(i.pointKey, i.span.Start)
 func (i *InterleavingIter) computeSmallestPos() {
-	if i.span != nil && (i.pointKey == nil || i.cmp(i.startKey(), i.pointKey.UserKey) <= 0) {
-		i.withinSpan = true
-		i.pos = posKeyspanStart
-		return
-	}
-	i.withinSpan = false
-	if i.pointKey != nil {
-		i.pos = posPointKey
-		return
+	if i.err == nil {
+		if i.span != nil && (i.pointKey == nil || i.cmp(i.startKey(), i.pointKey.UserKey) <= 0) {
+			i.withinSpan = true
+			i.pos = posKeyspanStart
+			return
+		}
+		i.withinSpan = false
+		if i.pointKey != nil {
+			i.pos = posPointKey
+			return
+		}
 	}
 	i.pos = posExhausted
 }
@@ -578,30 +598,52 @@ func (i *InterleavingIter) computeSmallestPos() {
 //
 //	MAX(i.pointKey, i.span.End)
 func (i *InterleavingIter) computeLargestPos() {
-	if i.span != nil && (i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) > 0) {
-		i.withinSpan = true
-		i.pos = posKeyspanEnd
-		return
-	}
-	i.withinSpan = false
-	if i.pointKey != nil {
-		i.pos = posPointKey
-		return
+	if i.err == nil {
+		if i.span != nil && (i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) > 0) {
+			i.withinSpan = true
+			i.pos = posKeyspanEnd
+			return
+		}
+		i.withinSpan = false
+		if i.pointKey != nil {
+			i.pos = posPointKey
+			return
+		}
 	}
 	i.pos = posExhausted
 }
 
 // nextPos advances the iterator one position in the forward direction.
 func (i *InterleavingIter) nextPos() {
+	if invariants.Enabled {
+		defer func() {
+			if i.err != nil && i.pos != posExhausted {
+				panic(errors.AssertionFailedf("iterator has accumulated error but i.pos = %d", i.pos))
+			}
+		}()
+	}
+	// NB: If i.err != nil or any of the positioning methods performed in this
+	// function result in i.err != nil, we must set i.pos = posExhausted. We
+	// perform this check explicitly here, but if any of the branches below
+	// advance either iterator, they must also check i.err and set posExhausted
+	// if necessary.
+	if i.err != nil {
+		i.pos = posExhausted
+		return
+	}
+
 	switch i.pos {
 	case posExhausted:
-		i.pointKey, i.pointVal = i.pointIter.Next()
-		i.span = i.keyspanIter.Next()
-		i.checkForwardBound()
+		i.savePoint(i.pointIter.Next())
+		i.saveSpanForward(i.keyspanIter.Next())
 		i.savedKeyspan()
 		i.computeSmallestPos()
 	case posPointKey:
-		i.pointKey, i.pointVal = i.pointIter.Next()
+		i.savePoint(i.pointIter.Next())
+		if i.err != nil {
+			i.pos = posExhausted
+			return
+		}
 		// If we're not currently within the span, we want to chose the
 		// MIN(pointKey,span.Start), which is exactly the calculation performed
 		// by computeSmallestPos.
@@ -635,26 +677,45 @@ func (i *InterleavingIter) nextPos() {
 			i.pos = posKeyspanEnd
 		}
 	case posKeyspanEnd:
-		i.span = i.keyspanIter.Next()
-		i.checkForwardBound()
+		i.saveSpanForward(i.keyspanIter.Next())
 		i.savedKeyspan()
 		i.computeSmallestPos()
 	default:
-		panic(fmt.Sprintf("unexpected pos=%d\n", i.pos))
+		panic(fmt.Sprintf("unexpected pos=%d", i.pos))
 	}
 }
 
 // prevPos advances the iterator one position in the reverse direction.
 func (i *InterleavingIter) prevPos() {
+	if invariants.Enabled {
+		defer func() {
+			if i.err != nil && i.pos != posExhausted {
+				panic(errors.AssertionFailedf("iterator has accumulated error but i.pos = %d", i.pos))
+			}
+		}()
+	}
+	// NB: If i.err != nil or any of the positioning methods performed in this
+	// function result in i.err != nil, we must set i.pos = posExhausted. We
+	// perform this check explicitly here, but if any of the branches below
+	// advance either iterator, they must also check i.err and set posExhausted
+	// if necessary.
+	if i.err != nil {
+		i.pos = posExhausted
+		return
+	}
+
 	switch i.pos {
 	case posExhausted:
-		i.pointKey, i.pointVal = i.pointIter.Prev()
-		i.span = i.keyspanIter.Prev()
-		i.checkBackwardBound()
+		i.savePoint(i.pointIter.Prev())
+		i.saveSpanBackward(i.keyspanIter.Prev())
 		i.savedKeyspan()
 		i.computeLargestPos()
 	case posPointKey:
-		i.pointKey, i.pointVal = i.pointIter.Prev()
+		i.savePoint(i.pointIter.Prev())
+		if i.err != nil {
+			i.pos = posExhausted
+			return
+		}
 		// If we're not currently covered by the span, we want to chose the
 		// MAX(pointKey,span.End), which is exactly the calculation performed
 		// by computeLargestPos.
@@ -676,8 +737,7 @@ func (i *InterleavingIter) prevPos() {
 			}
 		}
 	case posKeyspanStart:
-		i.span = i.keyspanIter.Prev()
-		i.checkBackwardBound()
+		i.saveSpanBackward(i.keyspanIter.Prev())
 		i.savedKeyspan()
 		i.computeLargestPos()
 	case posKeyspanEnd:
@@ -688,7 +748,7 @@ func (i *InterleavingIter) prevPos() {
 			i.pos = posKeyspanStart
 		}
 	default:
-		panic(fmt.Sprintf("unexpected pos=%d\n", i.pos))
+		panic(fmt.Sprintf("unexpected pos=%d", i.pos))
 	}
 }
 
@@ -760,21 +820,19 @@ func (i *InterleavingIter) yieldPosition(
 
 // keyspanSeekGE seeks the keyspan iterator to the first span covering a key ≥ k.
 func (i *InterleavingIter) keyspanSeekGE(k []byte, prefix []byte) {
-	i.span = i.keyspanIter.SeekGE(k)
-	i.checkForwardBound()
+	i.saveSpanForward(i.keyspanIter.SeekGE(k))
 	i.savedKeyspan()
 }
 
 // keyspanSeekLT seeks the keyspan iterator to the last span covering a key < k.
 func (i *InterleavingIter) keyspanSeekLT(k []byte) {
-	i.span = i.keyspanIter.SeekLT(k)
-	i.checkBackwardBound()
+	i.saveSpanBackward(i.keyspanIter.SeekLT(k))
 	// The current span's start key is not guaranteed to be less than key,
 	// because of the bounds enforcement. Consider the following example:
 	//
 	// Bounds are set to [d,e). The user performs a SeekLT(d). The
 	// FragmentIterator.SeekLT lands on a span [b,f). This span has a start key
-	// less than d, as expected. Above, checkBackwardBound truncates the span to
+	// less than d, as expected. Above, saveSpanBackward truncates the span to
 	// match the iterator's current bounds, modifying the span to [d,e), which
 	// does not overlap the search space of [-∞, d).
 	//
@@ -786,12 +844,21 @@ func (i *InterleavingIter) keyspanSeekLT(k []byte) {
 	i.savedKeyspan()
 }
 
-func (i *InterleavingIter) checkForwardBound() {
+func (i *InterleavingIter) saveSpanForward(span *Span) {
+	i.span = span
 	i.truncated = false
 	i.truncatedSpan = Span{}
 	if i.span == nil {
+		i.err = firstError(i.err, i.keyspanIter.Error())
 		return
 	}
+	if invariants.Enabled {
+		if err := i.keyspanIter.Error(); err != nil {
+			panic(errors.WithSecondaryError(
+				errors.AssertionFailedf("pebble: %T keyspan iterator returned non-nil span %s while iter has error", i.keyspanIter, i.span),
+				err))
+		}
+	}
 	// Check the upper bound if we have one.
 	if i.upper != nil && i.cmp(i.span.Start, i.upper) >= 0 {
 		i.span = nil
@@ -839,12 +906,22 @@ func (i *InterleavingIter) checkForwardBound() {
 	}
 }
 
-func (i *InterleavingIter) checkBackwardBound() {
+func (i *InterleavingIter) saveSpanBackward(span *Span) {
+	i.span = span
 	i.truncated = false
 	i.truncatedSpan = Span{}
 	if i.span == nil {
+		i.err = firstError(i.err, i.keyspanIter.Error())
 		return
 	}
+	if invariants.Enabled {
+		if err := i.keyspanIter.Error(); err != nil {
+			panic(errors.WithSecondaryError(
+				errors.AssertionFailedf("pebble: %T keyspan iterator returned non-nil span %s while iter has error", i.keyspanIter, i.span),
+				err))
+		}
+	}
+
 	// Check the lower bound if we have one.
 	if i.lower != nil && i.cmp(i.span.End, i.lower) <= 0 {
 		i.span = nil
@@ -947,6 +1024,12 @@ func (i *InterleavingIter) verify(
 			panic("pebble: invariant violation: key < lower bound")
 		case k != nil && i.upper != nil && i.cmp(k.UserKey, i.upper) >= 0:
 			panic("pebble: invariant violation: key ≥ upper bound")
+		case i.err != nil && k != nil:
+			panic("pebble: invariant violation: accumulated error swallowed")
+		case i.err == nil && i.pointIter.Error() != nil:
+			panic("pebble: invariant violation: pointIter swallowed")
+		case i.err == nil && i.keyspanIter.Error() != nil:
+			panic("pebble: invariant violation: keyspanIter error swallowed")
 		}
 	}
 	return k, v
@@ -990,6 +1073,20 @@ func (i *InterleavingIter) startKey() []byte {
 	return i.span.Start
 }
 
+func (i *InterleavingIter) savePoint(key *base.InternalKey, value base.LazyValue) {
+	i.pointKey, i.pointVal = key, value
+	if key == nil {
+		i.err = firstError(i.err, i.pointIter.Error())
+	}
+	if invariants.Enabled {
+		if err := i.pointIter.Error(); key != nil && err != nil {
+			panic(errors.WithSecondaryError(
+				errors.AssertionFailedf("pebble: %T point iterator returned non-nil key %q while iter has error", i.pointIter, key),
+				err))
+		}
+	}
+}
+
 // Span returns the span covering the last key returned, if any. A span key is
 // considered to 'cover' a key if the key falls within the span's user key
 // bounds. The returned span is owned by the InterleavingIter. The caller is
@@ -1023,7 +1120,7 @@ func (i *InterleavingIter) Invalidate() {
 
 // Error implements (base.InternalIterator).Error.
 func (i *InterleavingIter) Error() error {
-	return firstError(i.pointIter.Error(), i.keyspanIter.Error())
+	return i.err
 }
 
 // Close implements (base.InternalIterator).Close.

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -51,6 +51,8 @@ type FragmentIterator interface {
 	Prev() *Span
 
 	// Error returns any accumulated error.
+	//
+	// TODO(jackson): Lift errors into return values on the positioning methods.
 	Error() error
 
 	// Close closes the iterator and returns any accumulated error. Exhausting

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -69,5 +69,5 @@ func Truncate(
 		}
 
 		return !out.Empty() && cmp(out.Start, out.End) < 0
-	})
+	}, cmp)
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go

@@ -195,7 +195,7 @@ func (n *node) incRef() {
 // new nodes pass contentsToo=false to preserve existing reference counts during
 // operations that should yield a net-zero change to descendant refcounts.
 // When a node is released, its contained files are dereferenced.
-func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
+func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
 	if n.ref.Add(-1) > 0 {
 		// Other references remain. Can't free.
 		return
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
 				if obsolete == nil {
 					panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
 				}
-				*obsolete = append(*obsolete, f)
+				// Reference counting is performed on the FileBacking. In the case
+				// of a virtual sstable, this reference counting is performed on
+				// a FileBacking which is shared by every single virtual sstable
+				// with the same backing sstable. If the reference count hits 0,
+				// then we know that the FileBacking won't be required by any
+				// sstable in Pebble, and that the backing sstable can be deleted.
+				*obsolete = append(*obsolete, f.FileBacking)
 			}
 		}
 		if !n.leaf {
@@ -761,8 +767,8 @@ type btree struct {
 
 // Release dereferences and clears the root node of the btree, removing all
 // items from the btree. In doing so, it decrements contained file counts.
-// It returns a slice of newly obsolete files, if any.
-func (t *btree) Release() (obsolete []*FileMetadata) {
+// It returns a slice of newly obsolete backing files, if any.
+func (t *btree) Release() (obsolete []*FileBacking) {
 	if t.root != nil {
 		t.root.decRef(true /* contentsToo */, &obsolete)
 		t.root = nil

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -17,20 +17,26 @@ import (
 type LevelMetadata struct {
 	level     int
 	totalSize uint64
-	tree      btree
+	// NumVirtual is the number of virtual sstables in the level.
+	NumVirtual uint64
+	// VirtualSize is the size of the virtual sstables in the level.
+	VirtualSize uint64
+	tree        btree
 }
 
 // clone makes a copy of the level metadata, implicitly increasing the ref
 // count of every file contained within lm.
 func (lm *LevelMetadata) clone() LevelMetadata {
 	return LevelMetadata{
-		level:     lm.level,
-		totalSize: lm.totalSize,
-		tree:      lm.tree.Clone(),
+		level:       lm.level,
+		totalSize:   lm.totalSize,
+		NumVirtual:  lm.NumVirtual,
+		VirtualSize: lm.VirtualSize,
+		tree:        lm.tree.Clone(),
 	}
 }
 
-func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
+func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
 	return lm.tree.Release()
 }
 
@@ -44,6 +50,10 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
 	lm.tree, _ = makeBTree(bcmp, files)
 	for _, f := range files {
 		lm.totalSize += f.Size
+		if f.Virtual {
+			lm.NumVirtual++
+			lm.VirtualSize += f.Size
+		}
 	}
 	return lm
 }
@@ -62,11 +72,19 @@ func (lm *LevelMetadata) insert(f *FileMetadata) error {
 		return err
 	}
 	lm.totalSize += f.Size
+	if f.Virtual {
+		lm.NumVirtual++
+		lm.VirtualSize += f.Size
+	}
 	return nil
 }
 
 func (lm *LevelMetadata) remove(f *FileMetadata) bool {
 	lm.totalSize -= f.Size
+	if f.Virtual {
+		lm.NumVirtual--
+		lm.VirtualSize -= f.Size
+	}
 	return lm.tree.Delete(f)
 }
 
@@ -303,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
 	return sum
 }
 
+// NumVirtual returns the number of virtual sstables in the level. Its runtime is
+// linear in the length of the slice.
+func (ls *LevelSlice) NumVirtual() uint64 {
+	var n uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			n++
+		}
+	}
+	return n
+}
+
+// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
+// level.
+func (ls *LevelSlice) VirtualSizeSum() uint64 {
+	var sum uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			sum += f.Size
+		}
+	}
+	return sum
+}
+
 // Reslice constructs a new slice backed by the same underlying level, with
 // new start and end positions. Reslice invokes the provided function, passing
 // two LevelIterators: one positioned to i's inclusive start and one

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -49,6 +49,9 @@ type TableStats struct {
 	// The number of point and range deletion entries in the table.
 	NumDeletions uint64
 	// NumRangeKeySets is the total number of range key sets in the table.
+	//
+	// NB: If there's a chance that the sstable contains any range key sets,
+	// then NumRangeKeySets must be > 0.
 	NumRangeKeySets uint64
 	// Estimate of the total disk space that may be dropped by this table's
 	// point deletions by compacting them.
@@ -169,22 +172,13 @@ type FileMetadata struct {
 	// FileNum is the file number.
 	//
 	// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
-	//
-	// TODO(bananabrick): Consider creating separate types for
-	// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
-	// an indentifier for the FileMetadata in Pebble, and also as a handle to
-	// perform reads and writes. We should ensure through types that
-	// FileMetadata.FileNum isn't used to perform reads, and that
-	// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
 	FileNum base.FileNum
 	// Size is the size of the file, in bytes. Size is an approximate value for
 	// virtual sstables.
 	//
-	// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
-	//
-	// TODO(bananabrick): Size is currently used in metrics, and for many key
-	// Pebble level heuristics. Make sure that the heuristics will still work
-	// appropriately with an approximate value of size.
+	// INVARIANTS:
+	// - When !FileMetadata.Virtual, Size == FileBacking.Size.
+	// - Size should be non-zero. Size 0 virtual sstables must not be created.
 	Size uint64
 	// File creation time in seconds since the epoch (1970-01-01 00:00:00
 	// UTC). For ingested sstables, this corresponds to the time the file was
@@ -309,6 +303,7 @@ type PhysicalFileMeta struct {
 //     The underlying file's size is stored in FileBacking.Size, though it could
 //     also be estimated or could correspond to just the referenced portion of
 //     a file (eg. if the file originated on another node).
+//   - Size must be > 0.
 //   - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
 //     This means that all keys in the virtual sstable must have seqnums within
 //     [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
@@ -1209,12 +1204,7 @@ func (v *Version) Unref() {
 		l := v.list
 		l.mu.Lock()
 		l.Remove(v)
-		obsolete := v.unrefFiles()
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
+		v.Deleted(v.unrefFiles())
 		l.mu.Unlock()
 	}
 }
@@ -1226,17 +1216,12 @@ func (v *Version) Unref() {
 func (v *Version) UnrefLocked() {
 	if v.refs.Add(-1) == 0 {
 		v.list.Remove(v)
-		obsolete := v.unrefFiles()
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
+		v.Deleted(v.unrefFiles())
 	}
 }
 
-func (v *Version) unrefFiles() []*FileMetadata {
-	var obsolete []*FileMetadata
+func (v *Version) unrefFiles() []*FileBacking {
+	var obsolete []*FileBacking
 	for _, lm := range v.Levels {
 		obsolete = append(obsolete, lm.release()...)
 	}
@@ -1368,16 +1353,20 @@ func (v *Version) Overlaps(
 // CheckOrdering checks that the files are consistent with respect to
 // increasing file numbers (for level 0 files) and increasing and non-
 // overlapping internal key ranges (for level non-0 files).
-func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
+func (v *Version) CheckOrdering(
+	cmp Compare, format base.FormatKey, order OrderingInvariants,
+) error {
 	for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
 		sublevelIter := v.L0SublevelFiles[sublevel].Iter()
-		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
+		// Sublevels have NEVER allowed split user keys, so we can pass
+		// ProhibitSplitUserKeys.
+		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
 
 	for level, lm := range v.Levels {
-		if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
+		if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
@@ -1446,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
 	v.list = nil // avoid memory leaks
 }
 
+// OrderingInvariants dictates the file ordering invariants active.
+type OrderingInvariants int8
+
+const (
+	// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
+	// contain the same user key.
+	ProhibitSplitUserKeys OrderingInvariants = iota
+	// AllowSplitUserKeys indicates that adjacent files within a level may
+	// contain the same user key. This is only allowed by historical format
+	// major versions.
+	//
+	// TODO(jackson): Remove.
+	AllowSplitUserKeys
+)
+
 // CheckOrdering checks that the files are consistent with respect to
 // seqnums (for level 0 files -- see detailed comment below) and increasing and non-
 // overlapping internal key ranges (for non-level 0 files).
-func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
+//
+// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
+// inclusive of the same user key. Pebble no longer creates version edits
+// installing such files, and Pebble databases with sufficiently high format
+// major version should no longer have any such files within their LSM.
+// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
+// earlier format major versions.
+func CheckOrdering(
+	cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
+) error {
 	// The invariants to check for L0 sublevels are the same as the ones to
 	// check for all other levels. However, if L0 is not organized into
 	// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1527,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
 						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
 						f.Smallest.Pretty(format), f.Largest.Pretty(format))
 				}
-				if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
-					return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
-						errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
-						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
-						f.Smallest.Pretty(format), f.Largest.Pretty(format))
+
+				// What's considered "overlapping" is dependent on the format
+				// major version. If ordering=ProhibitSplitUserKeys, then both
+				// files cannot contain keys with the same user keys. If the
+				// bounds have the same user key, the previous file's boundary
+				// must have a Trailer indicating that it's exclusive.
+				switch ordering {
+				case AllowSplitUserKeys:
+					if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				case ProhibitSplitUserKeys:
+					if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				default:
+					panic("unreachable")
 				}
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -783,6 +783,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
 	}
 	for _, fb := range ve.CreatedBackingTables {
+		if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
+			// There is already a FileBacking associated with fb.DiskFileNum.
+			// This should never happen. There must always be only one FileBacking
+			// associated with a backing sstable.
+			panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
+		}
 		b.AddedFileBacking[fb.DiskFileNum] = fb
 	}
 
@@ -846,6 +852,9 @@ func AccumulateIncompleteAndApplySingleVE(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	backingStateMap map[base.DiskFileNum]*FileBacking,
+	addBackingFunc func(*FileBacking),
+	removeBackingFunc func(base.DiskFileNum),
+	orderingInvariants OrderingInvariants,
 ) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
 	if len(ve.RemovedBackingTables) != 0 {
 		panic("pebble: invalid incomplete version edit")
@@ -857,14 +866,14 @@ func AccumulateIncompleteAndApplySingleVE(
 	}
 	zombies = make(map[base.DiskFileNum]uint64)
 	v, err := b.Apply(
-		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
+		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
 	)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	for _, s := range b.AddedFileBacking {
-		backingStateMap[s.DiskFileNum] = s
+		addBackingFunc(s)
 	}
 
 	for fileNum := range zombies {
@@ -875,10 +884,9 @@ func AccumulateIncompleteAndApplySingleVE(
 			ve.RemovedBackingTables = append(
 				ve.RemovedBackingTables, fileNum,
 			)
-			delete(backingStateMap, fileNum)
+			removeBackingFunc(fileNum)
 		}
 	}
-
 	return v, zombies, nil
 }
 
@@ -899,6 +907,7 @@ func (b *BulkVersionEdit) Apply(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	zombies map[base.DiskFileNum]uint64,
+	orderingInvariants OrderingInvariants,
 ) (*Version, error) {
 	addZombie := func(state *FileBacking) {
 		if zombies != nil {
@@ -1082,7 +1091,7 @@ func (b *BulkVersionEdit) Apply(
 			} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
-			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			continue
@@ -1103,7 +1112,7 @@ func (b *BulkVersionEdit) Apply(
 					end.Prev()
 				}
 			})
-			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -52,15 +52,15 @@ import (
 //	                        │
 //	                        ╰── <?>.Next
 type UserIteratorConfig struct {
-	snapshot   uint64
-	comparer   *base.Comparer
-	miter      keyspan.MergingIter
-	biter      keyspan.BoundedIter
-	diter      keyspan.DefragmentingIter
-	liters     [manifest.NumLevels]keyspan.LevelIter
-	litersUsed int
-	onlySets   bool
-	bufs       *Buffers
+	snapshot     uint64
+	comparer     *base.Comparer
+	miter        keyspan.MergingIter
+	biter        keyspan.BoundedIter
+	diter        keyspan.DefragmentingIter
+	liters       [manifest.NumLevels]keyspan.LevelIter
+	litersUsed   int
+	internalKeys bool
+	bufs         *Buffers
 }
 
 // Buffers holds various buffers used for range key iteration. They're exposed
@@ -79,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
 
 // Init initializes the range key iterator stack for user iteration. The
 // resulting fragment iterator applies range key semantics, defragments spans
-// according to their user-observable state and, if onlySets = true, removes all
+// according to their user-observable state and, if !internalKeys, removes all
 // Keys other than RangeKeySets describing the current state of range keys. The
-// resulting spans contain Keys sorted by Suffix.
+// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
+// in which case they remain sorted by trailer descending).
 //
 // The snapshot sequence number parameter determines which keys are visible. Any
 // keys not visible at the provided snapshot are ignored.
@@ -91,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
 	lower, upper []byte,
 	hasPrefix *bool,
 	prefix *[]byte,
-	onlySets bool,
+	internalKeys bool,
 	bufs *Buffers,
 	iters ...keyspan.FragmentIterator,
 ) keyspan.FragmentIterator {
 	ui.snapshot = snapshot
 	ui.comparer = comparer
-	ui.onlySets = onlySets
+	ui.internalKeys = internalKeys
 	ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
 	ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
-	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	if internalKeys {
+		ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	} else {
+		ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	}
 	ui.litersUsed = 0
 	ui.bufs = bufs
 	return &ui.diter
@@ -136,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
 // of unset keys, removal of keys overwritten by a set at the same suffix, etc)
 // and then non-RangeKeySet keys are removed. The resulting transformed spans
 // only contain RangeKeySets describing the state visible at the provided
-// sequence number, and hold their Keys sorted by Suffix.
+// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
+// is true, then keys remain sorted by trailer.
 func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
 	// Apply shadowing of keys.
 	dst.Start = s.Start
@@ -148,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 	if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
 		return err
 	}
-	// During user iteration over range keys, unsets and deletes don't matter.
-	// Remove them if onlySets = true. This step helps logical defragmentation
-	// during iteration.
+	if ui.internalKeys {
+		if s.KeysOrder != keyspan.ByTrailerDesc {
+			panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
+		}
+		dst.Keys = ui.bufs.sortBuf.Keys
+		keyspan.SortKeysByTrailer(&dst.Keys)
+		return nil
+	}
+	// During user iteration over range keys, unsets and deletes don't matter. This
+	// step helps logical defragmentation during iteration.
 	keys := ui.bufs.sortBuf.Keys
 	dst.Keys = dst.Keys[:0]
 	for i := range keys {
@@ -164,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
 				panic("pebble: keys unexpectedly not in ascending suffix order")
 			}
-			if ui.onlySets {
-				// Skip.
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
+			// Skip.
+			continue
 		case base.InternalKeyKindRangeKeyDelete:
-			if ui.onlySets {
-				// Skip.
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
+			// Skip.
+			continue
 		default:
 			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
 		}
@@ -193,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 // sequence numbers). It's intended for use during user iteration, when the
 // wrapped keyspan iterator is merging spans across all levels of the LSM.
 func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
+	// This method is not called with internalKeys = true.
+	if ui.internalKeys {
+		panic("unexpected call to ShouldDefragment with internalKeys = true")
+	}
 	// This implementation must only be used on spans that have transformed by
 	// ui.Transform. The transform applies shadowing, removes all keys besides
 	// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -208,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 	ret := true
 	for i := range a.Keys {
 		if invariants.Enabled {
-			if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
-				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
+			if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
+				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
 				panic("pebble: unexpected non-RangeKeySet during defragmentation")
 			}
 			if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||

vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go

@@ -384,14 +384,18 @@ func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
 	return ret
 }
 
+func abs(a int64) int64 {
+	if a < 0 {
+		return -a
+	}
+	return a
+}
+
 // RandomSeparator returns a random alphabetic key k such that a < k < b,
 // pulling randomness from the provided random number generator. If dst is
 // provided and the generated key fits within dst's capacity, the returned slice
 // will use dst's memory.
 //
-// The keys a and b must be purely alphabetic, consisting only of characters
-// a-z.
-//
 // If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
 // will consist of the prefix P appended with the provided suffix. A zero suffix
 // generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
@@ -405,6 +409,10 @@ func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
 //
 // RandomSeparator panics if a or b fails to decode.
 func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
+	if Comparer.Compare(a, b) >= 0 {
+		return nil
+	}
+
 	// Determine both keys' logical prefixes and suffixes.
 	ai := Comparer.Split(a)
 	bi := Comparer.Split(b)
@@ -431,31 +439,56 @@ func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Ra
 	diff := bpIdx - apIdx
 	generatedIdx := bpIdx
 	if diff > 0 {
+		var add int64 = diff + 1
+		var start int64 = apIdx
 		if as == 1 {
-			// There's no expressible key with suffix `a` greater than a@1, so
+			// There's no expressible key with prefix a greater than a@1. So,
 			// exclude ap.
-			generatedIdx = rng.Int63n(diff) + apIdx + 1
-		} else {
-			generatedIdx = rng.Int63n(diff+1) + apIdx
+			start = apIdx + 1
+			add = diff
+		}
+		if bs == 0 {
+			// No key with prefix b can sort before b@0. We don't want to pick b.
+			add--
+		}
+		// We're allowing generated id to be in the range [start, start + add - 1].
+		if start > start+add-1 {
+			return nil
+		}
+		// If we can generate a key which is actually in the middle of apIdx
+		// and bpIdx use it so that we don't have to bother about timestamps.
+		generatedIdx = rng.Int63n(add) + start
+		for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
+			generatedIdx = rng.Int63n(add) + start
 		}
 	}
 
 	switch {
 	case generatedIdx == apIdx && generatedIdx == bpIdx:
-		if bs-as <= 1 {
+		if abs(bs-as) <= 1 {
 			// There's no expressible suffix between the two, and there's no
 			// possible separator key.
 			return nil
 		}
+		// The key b is >= key a, but has the same prefix, so b must have the
+		// smaller timestamp, unless a has timestamp of 0.
+		//
 		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
 		// any suffix we generate will be greater than it.
-		if (as != 0 && suffix >= as) || suffix <= bs {
-			suffix = rng.Int63n(bs-as-1) + as + 1
+		if as == 0 {
+			// bs > as
+			suffix = bs + rng.Int63n(10) + 1
+		} else {
+			// bs < as.
+			// Generate suffix in range [bs + 1, as - 1]
+			suffix = bs + 1 + rng.Int63n(as-bs-1)
 		}
 	case generatedIdx == apIdx:
 		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
 		// any suffix we generate will be greater than it.
-		if as != 0 && suffix >= as {
+		if as == 0 && suffix == 0 {
+			suffix++
+		} else if as != 0 && suffix >= as {
 			suffix = rng.Int63n(as)
 		}
 	case generatedIdx == bpIdx:
@@ -468,7 +501,12 @@ func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Ra
 	} else {
 		dst = dst[:cap(dst)]
 	}
-	w := WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
+	var w int
+	if suffix == 0 {
+		w = WriteKey(dst, Alpha(maxLength), generatedIdx)
+	} else {
+		w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
+	}
 	return dst[:w]
 }