Update dependencies

vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go

@@ -18,22 +18,21 @@
 package arenaskl
 
 import (
-	"math"
 	"sync/atomic"
 	"unsafe"
 
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/constants"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // Arena is lock-free.
 type Arena struct {
-	n   uint64
+	n   atomic.Uint64
 	buf []byte
 }
 
-const (
-	align4 = 3
-)
+const nodeAlignment = 4
 
 var (
 	// ErrArenaFull indicates that the arena is full and cannot perform any more
@@ -44,20 +43,28 @@ var (
 // NewArena allocates a new arena using the specified buffer as the backing
 // store.
 func NewArena(buf []byte) *Arena {
-	// Don't store data at position 0 in order to reserve offset=0 as a kind
-	// of nil pointer.
-	return &Arena{
-		n:   1,
+	if len(buf) > constants.MaxUint32OrInt {
+		if invariants.Enabled {
+			panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
+		}
+		buf = buf[:constants.MaxUint32OrInt]
+	}
+	a := &Arena{
 		buf: buf,
 	}
+	// We don't store data at position 0 in order to reserve offset=0 as a kind of
+	// nil pointer.
+	a.n.Store(1)
+	return a
 }
 
 // Size returns the number of bytes allocated by the arena.
 func (a *Arena) Size() uint32 {
-	s := atomic.LoadUint64(&a.n)
-	if s > math.MaxUint32 {
-		// Saturate at MaxUint32.
-		return math.MaxUint32
+	s := a.n.Load()
+	if s > constants.MaxUint32OrInt {
+		// The last failed allocation can push the size higher than len(a.buf).
+		// Saturate at the maximum representable offset.
+		return constants.MaxUint32OrInt
 	}
 	return uint32(s)
 }
@@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 {
 	return uint32(len(a.buf))
 }
 
-func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
+// alloc allocates a buffer of the given size and with the given alignment
+// (which must be a power of 2).
+//
+// If overflow is not 0, it also ensures that many bytes after the buffer are
+// inside the arena (this is used for structures that are larger than the
+// requested size but don't use those extra bytes).
+func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
+	if invariants.Enabled && (alignment&(alignment-1)) != 0 {
+		panic(errors.AssertionFailedf("invalid alignment %d", alignment))
+	}
 	// Verify that the arena isn't already full.
-	origSize := atomic.LoadUint64(&a.n)
+	origSize := a.n.Load()
 	if int(origSize) > len(a.buf) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Pad the allocation with enough bytes to ensure the requested alignment.
-	padded := uint32(size) + align
+	padded := uint64(size) + uint64(alignment) - 1
 
-	newSize := atomic.AddUint64(&a.n, uint64(padded))
-	if int(newSize)+int(overflow) > len(a.buf) {
+	newSize := a.n.Add(padded)
+	if newSize+uint64(overflow) > uint64(len(a.buf)) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Return the aligned offset.
-	offset := (uint32(newSize) - padded + align) & ^align
-	return offset, padded, nil
+	offset := (uint32(newSize) - size) & ^(alignment - 1)
+	return offset, uint32(padded), nil
 }
 
 func (a *Arena) getBytes(offset uint32, size uint32) []byte {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go

@@ -18,7 +18,6 @@
 package arenaskl
 
 import (
-	"encoding/binary"
 	"sync"
 
 	"github.com/cockroachdb/pebble/internal/base"
@@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
 }
 
 func (it *Iterator) decodeKey() {
-	b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
-	// This is a manual inline of base.DecodeInternalKey, because the Go compiler
-	// seems to refuse to automatically inline it currently.
-	l := len(b) - 8
-	if l >= 0 {
-		it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
-		it.key.UserKey = b[:l:l]
-	} else {
-		it.key.Trailer = uint64(base.InternalKeyKindInvalid)
-		it.key.UserKey = nil
-	}
+	it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
+	it.key.Trailer = it.nd.keyTrailer
 }
 
 func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go

@@ -29,25 +29,27 @@ import (
 // is used here. If a key/value overflows a uint32, it should not be added to
 // the skiplist.
 func MaxNodeSize(keySize, valueSize uint32) uint64 {
-	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
+	const maxPadding = nodeAlignment - 1
+	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
 }
 
 type links struct {
-	nextOffset uint32
-	prevOffset uint32
+	nextOffset atomic.Uint32
+	prevOffset atomic.Uint32
 }
 
 func (l *links) init(prevOffset, nextOffset uint32) {
-	l.nextOffset = nextOffset
-	l.prevOffset = prevOffset
+	l.nextOffset.Store(nextOffset)
+	l.prevOffset.Store(prevOffset)
 }
 
 type node struct {
 	// Immutable fields, so no need to lock to access key.
-	keyOffset uint32
-	keySize   uint32
-	valueSize uint32
-	allocSize uint32
+	keyOffset  uint32
+	keySize    uint32
+	keyTrailer uint64
+	valueSize  uint32
+	allocSize  uint32
 
 	// Most nodes do not need to use the full height of the tower, since the
 	// probability of each successive level decreases exponentially. Because
@@ -65,7 +67,7 @@ func newNode(
 	if height < 1 || height > maxHeight {
 		panic("height cannot be less than one or greater than the max height")
 	}
-	keySize := key.Size()
+	keySize := len(key.UserKey)
 	if int64(keySize) > math.MaxUint32 {
 		panic("key is too large")
 	}
@@ -81,8 +83,8 @@ func newNode(
 	if err != nil {
 		return
 	}
-
-	key.Encode(nd.getKeyBytes(arena))
+	nd.keyTrailer = key.Trailer
+	copy(nd.getKeyBytes(arena), key.UserKey)
 	copy(nd.getValue(arena), value)
 	return
 }
@@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod
 	unusedSize := uint32((maxHeight - int(height)) * linksSize)
 	nodeSize := uint32(maxNodeSize) - unusedSize
 
-	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
+	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
 	if err != nil {
 		return
 	}
@@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte {
 }
 
 func (n *node) nextOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].nextOffset)
+	return n.tower[h].nextOffset.Load()
 }
 
 func (n *node) prevOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].prevOffset)
+	return n.tower[h].prevOffset.Load()
 }
 
 func (n *node) casNextOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
+	return n.tower[h].nextOffset.CompareAndSwap(old, val)
 }
 
 func (n *node) casPrevOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
+	return n.tower[h].prevOffset.CompareAndSwap(old, val)
 }

vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go

@@ -44,7 +44,6 @@ Key differences:
 package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
 
 import (
-	"encoding/binary"
 	"math"
 	"runtime"
 	"sync/atomic"
@@ -80,7 +79,7 @@ type Skiplist struct {
 	cmp    base.Compare
 	head   *node
 	tail   *node
-	height uint32 // Current height. 1 <= height <= maxHeight. CAS.
+	height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
 
 	// If set to true by tests, then extra delays are added to make it easier to
 	// detect unusual race conditions.
@@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
 	headOffset := arena.getPointerOffset(unsafe.Pointer(head))
 	tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
 	for i := 0; i < maxHeight; i++ {
-		head.tower[i].nextOffset = tailOffset
-		tail.tower[i].prevOffset = headOffset
+		head.tower[i].nextOffset.Store(tailOffset)
+		tail.tower[i].prevOffset.Store(headOffset)
 	}
 
 	*s = Skiplist{
-		arena:  arena,
-		cmp:    cmp,
-		head:   head,
-		tail:   tail,
-		height: 1,
+		arena: arena,
+		cmp:   cmp,
+		head:  head,
+		tail:  tail,
 	}
+	s.height.Store(1)
 }
 
 // Height returns the height of the highest tower within any of the nodes that
 // have ever been allocated as part of this skiplist.
-func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
+func (s *Skiplist) Height() uint32 { return s.height.Load() }
 
 // Arena returns the arena backing this skiplist.
 func (s *Skiplist) Arena() *Arena { return s.arena }
@@ -327,7 +326,7 @@ func (s *Skiplist) newNode(
 	// Try to increase s.height via CAS.
 	listHeight := s.Height()
 	for height > listHeight {
-		if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
+		if s.height.CompareAndSwap(listHeight, height) {
 			// Successfully increased skiplist.height.
 			break
 		}
@@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel(
 
 		offset, size := next.keyOffset, next.keySize
 		nextKey := s.arena.buf[offset : offset+size]
-		n := int32(size) - 8
-		cmp := s.cmp(key.UserKey, nextKey[:n])
+		cmp := s.cmp(key.UserKey, nextKey)
 		if cmp < 0 {
 			// We are done for this level, since prev.key < key < next.key.
 			break
 		}
 		if cmp == 0 {
 			// User-key equality.
-			var nextTrailer uint64
-			if n >= 0 {
-				nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
-			} else {
-				nextTrailer = uint64(base.InternalKeyKindInvalid)
-			}
-			if key.Trailer == nextTrailer {
+			if key.Trailer == next.keyTrailer {
 				// Internal key equality.
 				found = true
 				break
 			}
-			if key.Trailer > nextTrailer {
+			if key.Trailer > next.keyTrailer {
 				// We are done for this level, since prev.key < key < next.key.
 				break
 			}
@@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel(
 
 func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 	ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
-	n := int32(nd.keySize) - 8
-	cmp := s.cmp(ndKey[:n], key.UserKey)
+	cmp := s.cmp(ndKey, key.UserKey)
 	if cmp < 0 {
 		return true
 	}
@@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 		return false
 	}
 	// User-key equality.
-	var ndTrailer uint64
-	if n >= 0 {
-		ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
-	} else {
-		ndTrailer = uint64(base.InternalKeyKindInvalid)
-	}
-	if key.Trailer == ndTrailer {
+	if key.Trailer == nd.keyTrailer {
 		// Internal key equality.
 		return false
 	}
-	return key.Trailer < ndTrailer
+	return key.Trailer < nd.keyTrailer
 }
 
 func (s *Skiplist) getNext(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
+	offset := nd.tower[h].nextOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }
 
 func (s *Skiplist) getPrev(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
+	offset := nd.tower[h].prevOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -20,6 +20,11 @@ type FileNum uint64
 // String returns a string representation of the file number.
 func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
 
+// SafeFormat implements redact.SafeFormatter.
+func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%06d", redact.SafeUint(fn))
+}
+
 // DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
 // called if the caller can ensure that the FileNum belongs to a physical file
 // on disk. These could be manifests, log files, physical sstables on disk, the
@@ -40,6 +45,11 @@ type DiskFileNum struct {
 
 func (dfn DiskFileNum) String() string { return dfn.fn.String() }
 
+// SafeFormat implements redact.SafeFormatter.
+func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
+	dfn.fn.SafeFormat(w, verb)
+}
+
 // FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
 func (dfn DiskFileNum) FileNum() FileNum {
 	return dfn.fn

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -9,6 +9,17 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/cockroachdb/redact"
+)
+
+const (
+	// SeqNumZero is the zero sequence number, set by compactions if they can
+	// guarantee there are no keys underneath an internal key.
+	SeqNumZero = uint64(0)
+	// SeqNumStart is the first sequence number assigned to a key. Sequence
+	// numbers 1-9 are reserved for potential future use.
+	SeqNumStart = uint64(10)
 )
 
 // InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
@@ -152,6 +163,11 @@ func (k InternalKeyKind) String() string {
 	return fmt.Sprintf("UNKNOWN:%d", k)
 }
 
+// SafeFormat implements redact.SafeFormatter.
+func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Print(redact.SafeString(k.String()))
+}
+
 // InternalKey is a key used for the in-memory and on-disk partial DBs that
 // make up a pebble DB.
 //
@@ -365,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
 	return k.Trailer >> 8
 }
 
+// SeqNumFromTrailer returns the sequence number component of a trailer.
+func SeqNumFromTrailer(t uint64) uint64 {
+	return t >> 8
+}
+
 // Visible returns true if the key is visible at the specified snapshot
 // sequence number.
 func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {

vendor/github.com/cockroachdb/pebble/internal/base/iterator.go

@@ -7,6 +7,9 @@ package base
 import (
 	"fmt"
 	"time"
+
+	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/redact"
 )
 
 // InternalIterator iterates over a DB's key/value pairs in key order. Unlike
@@ -407,3 +410,35 @@ func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
 	s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
 	s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
 }
+
+func (s *InternalIteratorStats) String() string {
+	return redact.StringWithoutMarkers(s)
+}
+
+// SafeFormat implements the redact.SafeFormatter interface.
+func (s *InternalIteratorStats) SafeFormat(p redact.SafePrinter, verb rune) {
+	p.Printf("blocks: %s cached",
+		humanize.Bytes.Uint64(s.BlockBytesInCache),
+	)
+	if s.BlockBytes != s.BlockBytesInCache || s.BlockReadDuration != 0 {
+		p.Printf(", %s not cached (read time: %s)",
+			humanize.Bytes.Uint64(s.BlockBytes-s.BlockBytesInCache),
+			humanize.FormattedString(s.BlockReadDuration.String()),
+		)
+	}
+	p.Printf("; points: %s", humanize.Count.Uint64(s.PointCount))
+
+	if s.PointsCoveredByRangeTombstones != 0 {
+		p.Printf("(%s tombstoned)", humanize.Count.Uint64(s.PointsCoveredByRangeTombstones))
+	}
+	p.Printf(" (%s keys, %s values)",
+		humanize.Bytes.Uint64(s.KeyBytes),
+		humanize.Bytes.Uint64(s.ValueBytes),
+	)
+	if s.SeparatedPointValue.Count != 0 {
+		p.Printf("; separated: %s (%s, %s fetched)",
+			humanize.Count.Uint64(s.SeparatedPointValue.Count),
+			humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytes),
+			humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytesFetched))
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go

@@ -1,56 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package base
-
-import "fmt"
-
-// This file defines sequence numbers that are reserved for foreign keys i.e.
-// internal keys coming from other Pebble instances and existing in shared
-// storage, as those will "slot below" any internal keys added by our own Pebble
-// instance. Any keys created by this Pebble instance need to be greater than
-// all reserved sequence numbers (i.e. >= SeqNumStart).
-const (
-	// SeqNumZero is the zero sequence number, set by compactions if they can
-	// guarantee there are no keys underneath an internal key.
-	SeqNumZero = uint64(0)
-
-	// SeqNumL6 is the sequence number reserved for foreign keys in L6. This seqnum
-	// will be used to expose any range key sets as well as point keys in L6. Range
-	// deletes do not need to be exposed in L6.
-	SeqNumL6 = uint64(1)
-
-	// SeqNumL5 is the sequence number reserved for foreign keys in L5. This seqnum
-	// needs to be greater than SeqNumL6, so that range deletes in L5 can be
-	// exposed at this level and will correctly delete covering points in L6. Also
-	// range key unsets/dels will be exposed at this seqnum and will need to shadow
-	// overlapping range keys in L6.
-	//
-	// Note that we can use the same sequence number for all exposed keys in L5 as
-	// range dels do not delete points at the same seqnum, and range key
-	// unsets/deletes do not coalesce with range key sets at the same seqnum. Range
-	// key masking does not care about the sequence number of overlapping points
-	// (rather, it applies to points based on suffixes), so we can use this seqnum
-	// for all L5 keys exposed.
-	SeqNumL5 = uint64(2)
-
-	// Sequence numbers 3-9 are reserved for future use.
-
-	// SeqNumStart is the first sequence number assigned to a key written by
-	// ourselves.
-	SeqNumStart = uint64(10)
-)
-
-// SeqNumForLevel returns the appropriate reserved sequence number for keys in
-// foreign sstables at the specified level.
-func SeqNumForLevel(level int) uint64 {
-	switch level {
-	case 5:
-		return SeqNumL5
-	case 6:
-		return SeqNumL6
-	default:
-		panic(fmt.Sprintf("unexpected foreign sstable at level %d", level))
-	}
-}

vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go

@@ -63,14 +63,15 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/constants"
 	"golang.org/x/exp/rand"
 )
 
 const (
 	maxHeight    = 20
-	maxNodeSize  = int(unsafe.Sizeof(node{}))
-	linksSize    = int(unsafe.Sizeof(links{}))
-	maxNodesSize = math.MaxUint32
+	maxNodeSize  = uint64(unsafe.Sizeof(node{}))
+	linksSize    = uint64(unsafe.Sizeof(links{}))
+	maxNodesSize = constants.MaxUint32OrInt
 )
 
 var (
@@ -281,7 +282,7 @@ func (s *Skiplist) newNode(
 		panic("height cannot be less than one or greater than the max height")
 	}
 
-	unusedSize := (maxHeight - int(height)) * linksSize
+	unusedSize := uint64(maxHeight-int(height)) * linksSize
 	nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
 	if err != nil {
 		return 0, err
@@ -296,13 +297,13 @@ func (s *Skiplist) newNode(
 }
 
 func (s *Skiplist) alloc(size uint32) (uint32, error) {
-	offset := len(s.nodes)
+	offset := uint64(len(s.nodes))
 
 	// We only have a need for memory up to offset + size, but we never want
 	// to allocate a node whose tail points into unallocated memory.
 	minAllocSize := offset + maxNodeSize
-	if cap(s.nodes) < minAllocSize {
-		allocSize := cap(s.nodes) * 2
+	if uint64(cap(s.nodes)) < minAllocSize {
+		allocSize := uint64(cap(s.nodes)) * 2
 		if allocSize < minAllocSize {
 			allocSize = minAllocSize
 		}

vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go

@@ -71,14 +71,12 @@ func (h Handle) Get() []byte {
 
 // Release releases the reference to the cache entry.
 func (h Handle) Release() {
-	if h.value != nil {
-		h.value.release()
-	}
+	h.value.release()
 }
 
 type shard struct {
-	hits   int64
-	misses int64
+	hits   atomic.Int64
+	misses atomic.Int64
 
 	mu sync.RWMutex
 
@@ -119,15 +117,15 @@ func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
 	if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
 		value = e.acquireValue()
 		if value != nil {
-			atomic.StoreInt32(&e.referenced, 1)
+			e.referenced.Store(true)
 		}
 	}
 	c.mu.RUnlock()
 	if value == nil {
-		atomic.AddInt64(&c.misses, 1)
+		c.misses.Add(1)
 		return Handle{}
 	}
-	atomic.AddInt64(&c.hits, 1)
+	c.hits.Add(1)
 	return Handle{value: value}
 }
 
@@ -160,7 +158,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
 	case e.peekValue() != nil:
 		// cache entry was a hot or cold page
 		e.setValue(value)
-		atomic.StoreInt32(&e.referenced, 1)
+		e.referenced.Store(true)
 		delta := int64(len(value.buf)) - e.size
 		e.size = int64(len(value.buf))
 		if e.ptype == etHot {
@@ -176,7 +174,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
 		// cache entry was a test page
 		c.sizeTest -= e.size
 		c.countTest--
-		c.metaDel(e)
+		c.metaDel(e).release()
 		c.metaCheck(e)
 
 		e.size = int64(len(value.buf))
@@ -185,7 +183,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
 			c.coldTarget = c.targetSize()
 		}
 
-		atomic.StoreInt32(&e.referenced, 0)
+		e.referenced.Store(false)
 		e.setValue(value)
 		e.ptype = etHot
 		if c.metaAdd(k, e) {
@@ -233,37 +231,75 @@ func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
 		return
 	}
 
-	c.mu.Lock()
-	defer c.mu.Unlock()
+	var deletedValue *Value
+	func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
 
-	e := c.blocks.Get(k)
-	if e == nil {
-		return
-	}
-	c.metaEvict(e)
-
-	c.checkConsistency()
+		e := c.blocks.Get(k)
+		if e == nil {
+			return
+		}
+		deletedValue = c.metaEvict(e)
+		c.checkConsistency()
+	}()
+	// Now that the mutex has been dropped, release the reference which will
+	// potentially free the memory associated with the previous cached value.
+	deletedValue.release()
 }
 
 // EvictFile evicts all of the cache values for the specified file.
 func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
 	fkey := key{fileKey{id, fileNum}, 0}
+	for c.evictFileRun(fkey) {
+		// Sched switch to give another goroutine an opportunity to acquire the
+		// shard mutex.
+		runtime.Gosched()
+	}
+}
+
+func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
+	// If most of the file's blocks are held in the block cache, evicting all
+	// the blocks may take a while. We don't want to block the entire cache
+	// shard, forcing concurrent readers to wait until we're finished. We drop
+	// the mutex every [blocksPerMutexAcquisition] blocks to give other
+	// goroutines an opportunity to make progress.
+	const blocksPerMutexAcquisition = 5
+	c.mu.Lock()
+
+	// Releasing a value may result in free-ing it back to the memory allocator.
+	// This can have a nontrivial cost that we'd prefer to not pay while holding
+	// the shard mutex, so we collect the evicted values in a local slice and
+	// only release them in a defer after dropping the cache mutex.
+	var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
+	obsoleteValues := obsoleteValuesAlloc[:0]
+	defer func() {
+		c.mu.Unlock()
+		for _, v := range obsoleteValues {
+			v.release()
+		}
+	}()
+
 	blocks := c.files.Get(fkey)
 	if blocks == nil {
-		return
-	}
-	for b, n := blocks, (*entry)(nil); ; b = n {
-		n = b.fileLink.next
-		c.metaEvict(b)
-		if b == n {
-			break
-		}
+		// No blocks for this file.
+		return false
 	}
 
-	c.checkConsistency()
+	// b is the current head of the doubly linked list, and n is the entry after b.
+	for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
+		n = b.fileLink.next
+		obsoleteValues = append(obsoleteValues, c.metaEvict(b))
+		if b == n {
+			// b == n represents the case where b was the last entry remaining
+			// in the doubly linked list, which is why it pointed at itself. So
+			// no more entries left.
+			c.checkConsistency()
+			return false
+		}
+	}
+	// Exhausted blocksPerMutexAcquisition.
+	return true
 }
 
 func (c *shard) Free() {
@@ -274,7 +310,7 @@ func (c *shard) Free() {
 	// metaCheck call when the "invariants" build tag is specified.
 	for c.handHot != nil {
 		e := c.handHot
-		c.metaDel(c.handHot)
+		c.metaDel(c.handHot).release()
 		e.free()
 	}
 
@@ -359,12 +395,14 @@ func (c *shard) metaAdd(key key, e *entry) bool {
 
 // Remove the entry from the cache. This removes the entry from the blocks map,
 // the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
-// entry.
-func (c *shard) metaDel(e *entry) {
+// entry. Returns the deleted value that must be released, if any.
+func (c *shard) metaDel(e *entry) (deletedValue *Value) {
 	if value := e.peekValue(); value != nil {
 		value.ref.trace("metaDel")
 	}
-	e.setValue(nil)
+	// Remove the pointer to the value.
+	deletedValue = e.val
+	e.val = nil
 
 	c.blocks.Delete(e.key)
 	if entriesGoAllocated {
@@ -396,6 +434,7 @@ func (c *shard) metaDel(e *entry) {
 	} else {
 		c.files.Put(fkey, next)
 	}
+	return deletedValue
 }
 
 // Check that the specified entry is not referenced by the cache.
@@ -455,7 +494,7 @@ func (c *shard) metaCheck(e *entry) {
 	}
 }
 
-func (c *shard) metaEvict(e *entry) {
+func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
 	switch e.ptype {
 	case etHot:
 		c.sizeHot -= e.size
@@ -467,9 +506,10 @@ func (c *shard) metaEvict(e *entry) {
 		c.sizeTest -= e.size
 		c.countTest--
 	}
-	c.metaDel(e)
+	evictedValue = c.metaDel(e)
 	c.metaCheck(e)
 	e.free()
+	return evictedValue
 }
 
 func (c *shard) evict() {
@@ -491,8 +531,8 @@ func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
 
 	e := c.handCold
 	if e.ptype == etCold {
-		if atomic.LoadInt32(&e.referenced) == 1 {
-			atomic.StoreInt32(&e.referenced, 0)
+		if e.referenced.Load() {
+			e.referenced.Store(false)
 			e.ptype = etHot
 			c.sizeCold -= e.size
 			c.countCold--
@@ -528,8 +568,8 @@ func (c *shard) runHandHot() {
 
 	e := c.handHot
 	if e.ptype == etHot {
-		if atomic.LoadInt32(&e.referenced) == 1 {
-			atomic.StoreInt32(&e.referenced, 0)
+		if e.referenced.Load() {
+			e.referenced.Store(false)
 		} else {
 			e.ptype = etCold
 			c.sizeHot -= e.size
@@ -564,7 +604,7 @@ func (c *shard) runHandTest() {
 		if c.coldTarget < 0 {
 			c.coldTarget = 0
 		}
-		c.metaDel(e)
+		c.metaDel(e).release()
 		c.metaCheck(e)
 		e.free()
 	}
@@ -587,7 +627,7 @@ type Metrics struct {
 // Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
 // used for page replacement
 // (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
-// order to provide better concurrency, 2 x NumCPUs shards are created, with
+// order to provide better concurrency, 4 x NumCPUs shards are created, with
 // each shard being given 1/n of the target cache size. The Clock-PRO algorithm
 // is run independently on each shard.
 //
@@ -621,9 +661,9 @@ type Metrics struct {
 // used in combination by specifying `-tags invariants,tracing`. Note that
 // "tracing" produces a significant slowdown, while "invariants" does not.
 type Cache struct {
-	refs    int64
+	refs    atomic.Int64
 	maxSize int64
-	idAlloc uint64
+	idAlloc atomic.Uint64
 	shards  []shard
 
 	// Traces recorded by Cache.trace. Used for debugging.
@@ -643,17 +683,44 @@ type Cache struct {
 //	defer c.Unref()
 //	d, err := pebble.Open(pebble.Options{Cache: c})
 func New(size int64) *Cache {
-	return newShards(size, 2*runtime.GOMAXPROCS(0))
+	// How many cache shards should we create?
+	//
+	// Note that the probability two processors will try to access the same
+	// shard at the same time increases superlinearly with the number of
+	// processors (Eg, consider the brithday problem where each CPU is a person,
+	// and each shard is a possible birthday).
+	//
+	// We could consider growing the number of shards superlinearly, but
+	// increasing the shard count may reduce the effectiveness of the caching
+	// algorithm if frequently-accessed blocks are insufficiently distributed
+	// across shards. If a shard's size is smaller than a single frequently
+	// scanned sstable, then the shard will be unable to hold the entire
+	// frequently-scanned table in memory despite other shards still holding
+	// infrequently accessed blocks.
+	//
+	// Experimentally, we've observed contention contributing to tail latencies
+	// at 2 shards per processor. For now we use 4 shards per processor,
+	// recognizing this may not be final word.
+	m := 4 * runtime.GOMAXPROCS(0)
+
+	// In tests we can use large CPU machines with small cache sizes and have
+	// many caches in existence at a time. If sharding into m shards would
+	// produce too small shards, constrain the number of shards to 4.
+	const minimumShardSize = 4 << 20 // 4 MiB
+	if m > 4 && int(size)/m < minimumShardSize {
+		m = 4
+	}
+	return newShards(size, m)
 }
 
 func newShards(size int64, shards int) *Cache {
 	c := &Cache{
-		refs:    1,
 		maxSize: size,
-		idAlloc: 1,
 		shards:  make([]shard, shards),
 	}
-	c.trace("alloc", c.refs)
+	c.refs.Store(1)
+	c.idAlloc.Store(1)
+	c.trace("alloc", c.refs.Load())
 	for i := range c.shards {
 		c.shards[i] = shard{
 			maxSize:    size / int64(len(c.shards)),
@@ -669,7 +736,7 @@ func newShards(size int64, shards int) *Cache {
 	// Note: this is a no-op if invariants are disabled or race is enabled.
 	invariants.SetFinalizer(c, func(obj interface{}) {
 		c := obj.(*Cache)
-		if v := atomic.LoadInt64(&c.refs); v != 0 {
+		if v := c.refs.Load(); v != 0 {
 			c.tr.Lock()
 			fmt.Fprintf(os.Stderr,
 				"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
@@ -716,7 +783,7 @@ func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *sh
 // Ref adds a reference to the cache. The cache only remains valid as long a
 // reference is maintained to it.
 func (c *Cache) Ref() {
-	v := atomic.AddInt64(&c.refs, 1)
+	v := c.refs.Add(1)
 	if v <= 1 {
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	}
@@ -725,7 +792,7 @@ func (c *Cache) Ref() {
 
 // Unref releases a reference on the cache.
 func (c *Cache) Unref() {
-	v := atomic.AddInt64(&c.refs, -1)
+	v := c.refs.Add(-1)
 	c.trace("unref", v)
 	switch {
 	case v < 0:
@@ -785,14 +852,14 @@ func (c *Cache) Size() int64 {
 // manually managed. The caller MUST either add the value to the cache (via
 // Cache.Set), or release the value (via Cache.Free). Failure to do so will
 // result in a memory leak.
-func (c *Cache) Alloc(n int) *Value {
+func Alloc(n int) *Value {
 	return newValue(n)
 }
 
 // Free frees the specified value. The buffer associated with the value will
 // possibly be reused, making it invalid to use the buffer after calling
 // Free. Do not call Free on a value that has been added to the cache.
-func (c *Cache) Free(v *Value) {
+func Free(v *Value) {
 	if n := v.refs(); n > 1 {
 		panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
 	}
@@ -829,8 +896,8 @@ func (c *Cache) Metrics() Metrics {
 		m.Count += int64(s.blocks.Count())
 		m.Size += s.sizeHot + s.sizeCold
 		s.mu.RUnlock()
-		m.Hits += atomic.LoadInt64(&s.hits)
-		m.Misses += atomic.LoadInt64(&s.misses)
+		m.Hits += s.hits.Load()
+		m.Misses += s.misses.Load()
 	}
 	return m
 }
@@ -838,5 +905,5 @@ func (c *Cache) Metrics() Metrics {
 // NewID returns a new ID to be used as a namespace for cached file
 // blocks.
 func (c *Cache) NewID() uint64 {
-	return atomic.AddUint64(&c.idAlloc, 1)
+	return c.idAlloc.Add(1)
 }

vendor/github.com/cockroachdb/pebble/internal/cache/entry.go

@@ -4,6 +4,8 @@
 
 package cache
 
+import "sync/atomic"
+
 type entryType int8
 
 const (
@@ -56,7 +58,7 @@ type entry struct {
 	ptype entryType
 	// referenced is atomically set to indicate that this entry has been accessed
 	// since the last time one of the clock hands swept it.
-	referenced int32
+	referenced atomic.Bool
 	shard      *shard
 	// Reference count for the entry. The entry is freed when the reference count
 	// drops to zero.
@@ -137,9 +139,7 @@ func (e *entry) setValue(v *Value) {
 	}
 	old := e.val
 	e.val = v
-	if old != nil {
-		old.release()
-	}
+	old.release()
 }
 
 func (e *entry) peekValue() *Value {

vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_normal.go

@@ -10,33 +10,37 @@ package cache
 import (
 	"fmt"
 	"sync/atomic"
+
+	"github.com/cockroachdb/redact"
 )
 
 // refcnt provides an atomic reference count. This version is used when the
 // "tracing" build tag is not enabled. See refcnt_tracing.go for the "tracing"
 // enabled version.
-type refcnt int32
+type refcnt struct {
+	val atomic.Int32
+}
 
 // initialize the reference count to the specified value.
 func (v *refcnt) init(val int32) {
-	*v = refcnt(val)
+	v.val.Store(val)
 }
 
 func (v *refcnt) refs() int32 {
-	return atomic.LoadInt32((*int32)(v))
+	return v.val.Load()
 }
 
 func (v *refcnt) acquire() {
-	switch v := atomic.AddInt32((*int32)(v), 1); {
+	switch v := v.val.Add(1); {
 	case v <= 1:
-		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+		panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
 	}
 }
 
 func (v *refcnt) release() bool {
-	switch v := atomic.AddInt32((*int32)(v), -1); {
+	switch v := v.val.Add(-1); {
 	case v < 0:
-		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+		panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
 	case v == 0:
 		return true
 	default:

vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_tracing.go

@@ -19,22 +19,22 @@ import (
 // debugging logic errors in manipulating the reference count. This version is
 // used when the "tracing" build tag is enabled.
 type refcnt struct {
-	val int32
+	val atomic.Int32
 	sync.Mutex
 	msgs []string
 }
 
 func (v *refcnt) init(val int32) {
-	v.val = val
+	v.val.Store(val)
 	v.trace("init")
 }
 
 func (v *refcnt) refs() int32 {
-	return atomic.LoadInt32(&v.val)
+	return v.val.Load()
 }
 
 func (v *refcnt) acquire() {
-	switch n := atomic.AddInt32(&v.val, 1); {
+	switch n := v.val.Add(1); {
 	case n <= 1:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
 	}
@@ -42,7 +42,7 @@ func (v *refcnt) acquire() {
 }
 
 func (v *refcnt) release() bool {
-	n := atomic.AddInt32(&v.val, -1)
+	n := v.val.Add(-1)
 	switch {
 	case n < 0:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))

vendor/github.com/cockroachdb/pebble/internal/cache/value.go

@@ -40,7 +40,7 @@ func (v *Value) acquire() {
 }
 
 func (v *Value) release() {
-	if v.ref.release() {
+	if v != nil && v.ref.release() {
 		v.free()
 	}
 }

vendor/github.com/cockroachdb/pebble/internal/constants/constants.go

@@ -0,0 +1,17 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package constants
+
+const (
+	// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
+	oneIf64Bit = ^uint(0) >> 63
+
+	// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
+	// - MaxUint32 on 64-bit platforms;
+	// - MaxInt on 32-bit platforms.
+	// It is used when slices are limited to Uint32 on 64-bit platforms (the
+	// length limit for slices is naturally MaxInt on 32-bit platforms).
+	MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
+)

vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go

@@ -35,11 +35,11 @@ type config struct {
 	suffix []string
 }
 
-// IEC produces human readable representations of integer values in IEC units.
-var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}}
+// Bytes produces human readable representations of byte values in IEC units.
+var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
 
-// SI produces human readable representations of integer values in SI units.
-var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}}
+// Count produces human readable representations of unitless values in SI units.
+var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
 
 // Int64 produces a human readable representation of the value.
 func (c *config) Int64(s int64) FormattedString {
@@ -54,18 +54,6 @@ func (c *config) Uint64(s uint64) FormattedString {
 	return FormattedString(humanate(s, c.base, c.suffix))
 }
 
-// Int64 produces a human readable representation of the value in IEC units
-// (base 1024).
-func Int64(s int64) FormattedString {
-	return IEC.Int64(s)
-}
-
-// Uint64 produces a human readable representation of the value in IEC units
-// (base 1024).
-func Uint64(s uint64) FormattedString {
-	return IEC.Uint64(s)
-}
-
 // FormattedString represents a human readable representation of a value. It
 // implements the redact.SafeValue interface to signal that it represents a
 // a string that does not need to be redacted.

vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package invalidating
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/fastrand"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
+// MaybeWrapIfInvariants does nothing in non-invariant builds.
+func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
+	if invariants.Enabled {
+		if fastrand.Uint32n(10) == 1 {
+			return NewIter(iter)
+		}
+	}
+	return iter
+}
+
+// iter tests unsafe key/value slice reuse by modifying the last
+// returned key/value to all 1s.
+type iter struct {
+	iter        base.InternalIterator
+	lastKey     *base.InternalKey
+	lastValue   base.LazyValue
+	ignoreKinds [base.InternalKeyKindMax + 1]bool
+	err         error
+}
+
+// Option configures the behavior of an invalidating iterator.
+type Option interface {
+	apply(*iter)
+}
+
+type funcOpt func(*iter)
+
+func (f funcOpt) apply(i *iter) { f(i) }
+
+// IgnoreKinds constructs an Option that configures an invalidating iterator to
+// skip trashing k/v pairs with the provided key kinds. Some iterators provided
+// key stability guarantees for specific key kinds.
+func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
+	return funcOpt(func(i *iter) {
+		for _, kind := range kinds {
+			i.ignoreKinds[kind] = true
+		}
+	})
+}
+
+// NewIter constructs a new invalidating iterator that wraps the provided
+// iterator, trashing buffers for previously returned keys.
+func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
+	i := &iter{iter: originalIterator}
+	for _, opt := range opts {
+		opt.apply(i)
+	}
+	return i
+}
+
+func (i *iter) update(
+	key *base.InternalKey, value base.LazyValue,
+) (*base.InternalKey, base.LazyValue) {
+	i.trashLastKV()
+	if key == nil {
+		i.lastKey = nil
+		i.lastValue = base.LazyValue{}
+		return nil, base.LazyValue{}
+	}
+
+	i.lastKey = &base.InternalKey{}
+	*i.lastKey = key.Clone()
+	i.lastValue = base.LazyValue{
+		ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
+	}
+	if value.Fetcher != nil {
+		fetcher := new(base.LazyFetcher)
+		*fetcher = *value.Fetcher
+		i.lastValue.Fetcher = fetcher
+	}
+	return i.lastKey, i.lastValue
+}
+
+func (i *iter) trashLastKV() {
+	if i.lastKey == nil {
+		return
+	}
+	if i.ignoreKinds[i.lastKey.Kind()] {
+		return
+	}
+
+	if i.lastKey != nil {
+		for j := range i.lastKey.UserKey {
+			i.lastKey.UserKey[j] = 0xff
+		}
+		i.lastKey.Trailer = 0xffffffffffffffff
+	}
+	for j := range i.lastValue.ValueOrHandle {
+		i.lastValue.ValueOrHandle[j] = 0xff
+	}
+	if i.lastValue.Fetcher != nil {
+		// Not all the LazyFetcher fields are visible, so we zero out the last
+		// value's Fetcher struct entirely.
+		*i.lastValue.Fetcher = base.LazyFetcher{}
+	}
+}
+
+func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekGE(key, flags))
+}
+
+func (i *iter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
+}
+
+func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekLT(key, flags))
+}
+
+func (i *iter) First() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.First())
+}
+
+func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Last())
+}
+
+func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Next())
+}
+
+func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Prev())
+}
+
+func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.NextPrefix(succKey))
+}
+
+func (i *iter) Error() error {
+	if err := i.iter.Error(); err != nil {
+		return err
+	}
+	return i.err
+}
+
+func (i *iter) Close() error {
+	return i.iter.Close()
+}
+
+func (i *iter) SetBounds(lower, upper []byte) {
+	i.iter.SetBounds(lower, upper)
+}
+
+func (i *iter) String() string {
+	return i.iter.String()
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 	// fragments before this span. Defragment backward to find the start of the
 	// defragmented span.
 	i.defragmentBackward()
+
+	// Defragmenting backward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosPrev {
 		// Next once back onto the span.
 		i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 	// fragments after this span. Defragment forward to find the end of the
 	// defragmented span.
 	i.defragmentForward()
+
+	// Defragmenting forward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosNext {
 		// Prev once back onto the span.
 		i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
 		// Next once to move onto y, defragment forward to land on the first z
 		// position.
 		i.iterSpan = i.iter.Next()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no next span while switching directions")
 		}
 		// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
 		// Prev once to move onto y, defragment backward to land on the last x
 		// position.
 		i.iterSpan = i.iter.Prev()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no previous span while switching directions")
 		}
 		// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Next()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Prev()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go

@@ -4,6 +4,8 @@
 
 package keyspan
 
+import "github.com/cockroachdb/pebble/internal/base"
+
 // FilterFunc defines a transform from the input Span into the output Span. The
 // function returns true if the Span should be returned by the iterator, and
 // false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
 type filteringIter struct {
 	iter     FragmentIterator
 	filterFn FilterFunc
+	cmp      base.Compare
 
 	// span is a mutable Span passed to the filterFn. The filterFn is free to
 	// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
 
 // Filter returns a new filteringIter that will filter the Spans from the
 // provided child iterator using the provided FilterFunc.
-func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
-	return &filteringIter{iter: iter, filterFn: filter}
+func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
+	return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
 }
 
 // SeekGE implements FragmentIterator.
 func (i *filteringIter) SeekGE(key []byte) *Span {
-	return i.filter(i.iter.SeekGE(key), +1)
+	span := i.filter(i.iter.SeekGE(key), +1)
+	// i.filter could return a span that's less than key, _if_ the filterFunc
+	// (which has no knowledge of the seek key) mutated the span to end at a key
+	// less than or equal to `key`. Detect this case and next/invalidate the iter.
+	if span != nil && i.cmp(span.End, key) <= 0 {
+		return i.Next()
+	}
+	return span
 }
 
 // SeekLT implements FragmentIterator.
 func (i *filteringIter) SeekLT(key []byte) *Span {
-	return i.filter(i.iter.SeekLT(key), -1)
+	span := i.filter(i.iter.SeekLT(key), -1)
+	// i.filter could return a span that's >= key, _if_ the filterFunc (which has
+	// no knowledge of the seek key) mutated the span to start at a key greater
+	// than or equal to `key`. Detect this case and prev/invalidate the iter.
+	if span != nil && i.cmp(span.Start, key) >= 0 {
+		return i.Prev()
+	}
+	return span
 }
 
 // First implements FragmentIterator.

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -51,6 +51,8 @@ type FragmentIterator interface {
 	Prev() *Span
 
 	// Error returns any accumulated error.
+	//
+	// TODO(jackson): Lift errors into return values on the positioning methods.
 	Error() error
 
 	// Close closes the iterator and returns any accumulated error. Exhausting
@@ -70,9 +72,6 @@ type SpanIterOptions struct {
 	// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
 	// when iterating over range keys.
 	RangeKeyFilters []base.BlockPropertyFilter
-	// Level specifies the level where this sstable is being read. Must be
-	// specified for foreign (i.e. shared not-created-by-this-instance) sstables.
-	Level manifest.Level
 }
 
 // Iter is an iterator over a set of fragmented spans.

vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go

@@ -48,102 +48,3 @@ func VisibleTransform(snapshot uint64) Transformer {
 		return nil
 	})
 }
-
-// TransformerIter is a FragmentIterator that applies a Transformer on all
-// returned keys. Used for when a caller needs to apply a transformer on an
-// iterator but does not otherwise need the mergingiter's merging ability.
-type TransformerIter struct {
-	FragmentIterator
-
-	// Transformer is applied on every Span returned by this iterator.
-	Transformer Transformer
-	// Comparer in use for this keyspace.
-	Compare base.Compare
-
-	span Span
-	err  error
-}
-
-func (t *TransformerIter) applyTransform(span *Span) *Span {
-	t.span = Span{
-		Start: t.span.Start[:0],
-		End:   t.span.End[:0],
-		Keys:  t.span.Keys[:0],
-	}
-	if err := t.Transformer.Transform(t.Compare, *span, &t.span); err != nil {
-		t.err = err
-		return nil
-	}
-	return &t.span
-}
-
-// SeekGE implements the FragmentIterator interface.
-func (t *TransformerIter) SeekGE(key []byte) *Span {
-	span := t.FragmentIterator.SeekGE(key)
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// SeekLT implements the FragmentIterator interface.
-func (t *TransformerIter) SeekLT(key []byte) *Span {
-	span := t.FragmentIterator.SeekLT(key)
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// First implements the FragmentIterator interface.
-func (t *TransformerIter) First() *Span {
-	span := t.FragmentIterator.First()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Last implements the FragmentIterator interface.
-func (t *TransformerIter) Last() *Span {
-	span := t.FragmentIterator.Last()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Next implements the FragmentIterator interface.
-func (t *TransformerIter) Next() *Span {
-	span := t.FragmentIterator.Next()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Prev implements the FragmentIterator interface.
-func (t *TransformerIter) Prev() *Span {
-	span := t.FragmentIterator.Prev()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Error implements the FragmentIterator interface.
-func (t *TransformerIter) Error() error {
-	if t.err != nil {
-		return t.err
-	}
-	return t.FragmentIterator.Error()
-}
-
-// Close implements the FragmentIterator interface.
-func (t *TransformerIter) Close() error {
-	err := t.FragmentIterator.Close()
-	if err != nil {
-		return err
-	}
-	return t.err
-}

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -14,7 +14,7 @@ func Truncate(
 	iter FragmentIterator,
 	lower, upper []byte,
 	start, end *base.InternalKey,
-	panicOnPartialOverlap bool,
+	panicOnUpperTruncate bool,
 ) FragmentIterator {
 	return Filter(iter, func(in *Span, out *Span) (keep bool) {
 		out.Start, out.End = in.Start, in.End
@@ -57,7 +57,6 @@ func Truncate(
 		var truncated bool
 		// Truncate the bounds to lower and upper.
 		if cmp(in.Start, lower) < 0 {
-			truncated = true
 			out.Start = lower
 		}
 		if cmp(in.End, upper) > 0 {
@@ -65,10 +64,10 @@ func Truncate(
 			out.End = upper
 		}
 
-		if panicOnPartialOverlap && truncated {
-			panic("pebble: bounds should not be truncated")
+		if panicOnUpperTruncate && truncated {
+			panic("pebble: upper bound should not be truncated")
 		}
 
 		return !out.Empty() && cmp(out.Start, out.End) < 0
-	})
+	}, cmp)
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go

@@ -108,7 +108,7 @@ type annotation struct {
 }
 
 type leafNode struct {
-	ref   int32
+	ref   atomic.Int32
 	count int16
 	leaf  bool
 	// subtreeCount holds the count of files in the entire subtree formed by
@@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node {
 func newLeafNode() *node {
 	n := leafToNode(new(leafNode))
 	n.leaf = true
-	n.ref = 1
+	n.ref.Store(1)
 	return n
 }
 
 func newNode() *node {
 	n := new(node)
-	n.ref = 1
+	n.ref.Store(1)
 	return n
 }
 
@@ -159,7 +159,7 @@ func newNode() *node {
 // When a node is cloned, the provided pointer will be redirected to the new
 // mutable node.
 func mut(n **node) *node {
-	if atomic.LoadInt32(&(*n).ref) == 1 {
+	if (*n).ref.Load() == 1 {
 		// Exclusive ownership. Can mutate in place.
 
 		// Whenever a node will be mutated, reset its annotations to be marked
@@ -186,7 +186,7 @@ func mut(n **node) *node {
 
 // incRef acquires a reference to the node.
 func (n *node) incRef() {
-	atomic.AddInt32(&n.ref, 1)
+	n.ref.Add(1)
 }
 
 // decRef releases a reference to the node. If requested, the method will unref
@@ -195,8 +195,8 @@ func (n *node) incRef() {
 // new nodes pass contentsToo=false to preserve existing reference counts during
 // operations that should yield a net-zero change to descendant refcounts.
 // When a node is released, its contained files are dereferenced.
-func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
-	if atomic.AddInt32(&n.ref, -1) > 0 {
+func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
+	if n.ref.Add(-1) > 0 {
 		// Other references remain. Can't free.
 		return
 	}
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
 				if obsolete == nil {
 					panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
 				}
-				*obsolete = append(*obsolete, f)
+				// Reference counting is performed on the FileBacking. In the case
+				// of a virtual sstable, this reference counting is performed on
+				// a FileBacking which is shared by every single virtual sstable
+				// with the same backing sstable. If the reference count hits 0,
+				// then we know that the FileBacking won't be required by any
+				// sstable in Pebble, and that the backing sstable can be deleted.
+				*obsolete = append(*obsolete, f.FileBacking)
 			}
 		}
 		if !n.leaf {
@@ -761,8 +767,8 @@ type btree struct {
 
 // Release dereferences and clears the root node of the btree, removing all
 // items from the btree. In doing so, it decrements contained file counts.
-// It returns a slice of newly obsolete files, if any.
-func (t *btree) Release() (obsolete []*FileMetadata) {
+// It returns a slice of newly obsolete backing files, if any.
+func (t *btree) Release() (obsolete []*FileBacking) {
 	if t.root != nil {
 		t.root.decRef(true /* contentsToo */, &obsolete)
 		t.root = nil

vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go

@@ -712,7 +712,7 @@ func (s *L0Sublevels) calculateFlushSplitKeys(flushSplitMaxBytes int64) {
 // InitCompactingFileInfo initializes internal flags relating to compacting
 // files. Must be called after sublevel initialization.
 //
-// Requires DB.mu to be held.
+// Requires DB.mu *and* the manifest lock to be held.
 func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
 	for i := range s.orderedIntervals {
 		s.orderedIntervals[i].compactingFileCount = 0
@@ -735,6 +735,13 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
 		if !f.IsCompacting() {
 			continue
 		}
+		if invariants.Enabled {
+			if s.cmp(s.orderedIntervals[f.minIntervalIndex].startKey.key, f.Smallest.UserKey) != 0 || s.cmp(s.orderedIntervals[f.maxIntervalIndex+1].startKey.key, f.Largest.UserKey) != 0 {
+				panic(fmt.Sprintf("file %s has inconsistent L0 Sublevel interval bounds: %s-%s, %s-%s", f.FileNum,
+					s.orderedIntervals[f.minIntervalIndex].startKey.key, s.orderedIntervals[f.maxIntervalIndex+1].startKey.key,
+					f.Smallest.UserKey, f.Largest.UserKey))
+			}
+		}
 		for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
 			interval := &s.orderedIntervals[i]
 			interval.compactingFileCount++

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -15,20 +15,28 @@ import (
 // LevelMetadata contains metadata for all of the files within
 // a level of the LSM.
 type LevelMetadata struct {
-	level int
-	tree  btree
+	level     int
+	totalSize uint64
+	// NumVirtual is the number of virtual sstables in the level.
+	NumVirtual uint64
+	// VirtualSize is the size of the virtual sstables in the level.
+	VirtualSize uint64
+	tree        btree
 }
 
 // clone makes a copy of the level metadata, implicitly increasing the ref
 // count of every file contained within lm.
 func (lm *LevelMetadata) clone() LevelMetadata {
 	return LevelMetadata{
-		level: lm.level,
-		tree:  lm.tree.Clone(),
+		level:       lm.level,
+		totalSize:   lm.totalSize,
+		NumVirtual:  lm.NumVirtual,
+		VirtualSize: lm.VirtualSize,
+		tree:        lm.tree.Clone(),
 	}
 }
 
-func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
+func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
 	return lm.tree.Release()
 }
 
@@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
 	var lm LevelMetadata
 	lm.level = level
 	lm.tree, _ = makeBTree(bcmp, files)
+	for _, f := range files {
+		lm.totalSize += f.Size
+		if f.Virtual {
+			lm.NumVirtual++
+			lm.VirtualSize += f.Size
+		}
+	}
 	return lm
 }
 
@@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
 	return t, newLevelSlice(t.Iter())
 }
 
+func (lm *LevelMetadata) insert(f *FileMetadata) error {
+	if err := lm.tree.Insert(f); err != nil {
+		return err
+	}
+	lm.totalSize += f.Size
+	if f.Virtual {
+		lm.NumVirtual++
+		lm.VirtualSize += f.Size
+	}
+	return nil
+}
+
+func (lm *LevelMetadata) remove(f *FileMetadata) bool {
+	lm.totalSize -= f.Size
+	if f.Virtual {
+		lm.NumVirtual--
+		lm.VirtualSize -= f.Size
+	}
+	return lm.tree.Delete(f)
+}
+
 // Empty indicates whether there are any files in the level.
 func (lm *LevelMetadata) Empty() bool {
 	return lm.tree.Count() == 0
@@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int {
 	return lm.tree.Count()
 }
 
+// Size returns the cumulative size of all the files within the level.
+func (lm *LevelMetadata) Size() uint64 {
+	return lm.totalSize
+}
+
 // Iter constructs a LevelIterator over the entire level.
 func (lm *LevelMetadata) Iter() LevelIterator {
 	return LevelIterator{iter: lm.tree.Iter()}
@@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
 	return sum
 }
 
+// NumVirtual returns the number of virtual sstables in the level. Its runtime is
+// linear in the length of the slice.
+func (ls *LevelSlice) NumVirtual() uint64 {
+	var n uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			n++
+		}
+	}
+	return n
+}
+
+// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
+// level.
+func (ls *LevelSlice) VirtualSizeSum() uint64 {
+	var sum uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			sum += f.Size
+		}
+	}
+	return sum
+}
+
 // Reslice constructs a new slice backed by the same underlying level, with
 // new start and end positions. Reslice invokes the provided function, passing
 // two LevelIterators: one positioned to i's inclusive start and one

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -49,6 +49,9 @@ type TableStats struct {
 	// The number of point and range deletion entries in the table.
 	NumDeletions uint64
 	// NumRangeKeySets is the total number of range key sets in the table.
+	//
+	// NB: If there's a chance that the sstable contains any range key sets,
+	// then NumRangeKeySets must be > 0.
 	NumRangeKeySets uint64
 	// Estimate of the total disk space that may be dropped by this table's
 	// point deletions by compacting them.
@@ -169,22 +172,13 @@ type FileMetadata struct {
 	// FileNum is the file number.
 	//
 	// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
-	//
-	// TODO(bananabrick): Consider creating separate types for
-	// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
-	// an indentifier for the FileMetadata in Pebble, and also as a handle to
-	// perform reads and writes. We should ensure through types that
-	// FileMetadata.FileNum isn't used to perform reads, and that
-	// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
 	FileNum base.FileNum
 	// Size is the size of the file, in bytes. Size is an approximate value for
 	// virtual sstables.
 	//
-	// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
-	//
-	// TODO(bananabrick): Size is currently used in metrics, and for many key
-	// Pebble level heuristics. Make sure that the heuristics will still work
-	// appropriately with an approximate value of size.
+	// INVARIANTS:
+	// - When !FileMetadata.Virtual, Size == FileBacking.Size.
+	// - Size should be non-zero. Size 0 virtual sstables must not be created.
 	Size uint64
 	// File creation time in seconds since the epoch (1970-01-01 00:00:00
 	// UTC). For ingested sstables, this corresponds to the time the file was
@@ -227,10 +221,6 @@ type FileMetadata struct {
 	// we'd have to write virtual sstable stats to the version edit.
 	Stats TableStats
 
-	// Deleted is set to true if a VersionEdit gets installed that has deleted
-	// this file. Protected by the manifest lock (see versionSet.logLock()).
-	Deleted bool
-
 	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
 	// pick L0 compactions. Only accurate for the most recent Version.
 	SubLevel         int
@@ -292,6 +282,34 @@ type PhysicalFileMeta struct {
 // VirtualFileMeta is used by functions which want a guarantee that their input
 // belongs to a virtual sst and not a physical sst.
 //
+// A VirtualFileMeta inherits all the same fields as a FileMetadata. These
+// fields have additional invariants imposed on them, and/or slightly varying
+// meanings:
+//   - Smallest and Largest (and their counterparts
+//     {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a
+//     key at that exact bound. We make the effort to determine the next smallest
+//     or largest key in an sstable after virtualizing it, to maintain this
+//     tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it
+//     could mean that a rangedel or range key ends at that user key, or has been
+//     truncated to that user key.
+//   - One invariant is that if a rangedel or range key is truncated on its
+//     upper bound, the virtual sstable *must* have a rangedel or range key
+//     sentinel key as its upper bound. This is because truncation yields
+//     an exclusive upper bound for the rangedel/rangekey, and if there are
+//     any points at that exclusive upper bound within the same virtual
+//     sstable, those could get uncovered by this truncation. We enforce this
+//     invariant in calls to keyspan.Truncate.
+//   - Size is an estimate of the size of the virtualized portion of this sstable.
+//     The underlying file's size is stored in FileBacking.Size, though it could
+//     also be estimated or could correspond to just the referenced portion of
+//     a file (eg. if the file originated on another node).
+//   - Size must be > 0.
+//   - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
+//     This means that all keys in the virtual sstable must have seqnums within
+//     [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
+//     a key with a seqnum at either of the bounds. Calculating tight seqnum
+//     bounds would be too expensive and deliver little value.
+//
 // NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
 type VirtualFileMeta struct {
 	*FileMetadata
@@ -589,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive
 	return true
 }
 
+// ContainedWithinSpan returns true if the file key range completely overlaps with the
+// given range ("end" is assumed to exclusive).
+func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool {
+	lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end)
+	return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel()))
+}
+
 // ContainsKeyType returns whether or not the file contains keys of the provided
 // type.
 func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
@@ -686,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 	if !verbose {
 		return b.String()
 	}
+	fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum)
 	if m.HasPointKeys {
 		fmt.Fprintf(&b, " points:[%s-%s]",
 			m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format))
@@ -701,7 +727,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 // representation.
 func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
 	// Split lines of the form:
-	//  000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
+	//  000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...]
 	fields := strings.FieldsFunc(s, func(c rune) bool {
 		switch c {
 		case ':', '[', '-', ']':
@@ -716,6 +742,19 @@ func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
 	m := &FileMetadata{}
 	for len(fields) > 0 {
 		prefix := fields[0]
+		if prefix == "seqnums" {
+			smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64)
+			if err != nil {
+				return m, errors.Newf("malformed input: %s: %s", s, err)
+			}
+			largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64)
+			if err != nil {
+				return m, errors.Newf("malformed input: %s: %s", s, err)
+			}
+			m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum
+			fields = fields[3:]
+			continue
+		}
 		smallest := base.ParsePrettyInternalKey(fields[1])
 		largest := base.ParsePrettyInternalKey(fields[2])
 		switch prefix {
@@ -983,6 +1022,9 @@ func NewVersion(
 		} else {
 			v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp)
 		}
+		for _, f := range files[l] {
+			v.Levels[l].totalSize += f.Size
+		}
 	}
 	if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
 		panic(err)
@@ -1015,7 +1057,7 @@ func NewVersion(
 // key in a higher level table that has both the same user key and a higher
 // sequence number.
 type Version struct {
-	refs int32
+	refs atomic.Int32
 
 	// The level 0 sstables are organized in a series of sublevels. Similar to
 	// the seqnum invariant in normal levels, there is no internal key in a
@@ -1145,12 +1187,12 @@ func ParseVersionDebug(
 
 // Refs returns the number of references to the version.
 func (v *Version) Refs() int32 {
-	return atomic.LoadInt32(&v.refs)
+	return v.refs.Load()
 }
 
 // Ref increments the version refcount.
 func (v *Version) Ref() {
-	atomic.AddInt32(&v.refs, 1)
+	v.refs.Add(1)
 }
 
 // Unref decrements the version refcount. If the last reference to the version
@@ -1158,16 +1200,11 @@ func (v *Version) Ref() {
 // Deleted callback is invoked. Requires that the VersionList mutex is NOT
 // locked.
 func (v *Version) Unref() {
-	if atomic.AddInt32(&v.refs, -1) == 0 {
+	if v.refs.Add(-1) == 0 {
 		l := v.list
 		l.mu.Lock()
 		l.Remove(v)
-		obsolete := v.unrefFiles()
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
+		v.Deleted(v.unrefFiles())
 		l.mu.Unlock()
 	}
 }
@@ -1177,19 +1214,14 @@ func (v *Version) Unref() {
 // the Deleted callback is invoked. Requires that the VersionList mutex is
 // already locked.
 func (v *Version) UnrefLocked() {
-	if atomic.AddInt32(&v.refs, -1) == 0 {
+	if v.refs.Add(-1) == 0 {
 		v.list.Remove(v)
-		obsolete := v.unrefFiles()
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
+		v.Deleted(v.unrefFiles())
 	}
 }
 
-func (v *Version) unrefFiles() []*FileMetadata {
-	var obsolete []*FileMetadata
+func (v *Version) unrefFiles() []*FileBacking {
+	var obsolete []*FileBacking
 	for _, lm := range v.Levels {
 		obsolete = append(obsolete, lm.release()...)
 	}
@@ -1321,16 +1353,20 @@ func (v *Version) Overlaps(
 // CheckOrdering checks that the files are consistent with respect to
 // increasing file numbers (for level 0 files) and increasing and non-
 // overlapping internal key ranges (for level non-0 files).
-func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
+func (v *Version) CheckOrdering(
+	cmp Compare, format base.FormatKey, order OrderingInvariants,
+) error {
 	for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
 		sublevelIter := v.L0SublevelFiles[sublevel].Iter()
-		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
+		// Sublevels have NEVER allowed split user keys, so we can pass
+		// ProhibitSplitUserKeys.
+		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
 
 	for level, lm := range v.Levels {
-		if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
+		if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
@@ -1399,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
 	v.list = nil // avoid memory leaks
 }
 
+// OrderingInvariants dictates the file ordering invariants active.
+type OrderingInvariants int8
+
+const (
+	// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
+	// contain the same user key.
+	ProhibitSplitUserKeys OrderingInvariants = iota
+	// AllowSplitUserKeys indicates that adjacent files within a level may
+	// contain the same user key. This is only allowed by historical format
+	// major versions.
+	//
+	// TODO(jackson): Remove.
+	AllowSplitUserKeys
+)
+
 // CheckOrdering checks that the files are consistent with respect to
 // seqnums (for level 0 files -- see detailed comment below) and increasing and non-
 // overlapping internal key ranges (for non-level 0 files).
-func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
+//
+// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
+// inclusive of the same user key. Pebble no longer creates version edits
+// installing such files, and Pebble databases with sufficiently high format
+// major version should no longer have any such files within their LSM.
+// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
+// earlier format major versions.
+func CheckOrdering(
+	cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
+) error {
 	// The invariants to check for L0 sublevels are the same as the ones to
 	// check for all other levels. However, if L0 is not organized into
 	// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1480,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
 						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
 						f.Smallest.Pretty(format), f.Largest.Pretty(format))
 				}
-				if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
-					return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
-						errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
-						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
-						f.Smallest.Pretty(format), f.Largest.Pretty(format))
+
+				// What's considered "overlapping" is dependent on the format
+				// major version. If ordering=ProhibitSplitUserKeys, then both
+				// files cannot contain keys with the same user keys. If the
+				// bounds have the same user key, the previous file's boundary
+				// must have a Trailer indicating that it's exclusive.
+				switch ordering {
+				case AllowSplitUserKeys:
+					if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				case ProhibitSplitUserKeys:
+					if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				default:
+					panic("unreachable")
 				}
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -450,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 	return nil
 }
 
-// String implements fmt.Stringer for a VersionEdit.
-func (v *VersionEdit) String() string {
+func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
 	var buf bytes.Buffer
 	if v.ComparerName != "" {
 		fmt.Fprintf(&buf, "  comparer:     %s", v.ComparerName)
@@ -482,7 +481,12 @@ func (v *VersionEdit) String() string {
 		fmt.Fprintf(&buf, "  deleted:       L%d %s\n", df.Level, df.FileNum)
 	}
 	for _, nf := range v.NewFiles {
-		fmt.Fprintf(&buf, "  added:         L%d %s", nf.Level, nf.Meta.String())
+		fmt.Fprintf(&buf, "  added:         L%d", nf.Level)
+		if verbose {
+			fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
+		} else {
+			fmt.Fprintf(&buf, " %s", nf.Meta.String())
+		}
 		if nf.Meta.CreationTime != 0 {
 			fmt.Fprintf(&buf, " (%s)",
 				time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -492,6 +496,16 @@ func (v *VersionEdit) String() string {
 	return buf.String()
 }
 
+// DebugString is a more verbose version of String(). Use this in tests.
+func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
+	return v.string(true /* verbose */, fmtKey)
+}
+
+// String implements fmt.Stringer for a VersionEdit.
+func (v *VersionEdit) String() string {
+	return v.string(false /* verbose */, base.DefaultFormatter)
+}
+
 // Encode encodes an edit to the specified writer.
 func (v *VersionEdit) Encode(w io.Writer) error {
 	e := versionEditEncoder{new(bytes.Buffer)}
@@ -769,6 +783,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
 	}
 	for _, fb := range ve.CreatedBackingTables {
+		if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
+			// There is already a FileBacking associated with fb.DiskFileNum.
+			// This should never happen. There must always be only one FileBacking
+			// associated with a backing sstable.
+			panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
+		}
 		b.AddedFileBacking[fb.DiskFileNum] = fb
 	}
 
@@ -832,6 +852,9 @@ func AccumulateIncompleteAndApplySingleVE(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	backingStateMap map[base.DiskFileNum]*FileBacking,
+	addBackingFunc func(*FileBacking),
+	removeBackingFunc func(base.DiskFileNum),
+	orderingInvariants OrderingInvariants,
 ) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
 	if len(ve.RemovedBackingTables) != 0 {
 		panic("pebble: invalid incomplete version edit")
@@ -843,14 +866,14 @@ func AccumulateIncompleteAndApplySingleVE(
 	}
 	zombies = make(map[base.DiskFileNum]uint64)
 	v, err := b.Apply(
-		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
+		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
 	)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	for _, s := range b.AddedFileBacking {
-		backingStateMap[s.DiskFileNum] = s
+		addBackingFunc(s)
 	}
 
 	for fileNum := range zombies {
@@ -861,10 +884,9 @@ func AccumulateIncompleteAndApplySingleVE(
 			ve.RemovedBackingTables = append(
 				ve.RemovedBackingTables, fileNum,
 			)
-			delete(backingStateMap, fileNum)
+			removeBackingFunc(fileNum)
 		}
 	}
-
 	return v, zombies, nil
 }
 
@@ -885,6 +907,7 @@ func (b *BulkVersionEdit) Apply(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	zombies map[base.DiskFileNum]uint64,
+	orderingInvariants OrderingInvariants,
 ) (*Version, error) {
 	addZombie := func(state *FileBacking) {
 		if zombies != nil {
@@ -953,7 +976,7 @@ func (b *BulkVersionEdit) Apply(
 		// level.
 
 		for _, f := range deletedFilesMap {
-			if obsolete := v.Levels[level].tree.Delete(f); obsolete {
+			if obsolete := v.Levels[level].remove(f); obsolete {
 				// Deleting a file from the B-Tree may decrement its
 				// reference count. However, because we cloned the
 				// previous level's B-Tree, this should never result in a
@@ -962,7 +985,7 @@ func (b *BulkVersionEdit) Apply(
 				return nil, err
 			}
 			if f.HasRangeKeys {
-				if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
+				if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
 					// Deleting a file from the B-Tree may decrement its
 					// reference count. However, because we cloned the
 					// previous level's B-Tree, this should never result in a
@@ -1015,7 +1038,7 @@ func (b *BulkVersionEdit) Apply(
 			f.AllowedSeeks.Store(allowedSeeks)
 			f.InitAllowedSeeks = allowedSeeks
 
-			err := lm.tree.Insert(f)
+			err := lm.insert(f)
 			// We're adding this file to the new version, so increment the
 			// latest refs count.
 			f.LatestRef()
@@ -1023,7 +1046,7 @@ func (b *BulkVersionEdit) Apply(
 				return nil, errors.Wrap(err, "pebble")
 			}
 			if f.HasRangeKeys {
-				err = lmRange.tree.Insert(f)
+				err = lmRange.insert(f)
 				if err != nil {
 					return nil, errors.Wrap(err, "pebble")
 				}
@@ -1068,7 +1091,7 @@ func (b *BulkVersionEdit) Apply(
 			} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
-			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			continue
@@ -1089,7 +1112,7 @@ func (b *BulkVersionEdit) Apply(
 					end.Prev()
 				}
 			})
-			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -6,7 +6,6 @@ package rangekey
 
 import (
 	"bytes"
-	"fmt"
 	"math"
 	"sort"
 
@@ -53,15 +52,15 @@ import (
 //	                        │
 //	                        ╰── <?>.Next
 type UserIteratorConfig struct {
-	snapshot   uint64
-	comparer   *base.Comparer
-	miter      keyspan.MergingIter
-	biter      keyspan.BoundedIter
-	diter      keyspan.DefragmentingIter
-	liters     [manifest.NumLevels]keyspan.LevelIter
-	litersUsed int
-	onlySets   bool
-	bufs       *Buffers
+	snapshot     uint64
+	comparer     *base.Comparer
+	miter        keyspan.MergingIter
+	biter        keyspan.BoundedIter
+	diter        keyspan.DefragmentingIter
+	liters       [manifest.NumLevels]keyspan.LevelIter
+	litersUsed   int
+	internalKeys bool
+	bufs         *Buffers
 }
 
 // Buffers holds various buffers used for range key iteration. They're exposed
@@ -80,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
 
 // Init initializes the range key iterator stack for user iteration. The
 // resulting fragment iterator applies range key semantics, defragments spans
-// according to their user-observable state and, if onlySets = true, removes all
+// according to their user-observable state and, if !internalKeys, removes all
 // Keys other than RangeKeySets describing the current state of range keys. The
-// resulting spans contain Keys sorted by Suffix.
+// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
+// in which case they remain sorted by trailer descending).
 //
 // The snapshot sequence number parameter determines which keys are visible. Any
 // keys not visible at the provided snapshot are ignored.
@@ -92,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
 	lower, upper []byte,
 	hasPrefix *bool,
 	prefix *[]byte,
-	onlySets bool,
+	internalKeys bool,
 	bufs *Buffers,
 	iters ...keyspan.FragmentIterator,
 ) keyspan.FragmentIterator {
 	ui.snapshot = snapshot
 	ui.comparer = comparer
-	ui.onlySets = onlySets
+	ui.internalKeys = internalKeys
 	ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
 	ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
-	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	if internalKeys {
+		ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	} else {
+		ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	}
 	ui.litersUsed = 0
 	ui.bufs = bufs
 	return &ui.diter
@@ -137,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
 // of unset keys, removal of keys overwritten by a set at the same suffix, etc)
 // and then non-RangeKeySet keys are removed. The resulting transformed spans
 // only contain RangeKeySets describing the state visible at the provided
-// sequence number, and hold their Keys sorted by Suffix.
+// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
+// is true, then keys remain sorted by trailer.
 func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
 	// Apply shadowing of keys.
 	dst.Start = s.Start
@@ -149,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 	if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
 		return err
 	}
-	// During user iteration over range keys, unsets and deletes don't matter.
-	// Remove them if onlySets = true. This step helps logical defragmentation
-	// during iteration.
+	if ui.internalKeys {
+		if s.KeysOrder != keyspan.ByTrailerDesc {
+			panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
+		}
+		dst.Keys = ui.bufs.sortBuf.Keys
+		keyspan.SortKeysByTrailer(&dst.Keys)
+		return nil
+	}
+	// During user iteration over range keys, unsets and deletes don't matter. This
+	// step helps logical defragmentation during iteration.
 	keys := ui.bufs.sortBuf.Keys
 	dst.Keys = dst.Keys[:0]
 	for i := range keys {
@@ -165,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
 				panic("pebble: keys unexpectedly not in ascending suffix order")
 			}
-			if ui.onlySets {
-				// Skip.
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
+			// Skip.
+			continue
 		case base.InternalKeyKindRangeKeyDelete:
-			if ui.onlySets {
-				// Skip.
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
+			// Skip.
+			continue
 		default:
 			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
 		}
@@ -194,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 // sequence numbers). It's intended for use during user iteration, when the
 // wrapped keyspan iterator is merging spans across all levels of the LSM.
 func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
+	// This method is not called with internalKeys = true.
+	if ui.internalKeys {
+		panic("unexpected call to ShouldDefragment with internalKeys = true")
+	}
 	// This implementation must only be used on spans that have transformed by
 	// ui.Transform. The transform applies shadowing, removes all keys besides
 	// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -209,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 	ret := true
 	for i := range a.Keys {
 		if invariants.Enabled {
-			if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
-				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
+			if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
+				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
 				panic("pebble: unexpected non-RangeKeySet during defragmentation")
 			}
 			if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||
@@ -368,78 +378,3 @@ func coalesce(
 	}
 	return nil
 }
-
-// ForeignSSTTransformer implements a keyspan.Transformer for range keys in
-// foreign sstables (i.e. shared sstables not created by us). It is largely
-// similar to the Transform function implemented in UserIteratorConfig in that
-// it calls coalesce to remove range keys shadowed by other range keys, but also
-// retains the range key that does the shadowing. In addition, it elides
-// RangeKey unsets/dels in L6 as they are inapplicable when reading from a
-// different Pebble instance.
-type ForeignSSTTransformer struct {
-	Comparer *base.Comparer
-	Level    int
-	sortBuf  keyspan.KeysBySuffix
-}
-
-// Transform implements the Transformer interface.
-func (f *ForeignSSTTransformer) Transform(
-	cmp base.Compare, s keyspan.Span, dst *keyspan.Span,
-) error {
-	// Apply shadowing of keys.
-	dst.Start = s.Start
-	dst.End = s.End
-	f.sortBuf = keyspan.KeysBySuffix{
-		Cmp:  cmp,
-		Keys: f.sortBuf.Keys[:0],
-	}
-	if err := coalesce(f.Comparer.Equal, &f.sortBuf, math.MaxUint64, s.Keys); err != nil {
-		return err
-	}
-	keys := f.sortBuf.Keys
-	dst.Keys = dst.Keys[:0]
-	for i := range keys {
-		seqNum := keys[i].SeqNum()
-		switch keys[i].Kind() {
-		case base.InternalKeyKindRangeKeySet:
-			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
-				panic("pebble: keys unexpectedly not in ascending suffix order")
-			}
-			switch f.Level {
-			case 5:
-				fallthrough
-			case 6:
-				if seqNum != base.SeqNumForLevel(f.Level) {
-					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
-				}
-			}
-		case base.InternalKeyKindRangeKeyUnset:
-			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
-				panic("pebble: keys unexpectedly not in ascending suffix order")
-			}
-			fallthrough
-		case base.InternalKeyKindRangeKeyDelete:
-			switch f.Level {
-			case 5:
-				// Emit this key.
-				if seqNum != base.SeqNumForLevel(f.Level) {
-					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
-				}
-			case 6:
-				// Skip this key, as foreign sstable in L6 do not need to emit range key
-				// unsets/dels as they do not apply to any other sstables.
-				continue
-			}
-		default:
-			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
-		}
-		dst.Keys = append(dst.Keys, keyspan.Key{
-			Trailer: base.MakeTrailer(seqNum, keys[i].Kind()),
-			Suffix:  keys[i].Suffix,
-			Value:   keys[i].Value,
-		})
-	}
-	// coalesce results in dst.Keys being sorted by Suffix.
-	dst.KeysOrder = keyspan.BySuffixAsc
-	return nil
-}

vendor/github.com/cockroachdb/pebble/internal/rate/rate.go

@@ -1,387 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package rate provides a rate limiter.
-package rate // import "github.com/cockroachdb/pebble/internal/rate"
-
-import (
-	"context"
-	"math"
-	"sync"
-	"time"
-
-	"github.com/cockroachdb/errors"
-)
-
-// Limit defines the maximum frequency of some events.
-// Limit is represented as number of events per second.
-// A zero Limit allows no events.
-type Limit float64
-
-// Inf is the infinite rate limit; it allows all events (even if burst is zero).
-const Inf = Limit(math.MaxFloat64)
-
-// Every converts a minimum time interval between events to a Limit.
-func Every(interval time.Duration) Limit {
-	if interval <= 0 {
-		return Inf
-	}
-	return 1 / Limit(interval.Seconds())
-}
-
-// A Limiter controls how frequently events are allowed to happen.
-// It implements a "token bucket" of size b, initially full and refilled
-// at rate r tokens per second.
-// Informally, in any large enough time interval, the Limiter limits the
-// rate to r tokens per second, with a maximum burst size of b events.
-// As a special case, if r == Inf (the infinite rate), b is ignored.
-// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
-//
-// The zero value is a valid Limiter, but it will reject all events.
-// Use NewLimiter to create non-zero Limiters.
-//
-// Limiter has three main methods, Allow, Reserve, and Wait.
-// Most callers should use Wait.
-//
-// Each of the three methods consumes a single token.
-// They differ in their behavior when no token is available.
-// If no token is available, Allow returns false.
-// If no token is available, Reserve returns a reservation for a future token
-// and the amount of time the caller must wait before using it.
-// If no token is available, Wait blocks until one can be obtained
-// or its associated context.Context is canceled.
-//
-// The methods AllowN, ReserveN, and WaitN consume n tokens.
-type Limiter struct {
-	limit Limit
-	burst int
-
-	mu     sync.Mutex
-	tokens float64
-	// last is the last time the limiter's tokens field was updated
-	last time.Time
-	// lastEvent is the latest time of a rate-limited event (past or future)
-	lastEvent time.Time
-}
-
-// Limit returns the maximum overall event rate.
-func (lim *Limiter) Limit() Limit {
-	lim.mu.Lock()
-	defer lim.mu.Unlock()
-	return lim.limit
-}
-
-// Burst returns the maximum burst size. Burst is the maximum number of tokens
-// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
-// Burst values allow more events to happen at once.
-// A zero Burst allows no events, unless limit == Inf.
-func (lim *Limiter) Burst() int {
-	return lim.burst
-}
-
-// NewLimiter returns a new Limiter that allows events up to rate r and permits
-// bursts of at most b tokens.
-func NewLimiter(r Limit, b int) *Limiter {
-	return &Limiter{
-		limit: r,
-		burst: b,
-	}
-}
-
-// Allow is shorthand for AllowN(time.Now(), 1).
-func (lim *Limiter) Allow() bool {
-	return lim.AllowN(time.Now(), 1)
-}
-
-// AllowN reports whether n events may happen at time now.
-// Use this method if you intend to drop / skip events that exceed the rate limit.
-// Otherwise use Reserve or Wait.
-func (lim *Limiter) AllowN(now time.Time, n int) bool {
-	return lim.reserveN(now, n, 0).ok
-}
-
-// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
-// A Reservation may be canceled, which may enable the Limiter to permit additional events.
-type Reservation struct {
-	ok        bool
-	lim       *Limiter
-	tokens    int
-	timeToAct time.Time
-	// This is the Limit at reservation time, it can change later.
-	limit Limit
-}
-
-// OK returns whether the limiter can provide the requested number of tokens
-// within the maximum wait time.  If OK is false, Delay returns InfDuration, and
-// Cancel does nothing.
-func (r *Reservation) OK() bool {
-	return r.ok
-}
-
-// Delay is shorthand for DelayFrom(time.Now()).
-func (r *Reservation) Delay() time.Duration {
-	return r.DelayFrom(time.Now())
-}
-
-// InfDuration is the duration returned by Delay when a Reservation is not OK.
-const InfDuration = time.Duration(1<<63 - 1)
-
-// DelayFrom returns the duration for which the reservation holder must wait
-// before taking the reserved action.  Zero duration means act immediately.
-// InfDuration means the limiter cannot grant the tokens requested in this
-// Reservation within the maximum wait time.
-func (r *Reservation) DelayFrom(now time.Time) time.Duration {
-	if !r.ok {
-		return InfDuration
-	}
-	delay := r.timeToAct.Sub(now)
-	if delay < 0 {
-		return 0
-	}
-	return delay
-}
-
-// Cancel is shorthand for CancelAt(time.Now()).
-func (r *Reservation) Cancel() {
-	r.CancelAt(time.Now())
-}
-
-// CancelAt indicates that the reservation holder will not perform the reserved action
-// and reverses the effects of this Reservation on the rate limit as much as possible,
-// considering that other reservations may have already been made.
-func (r *Reservation) CancelAt(now time.Time) {
-	if !r.ok {
-		return
-	}
-
-	r.lim.mu.Lock()
-	defer r.lim.mu.Unlock()
-
-	if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
-		return
-	}
-
-	// calculate tokens to restore
-	// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
-	// after r was obtained. These tokens should not be restored.
-	restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
-	if restoreTokens <= 0 {
-		return
-	}
-	// advance time to now
-	now, _, tokens := r.lim.advance(now)
-	// calculate new number of tokens
-	tokens += restoreTokens
-	if burst := float64(r.lim.burst); tokens > burst {
-		tokens = burst
-	}
-	// update state
-	r.lim.last = now
-	r.lim.tokens = tokens
-	if r.timeToAct == r.lim.lastEvent {
-		prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
-		if !prevEvent.Before(now) {
-			r.lim.lastEvent = prevEvent
-		}
-	}
-}
-
-// Reserve is shorthand for ReserveN(time.Now(), 1).
-func (lim *Limiter) Reserve() Reservation {
-	return lim.ReserveN(time.Now(), 1)
-}
-
-// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
-// The Limiter takes this Reservation into account when allowing future events.
-// ReserveN returns false if n exceeds the Limiter's burst size.
-// Usage example:
-//
-//	r := lim.ReserveN(time.Now(), 1)
-//	if !r.OK() {
-//	  // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
-//	  return
-//	}
-//	time.Sleep(r.Delay())
-//	Act()
-//
-// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
-// If you need to respect a deadline or cancel the delay, use Wait instead.
-// To drop or skip events exceeding rate limit, use Allow instead.
-func (lim *Limiter) ReserveN(now time.Time, n int) Reservation {
-	return lim.reserveN(now, n, InfDuration)
-}
-
-// Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) Wait(ctx context.Context) (err error) {
-	return lim.WaitN(ctx, 1)
-}
-
-// WaitN blocks until lim permits n events to happen.
-// It returns an error if n exceeds the Limiter's burst size, the Context is
-// canceled, or the expected wait time exceeds the Context's Deadline.
-func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
-	if n > lim.burst && lim.limit != Inf {
-		return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst))
-	}
-	// Check if ctx is already cancelled
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
-	}
-	// Determine wait limit
-	now := time.Now()
-	waitLimit := InfDuration
-	if deadline, ok := ctx.Deadline(); ok {
-		waitLimit = deadline.Sub(now)
-	}
-	// Reserve
-	r := lim.reserveN(now, n, waitLimit)
-	if !r.ok {
-		return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n))
-	}
-	// Wait
-	d := r.DelayFrom(now)
-	if d <= 0 {
-		return nil
-	}
-	t := time.NewTimer(d)
-	defer t.Stop()
-	select {
-	case <-t.C:
-		// We can proceed.
-		return nil
-	case <-ctx.Done():
-		// Context was canceled before we could proceed.  Cancel the
-		// reservation, which may permit other events to proceed sooner.
-		r.Cancel()
-		return ctx.Err()
-	}
-}
-
-// Delay is shorthand for DelayN(time.Now(), 1).
-func (lim *Limiter) Delay() time.Duration {
-	return lim.DelayN(time.Now(), 1)
-}
-
-// DelayN returns the delay to wait to permit n events to happen. Zero duration
-// means act immediately. InfDuration means the limiter cannot grant the tokens
-// requested within the maximum wait time.
-func (lim *Limiter) DelayN(now time.Time, n int) time.Duration {
-	r := lim.reserveN(now, n, InfDuration)
-	return r.DelayFrom(now)
-}
-
-// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
-func (lim *Limiter) SetLimit(newLimit Limit) {
-	lim.SetLimitAt(time.Now(), newLimit)
-}
-
-// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
-// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
-// before SetLimitAt was called.
-func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
-	lim.mu.Lock()
-	defer lim.mu.Unlock()
-
-	now, _, tokens := lim.advance(now)
-
-	lim.last = now
-	lim.tokens = tokens
-	lim.limit = newLimit
-}
-
-// reserveN is a helper method for AllowN, ReserveN, and WaitN.
-// maxFutureReserve specifies the maximum reservation wait duration allowed.
-// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
-func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
-	lim.mu.Lock()
-
-	if lim.limit == Inf {
-		lim.mu.Unlock()
-		return Reservation{
-			ok:        true,
-			lim:       lim,
-			tokens:    n,
-			timeToAct: now,
-		}
-	}
-
-	now, last, tokens := lim.advance(now)
-
-	// Calculate the remaining number of tokens resulting from the request.
-	tokens -= float64(n)
-
-	// Calculate the wait duration
-	var waitDuration time.Duration
-	if tokens < 0 {
-		waitDuration = lim.limit.durationFromTokens(-tokens)
-	}
-
-	// Decide result
-	ok := n <= lim.burst && waitDuration <= maxFutureReserve
-
-	// Prepare reservation
-	r := Reservation{
-		ok:    ok,
-		lim:   lim,
-		limit: lim.limit,
-	}
-	if ok {
-		r.tokens = n
-		r.timeToAct = now.Add(waitDuration)
-	}
-
-	// Update state
-	if ok {
-		lim.last = now
-		lim.tokens = tokens
-		lim.lastEvent = r.timeToAct
-	} else {
-		lim.last = last
-	}
-
-	lim.mu.Unlock()
-	return r
-}
-
-// advance calculates and returns an updated state for lim resulting from the passage of time.
-// lim is not changed.
-func (lim *Limiter) advance(
-	now time.Time,
-) (newNow time.Time, newLast time.Time, newTokens float64) {
-	last := lim.last
-	if now.Before(last) {
-		last = now
-	}
-
-	// Avoid making delta overflow below when last is very old.
-	maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
-	elapsed := now.Sub(last)
-	if elapsed > maxElapsed {
-		elapsed = maxElapsed
-	}
-
-	// Calculate the new number of tokens, due to time that passed.
-	delta := lim.limit.tokensFromDuration(elapsed)
-	tokens := lim.tokens + delta
-	if burst := float64(lim.burst); tokens > burst {
-		tokens = burst
-	}
-
-	return now, last, tokens
-}
-
-// durationFromTokens is a unit conversion function from the number of tokens to the duration
-// of time it takes to accumulate them at a rate of limit tokens per second.
-func (limit Limit) durationFromTokens(tokens float64) time.Duration {
-	seconds := tokens / float64(limit)
-	return time.Nanosecond * time.Duration(1e9*seconds)
-}
-
-// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
-// which could be accumulated during that duration at a rate of limit tokens per second.
-func (limit Limit) tokensFromDuration(d time.Duration) float64 {
-	return d.Seconds() * float64(limit)
-}

vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go

@@ -20,17 +20,27 @@ import (
 	"strings"
 
 	"github.com/cockroachdb/pebble/internal/base"
+	"golang.org/x/exp/constraints"
+	"golang.org/x/exp/rand"
 )
 
 const alpha = "abcdefghijklmnopqrstuvwxyz"
 
 const suffixDelim = '@'
 
+var inverseAlphabet = make(map[byte]int64, len(alpha))
+
+func init() {
+	for i := range alpha {
+		inverseAlphabet[alpha[i]] = int64(i)
+	}
+}
+
 // MaxSuffixLen is the maximum length of a suffix generated by this package.
-var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64))
+var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64)))
 
 // Comparer is the comparer for test keys generated by this package.
-var Comparer *base.Comparer = &base.Comparer{
+var Comparer = &base.Comparer{
 	Compare: compare,
 	Equal:   func(a, b []byte) bool { return compare(a, b) == 0 },
 	AbbreviatedKey: func(k []byte) uint64 {
@@ -136,7 +146,7 @@ func compareTimestamps(a, b []byte) int {
 // Keyspace describes a finite keyspace of unsuffixed test keys.
 type Keyspace interface {
 	// Count returns the number of keys that exist within this keyspace.
-	Count() int
+	Count() int64
 
 	// MaxLen returns the maximum length, in bytes, of a key within this
 	// keyspace. This is only guaranteed to return an upper bound.
@@ -144,20 +154,21 @@ type Keyspace interface {
 
 	// Slice returns the sub-keyspace from index i, inclusive, to index j,
 	// exclusive. The receiver is unmodified.
-	Slice(i, j int) Keyspace
+	Slice(i, j int64) Keyspace
 
 	// EveryN returns a key space that includes 1 key for every N keys in the
 	// original keyspace. The receiver is unmodified.
-	EveryN(n int) Keyspace
+	EveryN(n int64) Keyspace
 
-	key(buf []byte, i int) int
+	// key writes the i-th key to the buffer and returns the length.
+	key(buf []byte, i int64) int
 }
 
 // Divvy divides the provided keyspace into N equal portions, containing
 // disjoint keys evenly distributed across the keyspace.
-func Divvy(ks Keyspace, n int) []Keyspace {
+func Divvy(ks Keyspace, n int64) []Keyspace {
 	ret := make([]Keyspace, n)
-	for i := 0; i < n; i++ {
+	for i := int64(0); i < n; i++ {
 		ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
 	}
 	return ret
@@ -175,7 +186,7 @@ func Alpha(maxLength int) Keyspace {
 
 // KeyAt returns the i-th key within the keyspace with a suffix encoding the
 // timestamp t.
-func KeyAt(k Keyspace, i int, t int) []byte {
+func KeyAt(k Keyspace, i int64, t int64) []byte {
 	b := make([]byte, k.MaxLen()+MaxSuffixLen)
 	return b[:WriteKeyAt(b, k, i, t)]
 }
@@ -183,20 +194,20 @@ func KeyAt(k Keyspace, i int, t int) []byte {
 // WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
 // suffix encoding the timestamp t suffix. It returns the number of bytes
 // written.
-func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int {
+func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int {
 	n := WriteKey(dst, k, i)
 	n += WriteSuffix(dst[n:], t)
 	return n
 }
 
 // Suffix returns the test keys suffix representation of timestamp t.
-func Suffix(t int) []byte {
+func Suffix(t int64) []byte {
 	b := make([]byte, MaxSuffixLen)
 	return b[:WriteSuffix(b, t)]
 }
 
 // SuffixLen returns the exact length of the given suffix when encoded.
-func SuffixLen(t int) int {
+func SuffixLen(t int64) int {
 	// Begin at 1 for the '@' delimiter, 1 for a single digit.
 	n := 2
 	t /= 10
@@ -208,42 +219,42 @@ func SuffixLen(t int) int {
 }
 
 // ParseSuffix returns the integer representation of the encoded suffix.
-func ParseSuffix(s []byte) (int, error) {
-	return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
+func ParseSuffix(s []byte) (int64, error) {
+	return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64)
 }
 
 // WriteSuffix writes the test keys suffix representation of timestamp t to dst,
 // returning the number of bytes written.
-func WriteSuffix(dst []byte, t int) int {
+func WriteSuffix(dst []byte, t int64) int {
 	dst[0] = suffixDelim
 	n := 1
-	n += len(strconv.AppendInt(dst[n:n], int64(t), 10))
+	n += len(strconv.AppendInt(dst[n:n], t, 10))
 	return n
 }
 
 // Key returns the i-th unsuffixed key within the keyspace.
-func Key(k Keyspace, i int) []byte {
+func Key(k Keyspace, i int64) []byte {
 	b := make([]byte, k.MaxLen())
 	return b[:k.key(b, i)]
 }
 
 // WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
 // returns the number of bytes written.
-func WriteKey(dst []byte, k Keyspace, i int) int {
+func WriteKey(dst []byte, k Keyspace, i int64) int {
 	return k.key(dst, i)
 }
 
 type alphabet struct {
 	alphabet  []byte
 	maxLength int
-	headSkip  int
-	tailSkip  int
-	increment int
+	headSkip  int64
+	tailSkip  int64
+	increment int64
 }
 
-func (a alphabet) Count() int {
+func (a alphabet) Count() int64 {
 	// Calculate the total number of keys, ignoring the increment.
-	total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip)
+	total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip
 
 	// The increment dictates that we take every N keys, where N = a.increment.
 	// Consider a total containing the 5 keys:
@@ -263,20 +274,25 @@ func (a alphabet) MaxLen() int {
 	return a.maxLength
 }
 
-func (a alphabet) Slice(i, j int) Keyspace {
+func (a alphabet) Slice(i, j int64) Keyspace {
 	s := a
 	s.headSkip += i
 	s.tailSkip += a.Count() - j
 	return s
 }
 
-func (a alphabet) EveryN(n int) Keyspace {
+func (a alphabet) EveryN(n int64) Keyspace {
 	s := a
 	s.increment *= n
 	return s
 }
 
-func keyCount(n, l int) int {
+func keyCount(n, l int) int64 {
+	if n == 0 {
+		return 0
+	} else if n == 1 {
+		return int64(l)
+	}
 	// The number of representable keys in the keyspace is a function of the
 	// length of the alphabet n and the max key length l. Consider how the
 	// number of representable keys grows as l increases:
@@ -286,10 +302,10 @@ func keyCount(n, l int) int {
 	// l = 3: n + n^2 + n^3
 	// ...
 	// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
-	return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1)
+	return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1)
 }
 
-func (a alphabet) key(buf []byte, idx int) int {
+func (a alphabet) key(buf []byte, idx int64) int {
 	// This function generates keys of length 1..maxKeyLength, pulling
 	// characters from the alphabet. The idx determines which key to generate,
 	// generating the i-th lexicographically next key.
@@ -308,14 +324,14 @@ func (a alphabet) key(buf []byte, idx int) int {
 		keyCount(len(a.alphabet), a.maxLength))
 }
 
-func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
+func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int {
 	if keyCount == 0 || i > keyCount || i < 0 {
 		return 0
 	}
 
 	// Of the keyCount keys in the generative keyspace, how many are there
 	// starting with a particular character?
-	keysPerCharacter := keyCount / len(alphabet)
+	keysPerCharacter := keyCount / int64(len(alphabet))
 
 	// Find the character that the key at index i starts with and set it.
 	characterIdx := i / keysPerCharacter
@@ -347,3 +363,156 @@ func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
 	i = i - keysPerCharacter*characterIdx - 1
 	return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
 }
+
+// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey,
+// returning the index of a particular key, given the provided alphabet and max
+// length of a key.
+//
+// len(key) must be ≥ 1.
+func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
+	i, ok := alphabet[key[0]]
+	if !ok {
+		panic(fmt.Sprintf("unrecognized alphabet character %v", key[0]))
+	}
+	// How many keys exist that start with the preceding i characters? Each of
+	// the i characters themselves are a key, plus the count of all the keys
+	// with one less character for each.
+	ret := i + i*keyCount(len(alphabet), n-1)
+	if len(key) > 1 {
+		ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1)
+	}
+	return ret
+}
+
+func abs(a int64) int64 {
+	if a < 0 {
+		return -a
+	}
+	return a
+}
+
+// RandomSeparator returns a random alphabetic key k such that a < k < b,
+// pulling randomness from the provided random number generator. If dst is
+// provided and the generated key fits within dst's capacity, the returned slice
+// will use dst's memory.
+//
+// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
+// will consist of the prefix P appended with the provided suffix. A zero suffix
+// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
+// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b,
+// but the generated key will not use the provided suffix. Note that it's
+// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case
+// RandomSeparator returns nil.
+//
+// If RandomSeparator generates a new prefix, the generated prefix will have
+// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))).
+//
+// RandomSeparator panics if a or b fails to decode.
+func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
+	if Comparer.Compare(a, b) >= 0 {
+		return nil
+	}
+
+	// Determine both keys' logical prefixes and suffixes.
+	ai := Comparer.Split(a)
+	bi := Comparer.Split(b)
+	ap := a[:ai]
+	bp := b[:bi]
+	maxLength = max[int](maxLength, max[int](len(ap), len(bp)))
+	var as, bs int64
+	var err error
+	if ai != len(a) {
+		as, err = ParseSuffix(a[ai:])
+		if err != nil {
+			panic(fmt.Sprintf("failed to parse suffix of %q", a))
+		}
+	}
+	if bi != len(b) {
+		bs, err = ParseSuffix(b[bi:])
+		if err != nil {
+			panic(fmt.Sprintf("failed to parse suffix of %q", b))
+		}
+	}
+
+	apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength)
+	bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength)
+	diff := bpIdx - apIdx
+	generatedIdx := bpIdx
+	if diff > 0 {
+		var add int64 = diff + 1
+		var start int64 = apIdx
+		if as == 1 {
+			// There's no expressible key with prefix a greater than a@1. So,
+			// exclude ap.
+			start = apIdx + 1
+			add = diff
+		}
+		if bs == 0 {
+			// No key with prefix b can sort before b@0. We don't want to pick b.
+			add--
+		}
+		// We're allowing generated id to be in the range [start, start + add - 1].
+		if start > start+add-1 {
+			return nil
+		}
+		// If we can generate a key which is actually in the middle of apIdx
+		// and bpIdx use it so that we don't have to bother about timestamps.
+		generatedIdx = rng.Int63n(add) + start
+		for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
+			generatedIdx = rng.Int63n(add) + start
+		}
+	}
+
+	switch {
+	case generatedIdx == apIdx && generatedIdx == bpIdx:
+		if abs(bs-as) <= 1 {
+			// There's no expressible suffix between the two, and there's no
+			// possible separator key.
+			return nil
+		}
+		// The key b is >= key a, but has the same prefix, so b must have the
+		// smaller timestamp, unless a has timestamp of 0.
+		//
+		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
+		// any suffix we generate will be greater than it.
+		if as == 0 {
+			// bs > as
+			suffix = bs + rng.Int63n(10) + 1
+		} else {
+			// bs < as.
+			// Generate suffix in range [bs + 1, as - 1]
+			suffix = bs + 1 + rng.Int63n(as-bs-1)
+		}
+	case generatedIdx == apIdx:
+		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
+		// any suffix we generate will be greater than it.
+		if as == 0 && suffix == 0 {
+			suffix++
+		} else if as != 0 && suffix >= as {
+			suffix = rng.Int63n(as)
+		}
+	case generatedIdx == bpIdx:
+		if suffix <= bs {
+			suffix = bs + rng.Int63n(10) + 1
+		}
+	}
+	if sz := maxLength + SuffixLen(suffix); cap(dst) < sz {
+		dst = make([]byte, sz)
+	} else {
+		dst = dst[:cap(dst)]
+	}
+	var w int
+	if suffix == 0 {
+		w = WriteKey(dst, Alpha(maxLength), generatedIdx)
+	} else {
+		w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
+	}
+	return dst[:w]
+}
+
+func max[I constraints.Ordered](a, b I) I {
+	if b > a {
+		return b
+	}
+	return a
+}