Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/Makefile

@@ -16,6 +16,8 @@ all:
 	@echo "  make stressmeta"
 	@echo "  make crossversion-meta"
 	@echo "  make mod-update"
+	@echo "  make generate"
+	@echo "  make generate-test-data"
 	@echo "  make clean"
 
 override testflags :=
@@ -69,6 +71,16 @@ stress-crossversion:
 generate:
 	${GO} generate ${PKG}
 
+generate:
+
+# Note that the output of generate-test-data is not deterministic. This should
+# only be run manually as needed.
+.PHONY: generate-test-data
+generate-test-data:
+	${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
+	${GO} run -tags make_test_find_db ./tool/make_test_find_db.go
+	${GO} run -tags make_test_sstables ./tool/make_test_sstables.go
+
 mod-update:
 	${GO} get -u
 	${GO} mod tidy

vendor/github.com/cockroachdb/pebble/batch.go

@@ -265,6 +265,13 @@ type Batch struct {
 	// then it will only contain key kinds of IngestSST.
 	ingestedSSTBatch bool
 
+	// minimumFormatMajorVersion indicates the format major version required in
+	// order to commit this batch. If an operation requires a particular format
+	// major version, it ratchets the batch's minimumFormatMajorVersion. When
+	// the batch is committed, this is validated against the database's current
+	// format major version.
+	minimumFormatMajorVersion FormatMajorVersion
+
 	// Synchronous Apply uses the commit WaitGroup for both publishing the
 	// seqnum and waiting for the WAL fsync (if needed). Asynchronous
 	// ApplyNoSyncWait, which implies WriteOptions.Sync is true, uses the commit
@@ -413,6 +420,7 @@ func (b *Batch) refreshMemTableSize() {
 
 	b.countRangeDels = 0
 	b.countRangeKeys = 0
+	b.minimumFormatMajorVersion = 0
 	for r := b.Reader(); ; {
 		kind, key, value, ok := r.Next()
 		if !ok {
@@ -423,12 +431,22 @@ func (b *Batch) refreshMemTableSize() {
 			b.countRangeDels++
 		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
 			b.countRangeKeys++
+		case InternalKeyKindDeleteSized:
+			if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
+				b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
+			}
 		case InternalKeyKindIngestSST:
+			if b.minimumFormatMajorVersion < FormatFlushableIngest {
+				b.minimumFormatMajorVersion = FormatFlushableIngest
+			}
 			// This key kind doesn't contribute to the memtable size.
 			continue
 		}
 		b.memTableSize += memTableEntrySize(len(key), len(value))
 	}
+	if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys {
+		b.minimumFormatMajorVersion = FormatRangeKeys
+	}
 }
 
 // Apply the operations contained in the batch to the receiver batch.
@@ -598,6 +616,41 @@ func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
 	b.data = b.data[:pos+keyLen]
 }
 
+// AddInternalKey allows the caller to add an internal key of point key kinds to
+// a batch. Passing in an internal key of kind RangeKey* or RangeDelete will
+// result in a panic. Note that the seqnum in the internal key is effectively
+// ignored, even though the Kind is preserved. This is because the batch format
+// does not allow for a per-key seqnum to be specified, only a batch-wide one.
+//
+// Note that non-indexed keys (IngestKeyKind{LogData,IngestSST}) are not
+// supported with this method as they require specialized logic.
+func (b *Batch) AddInternalKey(key *base.InternalKey, value []byte, _ *WriteOptions) error {
+	keyLen := len(key.UserKey)
+	hasValue := false
+	switch key.Kind() {
+	case InternalKeyKindRangeDelete, InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		panic("unexpected range delete or range key kind in AddInternalKey")
+	case InternalKeyKindSingleDelete, InternalKeyKindDelete:
+		b.prepareDeferredKeyRecord(len(key.UserKey), key.Kind())
+	default:
+		b.prepareDeferredKeyValueRecord(keyLen, len(value), key.Kind())
+		hasValue = true
+	}
+	b.deferredOp.index = b.index
+	copy(b.deferredOp.Key, key.UserKey)
+	if hasValue {
+		copy(b.deferredOp.Value, value)
+	}
+	// TODO(peter): Manually inline DeferredBatchOp.Finish(). Mid-stack inlining
+	// in go1.13 will remove the need for this.
+	if b.index != nil {
+		if err := b.index.Add(b.deferredOp.offset); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Set adds an action to the batch that sets the key to map to the value.
 //
 // It is safe to modify the contents of the arguments after Set returns.
@@ -680,6 +733,72 @@ func (b *Batch) DeleteDeferred(keyLen int) *DeferredBatchOp {
 	return &b.deferredOp
 }
 
+// DeleteSized behaves identically to Delete, but takes an additional
+// argument indicating the size of the value being deleted. DeleteSized
+// should be preferred when the caller has the expectation that there exists
+// a single internal KV pair for the key (eg, the key has not been
+// overwritten recently), and the caller knows the size of its value.
+//
+// DeleteSized will record the value size within the tombstone and use it to
+// inform compaction-picking heuristics which strive to reduce space
+// amplification in the LSM. This "calling your shot" mechanic allows the
+// storage engine to more accurately estimate and reduce space amplification.
+//
+// It is safe to modify the contents of the arguments after DeleteSized
+// returns.
+func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions) error {
+	deferredOp := b.DeleteSizedDeferred(len(key), deletedValueSize)
+	copy(b.deferredOp.Key, key)
+	// TODO(peter): Manually inline DeferredBatchOp.Finish(). Check if in a
+	// later Go release this is unnecessary.
+	if b.index != nil {
+		if err := b.index.Add(deferredOp.offset); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DeleteSizedDeferred is similar to DeleteSized in that it adds a sized delete
+// operation to the batch, except it only takes in key length instead of a
+// complete key slice, letting the caller encode into the DeferredBatchOp.Key
+// slice and then call Finish() on the returned object.
+func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp {
+	if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
+		b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
+	}
+
+	// Encode the sum of the key length and the value in the value.
+	v := uint64(deletedValueSize) + uint64(keyLen)
+
+	// Encode `v` as a varint.
+	var buf [binary.MaxVarintLen64]byte
+	n := 0
+	{
+		x := v
+		for x >= 0x80 {
+			buf[n] = byte(x) | 0x80
+			x >>= 7
+			n++
+		}
+		buf[n] = byte(x)
+		n++
+	}
+
+	// NB: In batch entries and sstable entries, values are stored as
+	// varstrings. Here, the value is itself a simple varint. This results in an
+	// unnecessary double layer of encoding:
+	//     varint(n) varint(deletedValueSize)
+	// The first varint will always be 1-byte, since a varint-encoded uint64
+	// will never exceed 128 bytes. This unnecessary extra byte and wrapping is
+	// preserved to avoid special casing across the database, and in particular
+	// in sstable block decoding which is performance sensitive.
+	b.prepareDeferredKeyValueRecord(keyLen, n, InternalKeyKindDeleteSized)
+	b.deferredOp.index = b.index
+	copy(b.deferredOp.Value, buf[:n])
+	return &b.deferredOp
+}
+
 // SingleDelete adds an action to the batch that single deletes the entry for key.
 // See Writer.SingleDelete for more details on the semantics of SingleDelete.
 //
@@ -782,6 +901,9 @@ func (b *Batch) rangeKeySetDeferred(startLen, internalValueLen int) *DeferredBat
 
 func (b *Batch) incrementRangeKeysCount() {
 	b.countRangeKeys++
+	if b.minimumFormatMajorVersion < FormatRangeKeys {
+		b.minimumFormatMajorVersion = FormatRangeKeys
+	}
 	if b.index != nil {
 		b.rangeKeys = nil
 		b.rangeKeysSeqNum = 0
@@ -895,6 +1017,7 @@ func (b *Batch) ingestSST(fileNum base.FileNum) {
 	// is not reset because for the InternalKeyKindIngestSST the count is the
 	// number of sstable paths which have been added to the batch.
 	b.memTableSize = origMemTableSize
+	b.minimumFormatMajorVersion = FormatFlushableIngest
 }
 
 // Empty returns true if the batch is empty, and false otherwise.
@@ -1199,6 +1322,7 @@ func (b *Batch) Reset() {
 	b.commitStats = BatchCommitStats{}
 	b.commitErr = nil
 	b.applied.Store(false)
+	b.minimumFormatMajorVersion = 0
 	if b.data != nil {
 		if cap(b.data) > batchMaxRetainedSize {
 			// If the capacity of the buffer is larger than our maximum
@@ -1365,7 +1489,8 @@ func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, o
 	}
 	switch kind {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
-		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
+		InternalKeyKindDeleteSized:
 		*r, value, ok = batchDecodeStr(*r)
 		if !ok {
 			return 0, nil, nil, false
@@ -1500,7 +1625,8 @@ func (i *batchIter) value() []byte {
 
 	switch InternalKeyKind(data[offset]) {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
-		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
+		InternalKeyKindDeleteSized:
 		_, value, ok := batchDecodeStr(data[keyEnd:])
 		if !ok {
 			return nil
@@ -1944,7 +2070,8 @@ func (i *flushableBatchIter) value() base.LazyValue {
 	var ok bool
 	switch kind {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
-		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
+		InternalKeyKindDeleteSized:
 		keyEnd := i.offsets[i.index].keyEnd
 		_, value, ok = batchDecodeStr(i.data[keyEnd:])
 		if !ok {

vendor/github.com/cockroachdb/pebble/compaction.go

@@ -1333,10 +1333,10 @@ func (c *compaction) newInputIter(
 	// internal iterator interface). The resulting merged rangedel iterator is
 	// then included with the point levels in a single mergingIter.
 	newRangeDelIter := func(
-		f manifest.LevelFile, _ *IterOptions, bytesIterated *uint64,
+		f manifest.LevelFile, _ *IterOptions, l manifest.Level, bytesIterated *uint64,
 	) (keyspan.FragmentIterator, error) {
 		iter, rangeDelIter, err := newIters(context.Background(), f.FileMetadata,
-			nil /* iter options */, internalIterOpts{bytesIterated: &c.bytesIterated})
+			&IterOptions{level: l}, internalIterOpts{bytesIterated: &c.bytesIterated})
 		if err == nil {
 			// TODO(peter): It is mildly wasteful to open the point iterator only to
 			// immediately close it. One way to solve this would be to add new
@@ -1415,6 +1415,10 @@ func (c *compaction) newInputIter(
 	iterOpts := IterOptions{logger: c.logger}
 	// TODO(bananabrick): Get rid of the extra manifest.Level parameter and fold it into
 	// compactionLevel.
+	//
+	// TODO(bilal): when we start using strict obsolete sstables for L5 and L6
+	// in disaggregated storage, and rely on the obsolete bit, we will also need
+	// to configure the levelIter at these levels to hide the obsolete points.
 	addItersForLevel := func(level *compactionLevel, l manifest.Level) error {
 		iters = append(iters, newLevelIter(iterOpts, c.cmp, nil /* split */, newIters,
 			level.files.Iter(), l, &c.bytesIterated))
@@ -1455,8 +1459,10 @@ func (c *compaction) newInputIter(
 		// mergingIter.
 		iter := level.files.Iter()
 		for f := iter.First(); f != nil; f = iter.Next() {
-			rangeDelIter, err := newRangeDelIter(iter.Take(), nil, &c.bytesIterated)
+			rangeDelIter, err := newRangeDelIter(iter.Take(), nil, l, &c.bytesIterated)
 			if err != nil {
+				// The error will already be annotated with the BackingFileNum, so
+				// we annotate it with the FileNum.
 				return errors.Wrapf(err, "pebble: could not open table %s", errors.Safe(f.FileNum))
 			}
 			if rangeDelIter != emptyKeyspanIter {
@@ -1474,7 +1480,7 @@ func (c *compaction) newInputIter(
 		}
 		if hasRangeKeys {
 			li := &keyspan.LevelIter{}
-			newRangeKeyIterWrapper := func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
+			newRangeKeyIterWrapper := func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
 				iter, err := newRangeKeyIter(file, iterOptions)
 				if iter != nil {
 					// Ensure that the range key iter is not closed until the compaction is
@@ -1495,7 +1501,7 @@ func (c *compaction) newInputIter(
 				}
 				return iter, err
 			}
-			li.Init(keyspan.SpanIterOptions{}, c.cmp, newRangeKeyIterWrapper, level.files.Iter(), l, manifest.KeyTypeRange)
+			li.Init(keyspan.SpanIterOptions{Level: l}, c.cmp, newRangeKeyIterWrapper, level.files.Iter(), l, manifest.KeyTypeRange)
 			rangeKeyIters = append(rangeKeyIters, li)
 		}
 		return nil
@@ -2066,12 +2072,12 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) {
 	bytesFlushed = c.bytesIterated
 	d.mu.snapshots.cumulativePinnedCount += stats.cumulativePinnedKeys
 	d.mu.snapshots.cumulativePinnedSize += stats.cumulativePinnedSize
+	d.mu.versions.metrics.Keys.MissizedTombstonesCount += stats.countMissizedDels
 
 	d.maybeUpdateDeleteCompactionHints(c)
 	d.clearCompactingState(c, err != nil)
 	delete(d.mu.compact.inProgress, c)
 	d.mu.versions.incrementCompactions(c.kind, c.extraLevels)
-	d.mu.versions.incrementCompactionBytes(-c.bytesWritten)
 
 	var flushed flushableList
 	if err == nil {
@@ -2559,9 +2565,20 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) {
 	info.Duration = d.timeNow().Sub(startTime)
 	if err == nil {
 		d.mu.versions.logLock()
-		err = d.mu.versions.logAndApply(jobID, ve, c.metrics, false /* forceRotation */, func() []compactionInfo {
-			return d.getInProgressCompactionInfoLocked(c)
-		})
+		// Confirm if any of this compaction's inputs were deleted while this
+		// compaction was ongoing.
+		for i := range c.inputs {
+			c.inputs[i].files.Each(func(m *manifest.FileMetadata) {
+				if m.Deleted {
+					err = firstError(err, errors.New("pebble: file deleted by a concurrent operation, will retry compaction"))
+				}
+			})
+		}
+		if err == nil {
+			err = d.mu.versions.logAndApply(jobID, ve, c.metrics, false /* forceRotation */, func() []compactionInfo {
+				return d.getInProgressCompactionInfoLocked(c)
+			})
+		}
 		if err != nil {
 			// TODO(peter): untested.
 			for _, f := range pendingOutputs {
@@ -2617,6 +2634,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) {
 type compactStats struct {
 	cumulativePinnedKeys uint64
 	cumulativePinnedSize uint64
+	countMissizedDels    uint64
 }
 
 // runCompactions runs a compaction that produces new on-disk tables from
@@ -2850,10 +2868,12 @@ func (d *DB) runCompaction(
 			Path:    d.objProvider.Path(objMeta),
 			FileNum: fileNum,
 		})
-		writable = &compactionWritable{
-			Writable: writable,
-			versions: d.mu.versions,
-			written:  &c.bytesWritten,
+		if c.kind != compactionKindFlush {
+			writable = &compactionWritable{
+				Writable: writable,
+				versions: d.mu.versions,
+				written:  &c.bytesWritten,
+			}
 		}
 		createdFiles = append(createdFiles, fileNum.DiskFileNum())
 		cacheOpts := private.SSTableCacheOpts(d.cacheID, fileNum.DiskFileNum()).(sstable.WriterOption)
@@ -3232,7 +3252,10 @@ func (d *DB) runCompaction(
 					return nil, pendingOutputs, stats, err
 				}
 			}
-			if err := tw.Add(*key, val); err != nil {
+			// iter.snapshotPinned is broader than whether the point was covered by
+			// a RANGEDEL, but it is harmless to pass true when the callee will also
+			// independently discover that the point is obsolete.
+			if err := tw.AddWithForceObsolete(*key, val, iter.snapshotPinned); err != nil {
 				return nil, pendingOutputs, stats, err
 			}
 			if iter.snapshotPinned {
@@ -3285,6 +3308,11 @@ func (d *DB) runCompaction(
 		}
 	}
 
+	// The compaction iterator keeps track of a count of the number of DELSIZED
+	// keys that encoded an incorrect size. Propagate it up as a part of
+	// compactStats.
+	stats.countMissizedDels = iter.stats.countMissizedDels
+
 	if err := d.objProvider.Sync(); err != nil {
 		return nil, pendingOutputs, stats, err
 	}
@@ -3613,7 +3641,7 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) {
 	if len(filesToDelete) > 0 {
 		d.deleters.Add(1)
 		// Delete asynchronously if that could get held up in the pacer.
-		if d.opts.Experimental.MinDeletionRate > 0 {
+		if d.opts.TargetByteDeletionRate > 0 {
 			go d.paceAndDeleteObsoleteFiles(jobID, filesToDelete)
 		} else {
 			d.paceAndDeleteObsoleteFiles(jobID, filesToDelete)
@@ -3626,14 +3654,20 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) {
 func (d *DB) paceAndDeleteObsoleteFiles(jobID int, files []obsoleteFile) {
 	defer d.deleters.Done()
 	pacer := (pacer)(nilPacer)
-	if d.opts.Experimental.MinDeletionRate > 0 {
+	if d.opts.TargetByteDeletionRate > 0 {
 		pacer = newDeletionPacer(d.deletionLimiter, d.getDeletionPacerInfo)
 	}
 
 	for _, of := range files {
 		path := base.MakeFilepath(d.opts.FS, of.dir, of.fileType, of.fileNum)
 		if of.fileType == fileTypeTable {
-			_ = pacer.maybeThrottle(of.fileSize)
+			// Don't throttle deletion of shared objects.
+			meta, err := d.objProvider.Lookup(of.fileType, of.fileNum)
+			// If we get an error here, deleteObsoleteObject won't actually delete
+			// anything, so we don't need to throttle.
+			if err == nil && !meta.IsShared() {
+				_ = pacer.maybeThrottle(of.fileSize)
+			}
 			d.mu.Lock()
 			d.mu.versions.metrics.Table.ObsoleteCount--
 			d.mu.versions.metrics.Table.ObsoleteSize -= of.fileSize

vendor/github.com/cockroachdb/pebble/compaction_iter.go

@@ -6,6 +6,7 @@ package pebble
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"sort"
@@ -205,6 +206,21 @@ type compactionIter struct {
 	// compaction iterator was only returned because an open snapshot prevents
 	// its elision. This field only applies to point keys, and not to range
 	// deletions or range keys.
+	//
+	// snapshotPinned is also used to set the forceObsolete value in the call to
+	// Writer.AddWithForceObsolete. Note that in that call, it is sufficient to
+	// mark all keys obsoleted by RANGEDELs as forceObsolete=true and that the
+	// implementation of Writer.AddWithForceObsolete will itself discover other
+	// causes of obsolescence. We mention this since in the presence of MERGE,
+	// obsolescence due to multiple keys at the same user key is not fully
+	// represented by snapshotPinned=true:
+	//
+	// For MERGE, it is possible that doing the merge is interrupted even when
+	// the next point key is in the same stripe. This can happen if the loop in
+	// mergeNext gets interrupted by sameStripeNonSkippable.
+	// sameStripeNonSkippable occurs due to RANGEDELs that sort before
+	// SET/MERGE/DEL with the same seqnum, so the RANGEDEL does not necessarily
+	// delete the subsequent SET/MERGE/DEL keys.
 	snapshotPinned bool
 	// The index of the snapshot for the current key within the snapshots slice.
 	curSnapshotIdx    int
@@ -240,6 +256,10 @@ type compactionIter struct {
 	// The on-disk format major version. This informs the types of keys that
 	// may be written to disk during a compaction.
 	formatVersion FormatMajorVersion
+	stats         struct {
+		// count of DELSIZED keys that were missized.
+		countMissizedDels uint64
+	}
 }
 
 func newCompactionIter(
@@ -310,10 +330,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 	// respect to `iterKey` and related state:
 	//
 	// - `!skip && pos == iterPosNext`: `iterKey` is already at the next key.
-	// - `!skip && pos == iterPosCur`: We are at the key that has been returned.
+	// - `!skip && pos == iterPosCurForward`: We are at the key that has been returned.
 	//   To move forward we advance by one key, even if that lands us in the same
 	//   snapshot stripe.
-	// - `skip && pos == iterPosCur`: We are at the key that has been returned.
+	// - `skip && pos == iterPosCurForward`: We are at the key that has been returned.
 	//   To move forward we skip skippable entries in the stripe.
 	if i.pos == iterPosCurForward {
 		if i.skip {
@@ -382,11 +402,18 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 		} else if cover == keyspan.CoversInvisibly {
 			// i.iterKey would be deleted by a range deletion if there weren't
 			// any open snapshots. Mark it as pinned.
+			//
+			// NB: there are multiple places in this file where we call
+			// i.rangeDelFrag.Covers and this is the only one where we are writing
+			// to i.snapshotPinned. Those other cases occur in mergeNext where the
+			// caller is deciding whether the value should be merged or not, and the
+			// key is in the same snapshot stripe. Hence, snapshotPinned is by
+			// definition false in those cases.
 			i.snapshotPinned = true
 		}
 
 		switch i.iterKey.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			if i.elideTombstone(i.iterKey.UserKey) {
 				if i.curSnapshotIdx == 0 {
 					// If we're at the last snapshot stripe and the tombstone
@@ -410,6 +437,12 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 				i.skip = true
 				return &i.key, i.value
 
+			case InternalKeyKindDeleteSized:
+				// We may skip subsequent keys because of this tombstone. Scan
+				// ahead to see just how much data this tombstone drops and if
+				// the tombstone's value should be updated accordingly.
+				return i.deleteSizedNext()
+
 			case InternalKeyKindSingleDelete:
 				if i.singleDeleteNext() {
 					return &i.key, i.value
@@ -686,11 +719,12 @@ func (i *compactionIter) setNext() {
 			i.skip = true
 			return
 		case sameStripeSkippable:
-			// We're still in the same stripe. If this is a DEL/SINGLEDEL, we
-			// stop looking and emit a SETWITHDEL. Subsequent keys are
-			// eligible for skipping.
+			// We're still in the same stripe. If this is a
+			// DEL/SINGLEDEL/DELSIZED, we stop looking and emit a SETWITHDEL.
+			// Subsequent keys are eligible for skipping.
 			if i.iterKey.Kind() == InternalKeyKindDelete ||
-				i.iterKey.Kind() == InternalKeyKindSingleDelete {
+				i.iterKey.Kind() == InternalKeyKindSingleDelete ||
+				i.iterKey.Kind() == InternalKeyKindDeleteSized {
 				i.key.SetKind(InternalKeyKindSetWithDelete)
 				i.skip = true
 				return
@@ -715,7 +749,7 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 		}
 		key := i.iterKey
 		switch key.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			// We've hit a deletion tombstone. Return everything up to this point and
 			// then skip entries until the next snapshot stripe. We change the kind
 			// of the result key to a Set so that it shadows keys in lower
@@ -802,9 +836,9 @@ func (i *compactionIter) singleDeleteNext() bool {
 
 		key := i.iterKey
 		switch key.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
-			// We've hit a Delete, Merge or SetWithDelete, transform the
-			// SingleDelete into a full Delete.
+		case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+			// We've hit a Delete, DeleteSized, Merge, SetWithDelete, transform
+			// the SingleDelete into a full Delete.
 			i.key.SetKind(InternalKeyKindDelete)
 			i.skip = true
 			return true
@@ -825,6 +859,110 @@ func (i *compactionIter) singleDeleteNext() bool {
 	}
 }
 
+// deleteSizedNext processes a DELSIZED point tombstone. Unlike ordinary DELs,
+// these tombstones carry a value that's a varint indicating the size of the
+// entry (len(key)+len(value)) that the tombstone is expected to delete.
+//
+// When a deleteSizedNext is encountered, we skip ahead to see which keys, if
+// any, are elided as a result of the tombstone.
+func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
+	i.saveKey()
+	i.valid = true
+	i.skip = true
+
+	// The DELSIZED tombstone may have no value at all. This happens when the
+	// tombstone has already deleted the key that the user originally predicted.
+	// In this case, we still peek forward in case there's another DELSIZED key
+	// with a lower sequence number, in which case we'll adopt its value.
+	if len(i.iterValue) == 0 {
+		i.value = i.valueBuf[:0]
+	} else {
+		i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
+		i.value = i.valueBuf
+	}
+
+	// Loop through all the keys within this stripe that are skippable,
+	// totalling the size of their keys and values.
+	i.pos = iterPosNext
+	var elidedSize uint64
+	for i.nextInStripe() == sameStripeSkippable {
+		elidedSize += uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue))
+
+		if i.iterKey.Kind() == InternalKeyKindDeleteSized {
+			// We encountered a DELSIZED that's deleted by the original
+			// DELSIZED. These tombstones could've been intended to delete two
+			// distinct values, eg:
+			//
+			//     a.DELSIZED.9   a.SET.7   a.DELSIZED.5   a.SET.4
+			//
+			// If a.DELSIZED.9 has already deleted a.SET.7, its size has already
+			// been zeroed out. In this case, we want to adopt the value of the
+			// DELSIZED with the lower sequence number, in case the a.SET.4 key
+			// has not yet been elided.
+			i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
+			i.value = i.valueBuf
+			// Reset the elided total.
+			elidedSize = 0
+		}
+	}
+	// Reset skip if we landed outside the original stripe. Otherwise, we landed
+	// in the same stripe on a non-skippable key. In that case we should preserve
+	// `i.skip == true` such that later keys in the stripe will continue to be
+	// skipped.
+	if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey {
+		i.skip = false
+	}
+
+	if elidedSize == 0 {
+		// If we didn't find any keys that we could elide due to this tombstone
+		// (or at least none since the last DELSIZED encountered), we can return
+		// the key and our existing saved value verbatim.
+		return &i.key, i.value
+	}
+
+	// Some key(s) were elided as a result of this tombstone. Decode the
+	// tombstone's value to see if it matches the amount of data actually
+	// elided.
+	var v uint64
+	if len(i.value) > 0 {
+		var n int
+		v, n = binary.Uvarint(i.value)
+		if n != len(i.value) {
+			i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value))
+			i.valid = false
+			return nil, nil
+		}
+	}
+
+	// If the user's prediction of the value size was correct, then from now on
+	// it's predicted to delete nothing. Return an empty value.
+	//
+	// NB: It's important that we still always return the tombstone, because
+	// DELSIZED's values are a best effort optimization, not a correctness
+	// mechanic. There is no guarantee that this key was only set once before
+	// the DELSIZED was written.
+
+	// If the tombstone carried a larger size than what was actually elided,
+	// it's unclear what to do. The user-provided size was wrong, so it's
+	// unlikely to be accurate or meaningful. We could:
+	//
+	//   1. return the DELSIZED with the original user-provided size unmodified
+	//   2. return the DELZIZED with a zeroed size to reflect that a key was
+	//      elided, even if it wasn't the anticipated size.
+	//   3. subtract the elided size from the estimate and re-encode.
+	//   4. convert the DELSIZED into a value-less DEL, so that ordinary DEL
+	//      heuristics apply.
+	//
+	// We opt for (4) under the rationale that we can't rely on the
+	// user-provided size for accuracy, so ordinary DEL heuristics are safer.
+	i.value = i.valueBuf[:0]
+	if elidedSize != v {
+		i.stats.countMissizedDels++
+		i.key.SetKind(InternalKeyKindDelete)
+	}
+	return &i.key, i.value
+}
+
 func (i *compactionIter) saveKey() {
 	i.keyBuf = append(i.keyBuf[:0], i.iterKey.UserKey...)
 	i.key.UserKey = i.keyBuf

vendor/github.com/cockroachdb/pebble/compaction_picker.go

@@ -674,15 +674,16 @@ type candidateLevelInfo struct {
 	file manifest.LevelFile
 }
 
+func fileCompensation(f *fileMetadata) uint64 {
+	return uint64(f.Stats.PointDeletionsBytesEstimate) + f.Stats.RangeDeletionsBytesEstimate
+}
+
 // compensatedSize returns f's file size, inflated according to compaction
 // priorities.
-func compensatedSize(f *fileMetadata, pointTombstoneWeight float64) uint64 {
-	sz := f.Size
-	// Add in the estimate of disk space that may be reclaimed by compacting
-	// the file's tombstones.
-	sz += uint64(float64(f.Stats.PointDeletionsBytesEstimate) * pointTombstoneWeight)
-	sz += f.Stats.RangeDeletionsBytesEstimate
-	return sz
+func compensatedSize(f *fileMetadata) uint64 {
+	// Add in the estimate of disk space that may be reclaimed by compacting the
+	// file's tombstones.
+	return f.Size + fileCompensation(f)
 }
 
 // compensatedSizeAnnotator implements manifest.Annotator, annotating B-Tree
@@ -691,7 +692,6 @@ func compensatedSize(f *fileMetadata, pointTombstoneWeight float64) uint64 {
 // asynchronously, so its values are marked as cacheable only if a file's
 // stats have been loaded.
 type compensatedSizeAnnotator struct {
-	pointTombstoneWeight float64
 }
 
 var _ manifest.Annotator = compensatedSizeAnnotator{}
@@ -709,7 +709,7 @@ func (a compensatedSizeAnnotator) Accumulate(
 	f *fileMetadata, dst interface{},
 ) (v interface{}, cacheOK bool) {
 	vptr := dst.(*uint64)
-	*vptr = *vptr + compensatedSize(f, a.pointTombstoneWeight)
+	*vptr = *vptr + compensatedSize(f)
 	return vptr, f.StatsValid()
 }
 
@@ -724,10 +724,10 @@ func (a compensatedSizeAnnotator) Merge(src interface{}, dst interface{}) interf
 // iterator. Note that this function is linear in the files available to the
 // iterator. Use the compensatedSizeAnnotator if querying the total
 // compensated size of a level.
-func totalCompensatedSize(iter manifest.LevelIterator, pointTombstoneWeight float64) uint64 {
+func totalCompensatedSize(iter manifest.LevelIterator) uint64 {
 	var sz uint64
 	for f := iter.First(); f != nil; f = iter.Next() {
-		sz += compensatedSize(f, pointTombstoneWeight)
+		sz += compensatedSize(f)
 	}
 	return sz
 }
@@ -925,9 +925,7 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp
 	}
 }
 
-func calculateSizeAdjust(
-	inProgressCompactions []compactionInfo, pointTombstoneWeight float64,
-) [numLevels]int64 {
+func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]int64 {
 	// Compute a size adjustment for each level based on the in-progress
 	// compactions. We subtract the compensated size of start level inputs.
 	// Since compensated file sizes may be compensated because they reclaim
@@ -937,10 +935,16 @@ func calculateSizeAdjust(
 	var sizeAdjust [numLevels]int64
 	for i := range inProgressCompactions {
 		c := &inProgressCompactions[i]
+		// If this compaction's version edit has already been applied, there's
+		// no need to adjust: The LSM we'll examine will already reflect the
+		// new LSM state.
+		if c.versionEditApplied {
+			continue
+		}
 
 		for _, input := range c.inputs {
 			real := int64(input.files.SizeSum())
-			compensated := int64(totalCompensatedSize(input.files.Iter(), pointTombstoneWeight))
+			compensated := int64(totalCompensatedSize(input.files.Iter()))
 
 			if input.level != c.outputLevel {
 				sizeAdjust[input.level] -= compensated
@@ -967,10 +971,7 @@ func (p *compactionPickerByScore) calculateScores(
 	}
 	scores[0] = p.calculateL0Score(inProgressCompactions)
 
-	sizeAdjust := calculateSizeAdjust(
-		inProgressCompactions,
-		p.opts.Experimental.PointTombstoneWeight,
-	)
+	sizeAdjust := calculateSizeAdjust(inProgressCompactions)
 	for level := 1; level < numLevels; level++ {
 		levelSize := int64(levelCompensatedSize(p.vers.Levels[level])) + sizeAdjust[level]
 		scores[level].score = float64(levelSize) / float64(p.levelMaxBytes[level])
@@ -1088,14 +1089,20 @@ func (p *compactionPickerByScore) pickFile(
 
 	for f := startIter.First(); f != nil; f = startIter.Next() {
 		var overlappingBytes uint64
+		compacting := f.IsCompacting()
+		if compacting {
+			// Move on if this file is already being compacted. We'll likely
+			// still need to move past the overlapping output files regardless,
+			// but in cases where all start-level files are compacting we won't.
+			continue
+		}
 
 		// Trim any output-level files smaller than f.
-		for outputFile != nil && base.InternalCompare(cmp, outputFile.Largest, f.Smallest) < 0 {
+		for outputFile != nil && sstableKeyCompare(cmp, outputFile.Largest, f.Smallest) < 0 {
 			outputFile = outputIter.Next()
 		}
 
-		compacting := f.IsCompacting()
-		for outputFile != nil && base.InternalCompare(cmp, outputFile.Smallest, f.Largest) < 0 {
+		for outputFile != nil && sstableKeyCompare(cmp, outputFile.Smallest, f.Largest) <= 0 && !compacting {
 			overlappingBytes += outputFile.Size
 			compacting = compacting || outputFile.IsCompacting()
 
@@ -1114,7 +1121,32 @@ func (p *compactionPickerByScore) pickFile(
 			// If the file in the next level extends beyond f's largest key,
 			// break out and don't advance outputIter because f's successor
 			// might also overlap.
-			if base.InternalCompare(cmp, outputFile.Largest, f.Largest) > 0 {
+			//
+			// Note, we stop as soon as we encounter an output-level file with a
+			// largest key beyond the input-level file's largest bound. We
+			// perform a simple user key comparison here using sstableKeyCompare
+			// which handles the potential for exclusive largest key bounds.
+			// There's some subtlety when the bounds are equal (eg, equal and
+			// inclusive, or equal and exclusive). Current Pebble doesn't split
+			// user keys across sstables within a level (and in format versions
+			// FormatSplitUserKeysMarkedCompacted and later we guarantee no
+			// split user keys exist within the entire LSM). In that case, we're
+			// assured that neither the input level nor the output level's next
+			// file shares the same user key, so compaction expansion will not
+			// include them in any compaction compacting `f`.
+			//
+			// NB: If we /did/ allow split user keys, or we're running on an
+			// old database with an earlier format major version where there are
+			// existing split user keys, this logic would be incorrect. Consider
+			//    L1: [a#120,a#100] [a#80,a#60]
+			//    L2: [a#55,a#45] [a#35,a#25] [a#15,a#5]
+			// While considering the first file in L1, [a#120,a#100], we'd skip
+			// past all of the files in L2. When considering the second file in
+			// L1, we'd improperly conclude that the second file overlaps
+			// nothing in the second level and is cheap to compact, when in
+			// reality we'd need to expand the compaction to include all 5
+			// files.
+			if sstableKeyCompare(cmp, outputFile.Largest, f.Largest) > 0 {
 				break
 			}
 			outputFile = outputIter.Next()
@@ -1127,9 +1159,9 @@ func (p *compactionPickerByScore) pickFile(
 			continue
 		}
 
-		compSz := compensatedSize(f, p.opts.Experimental.PointTombstoneWeight)
+		compSz := compensatedSize(f)
 		scaledRatio := overlappingBytes * 1024 / compSz
-		if scaledRatio < smallestRatio && !f.IsCompacting() {
+		if scaledRatio < smallestRatio {
 			smallestRatio = scaledRatio
 			file = startIter.Take()
 		}
@@ -1191,7 +1223,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
 			fmt.Fprintf(&buf, "  %sL%d: %5.1f  %5.1f  %8s  %8s",
 				marker, info.level, info.score, info.origScore,
 				humanize.Int64(int64(totalCompensatedSize(
-					p.vers.Levels[info.level].Iter(), p.opts.Experimental.PointTombstoneWeight,
+					p.vers.Levels[info.level].Iter(),
 				))),
 				humanize.Int64(p.levelMaxBytes[info.level]),
 			)

vendor/github.com/cockroachdb/pebble/db.go

@@ -96,6 +96,22 @@ type Writer interface {
 	// It is safe to modify the contents of the arguments after Delete returns.
 	Delete(key []byte, o *WriteOptions) error
 
+	// DeleteSized behaves identically to Delete, but takes an additional
+	// argument indicating the size of the value being deleted. DeleteSized
+	// should be preferred when the caller has the expectation that there exists
+	// a single internal KV pair for the key (eg, the key has not been
+	// overwritten recently), and the caller knows the size of its value.
+	//
+	// DeleteSized will record the value size within the tombstone and use it to
+	// inform compaction-picking heuristics which strive to reduce space
+	// amplification in the LSM. This "calling your shot" mechanic allows the
+	// storage engine to more accurately estimate and reduce space
+	// amplification.
+	//
+	// It is safe to modify the contents of the arguments after DeleteSized
+	// returns.
+	DeleteSized(key []byte, valueSize uint32, _ *WriteOptions) error
+
 	// SingleDelete is similar to Delete in that it deletes the value for the given key. Like Delete,
 	// it is a blind operation that will succeed even if the given key does not exist.
 	//
@@ -620,6 +636,30 @@ func (d *DB) Delete(key []byte, opts *WriteOptions) error {
 	return nil
 }
 
+// DeleteSized behaves identically to Delete, but takes an additional
+// argument indicating the size of the value being deleted. DeleteSized
+// should be preferred when the caller has the expectation that there exists
+// a single internal KV pair for the key (eg, the key has not been
+// overwritten recently), and the caller knows the size of its value.
+//
+// DeleteSized will record the value size within the tombstone and use it to
+// inform compaction-picking heuristics which strive to reduce space
+// amplification in the LSM. This "calling your shot" mechanic allows the
+// storage engine to more accurately estimate and reduce space amplification.
+//
+// It is safe to modify the contents of the arguments after DeleteSized
+// returns.
+func (d *DB) DeleteSized(key []byte, valueSize uint32, opts *WriteOptions) error {
+	b := newBatch(d)
+	_ = b.DeleteSized(key, valueSize, opts)
+	if err := d.Apply(b, opts); err != nil {
+		return err
+	}
+	// Only release the batch on success.
+	b.release()
+	return nil
+}
+
 // SingleDelete adds an action to the batch that single deletes the entry for key.
 // See Writer.SingleDelete for more details on the semantics of SingleDelete.
 //
@@ -786,17 +826,19 @@ func (d *DB) applyInternal(batch *Batch, opts *WriteOptions, noSyncWait bool) er
 		return errors.New("pebble: WAL disabled")
 	}
 
+	if batch.minimumFormatMajorVersion != FormatMostCompatible {
+		if fmv := d.FormatMajorVersion(); fmv < batch.minimumFormatMajorVersion {
+			panic(fmt.Sprintf(
+				"pebble: batch requires at least format major version %d (current: %d)",
+				batch.minimumFormatMajorVersion, fmv,
+			))
+		}
+	}
+
 	if batch.countRangeKeys > 0 {
 		if d.split == nil {
 			return errNoSplit
 		}
-		if d.FormatMajorVersion() < FormatRangeKeys {
-			panic(fmt.Sprintf(
-				"pebble: range keys require at least format major version %d (current: %d)",
-				FormatRangeKeys, d.FormatMajorVersion(),
-			))
-		}
-
 		// TODO(jackson): Assert that all range key operands are suffixless.
 	}
 
@@ -1320,6 +1362,7 @@ func (i *Iterator) constructPointIter(
 	levelsIndex := len(levels)
 	mlevels = mlevels[:numMergingLevels]
 	levels = levels[:numLevelIters]
+	i.opts.snapshotForHideObsoletePoints = buf.dbi.seqNum
 	addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) {
 		li := &levels[levelsIndex]
 
@@ -1523,6 +1566,16 @@ func (d *DB) Close() error {
 	d.mu.Unlock()
 	d.deleters.Wait()
 	d.compactionSchedulers.Wait()
+
+	// Sanity check metrics.
+	if invariants.Enabled {
+		m := d.Metrics()
+		if m.Compact.NumInProgress > 0 || m.Compact.InProgressBytes > 0 {
+			d.mu.Lock()
+			panic(fmt.Sprintf("invalid metrics on close:\n%s", m))
+		}
+	}
+
 	d.mu.Lock()
 
 	// As a sanity check, ensure that there are no zombie tables. A non-zero count
@@ -1823,6 +1876,10 @@ func (d *DB) Metrics() *Metrics {
 type sstablesOptions struct {
 	// set to true will return the sstable properties in TableInfo
 	withProperties bool
+
+	// if set, return sstables that overlap the key range (end-exclusive)
+	start []byte
+	end   []byte
 }
 
 // SSTablesOption set optional parameter used by `DB.SSTables`.
@@ -1838,6 +1895,15 @@ func WithProperties() SSTablesOption {
 	}
 }
 
+// WithKeyRangeFilter ensures returned sstables overlap start and end (end-exclusive)
+// if start and end are both nil these properties have no effect
+func WithKeyRangeFilter(start, end []byte) SSTablesOption {
+	return func(opt *sstablesOptions) {
+		opt.end = end
+		opt.start = start
+	}
+}
+
 // SSTableInfo export manifest.TableInfo with sstable.Properties
 type SSTableInfo struct {
 	manifest.TableInfo
@@ -1883,6 +1949,9 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) {
 		iter := srcLevels[i].Iter()
 		j := 0
 		for m := iter.First(); m != nil; m = iter.Next() {
+			if opt.start != nil && opt.end != nil && !m.Overlaps(d.opts.Comparer.Compare, opt.start, opt.end, true /* exclusive end */) {
+				continue
+			}
 			destTables[j] = SSTableInfo{TableInfo: m.TableInfo()}
 			if opt.withProperties {
 				p, err := d.tableCache.getTableProperties(
@@ -1900,6 +1969,7 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) {
 		destLevels[i] = destTables[:j]
 		destTables = destTables[j:]
 	}
+
 	return destLevels, nil
 }
 

vendor/github.com/cockroachdb/pebble/external_iterator.go

@@ -209,15 +209,15 @@ func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterato
 				pointIter    internalIterator
 				err          error
 			)
-			pointIter, err = r.NewIterWithBlockPropertyFiltersAndContext(
-				ctx,
-				it.opts.LowerBound,
-				it.opts.UpperBound,
-				nil,   /* BlockPropertiesFilterer */
-				false, /* useFilterBlock */
-				&it.stats.InternalStats,
-				sstable.TrivialReaderProvider{Reader: r},
-			)
+			// We could set hideObsoletePoints=true, since we are reading at
+			// InternalKeySeqNumMax, but we don't bother since these sstables should
+			// not have obsolete points (so the performance optimization is
+			// unnecessary), and we don't want to bother constructing a
+			// BlockPropertiesFilterer that includes obsoleteKeyBlockPropertyFilter.
+			pointIter, err = r.NewIterWithBlockPropertyFiltersAndContextEtc(
+				ctx, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */
+				false /* hideObsoletePoints */, false, /* useFilterBlock */
+				&it.stats.InternalStats, sstable.TrivialReaderProvider{Reader: r})
 			if err != nil {
 				return nil, err
 			}

vendor/github.com/cockroachdb/pebble/flushable.go

@@ -184,7 +184,7 @@ func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) i
 }
 
 func (s *ingestedFlushable) constructRangeDelIter(
-	file *manifest.FileMetadata, _ *keyspan.SpanIterOptions,
+	file *manifest.FileMetadata, _ keyspan.SpanIterOptions,
 ) (keyspan.FragmentIterator, error) {
 	// Note that the keyspan level iter expects a non-nil iterator to be
 	// returned even if there is an error. So, we return the emptyKeyspanIter.

vendor/github.com/cockroachdb/pebble/format_major_version.go

@@ -151,8 +151,29 @@ const (
 	// compactions for files marked for compaction are complete.
 	FormatPrePebblev1MarkedCompacted
 
+	// ExperimentalFormatDeleteSizedAndObsolete is a format major version that adds support
+	// for deletion tombstones that encode the size of the value they're
+	// expected to delete. This format major version is required before the
+	// associated key kind may be committed through batch applications or
+	// ingests. It also adds support for keys that are marked obsolete (see
+	// sstable/format.go for details).
+	ExperimentalFormatDeleteSizedAndObsolete
+
+	// ExperimentalFormatVirtualSSTables is a format major version that adds support for
+	// virtual sstables that can reference a sub-range of keys in an underlying
+	// physical sstable. This information is persisted through new,
+	// backward-incompatible fields in the Manifest, and therefore requires
+	// a format major version.
+	ExperimentalFormatVirtualSSTables
+
+	// internalFormatNewest holds the newest format major version, including
+	// experimental ones excluded from the exported FormatNewest constant until
+	// they've stabilized. Used in tests.
+	internalFormatNewest FormatMajorVersion = iota - 1
+
 	// FormatNewest always contains the most recent format major version.
-	FormatNewest FormatMajorVersion = iota - 1
+	// TODO(jackson): restore FormatNewest to internalFormatNewest.
+	FormatNewest FormatMajorVersion = FormatPrePebblev1MarkedCompacted
 )
 
 // MaxTableFormat returns the maximum sstable.TableFormat that can be used at
@@ -170,6 +191,8 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 		return sstable.TableFormatPebblev2
 	case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev3
+	case ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
+		return sstable.TableFormatPebblev4
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
 	}
@@ -186,7 +209,8 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
 		return sstable.TableFormatLevelDB
 	case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
 		FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
-		FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
+		FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
+		ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
 		return sstable.TableFormatPebblev1
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -319,6 +343,12 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		}
 		return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
 	},
+	ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete)
+	},
+	ExperimentalFormatVirtualSSTables: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(ExperimentalFormatVirtualSSTables)
+	},
 }
 
 const formatVersionMarkerName = `format-version`
@@ -341,7 +371,7 @@ func lookupFormatMajorVersion(
 	if vers == FormatDefault {
 		return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers)
 	}
-	if vers > FormatNewest {
+	if vers > internalFormatNewest {
 		return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers)
 	}
 	return vers, m, nil
@@ -377,8 +407,8 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
 	if d.opts.ReadOnly {
 		return ErrReadOnly
 	}
-	if formatVers > FormatNewest {
-		// Guard against accidentally forgetting to update FormatNewest.
+	if formatVers > internalFormatNewest {
+		// Guard against accidentally forgetting to update internalFormatNewest.
 		return errors.Errorf("pebble: unknown format version %d", formatVers)
 	}
 	if d.mu.formatVers.vers > formatVers {

vendor/github.com/cockroachdb/pebble/get_iter.go

@@ -158,7 +158,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 			if n := len(g.l0); n > 0 {
 				files := g.l0[n-1].Iter()
 				g.l0 = g.l0[:n-1]
-				iterOpts := IterOptions{logger: g.logger}
+				iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
 				g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
 					files, manifest.L0Sublevel(n), internalIterOpts{})
 				g.levelIter.initRangeDel(&g.rangeDelIter)
@@ -177,7 +177,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 			continue
 		}
 
-		iterOpts := IterOptions{logger: g.logger}
+		iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
 		g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
 			g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
 		g.levelIter.initRangeDel(&g.rangeDelIter)

vendor/github.com/cockroachdb/pebble/internal.go

@@ -23,6 +23,7 @@ const (
 	InternalKeyKindRangeKeyUnset   = base.InternalKeyKindRangeKeyUnset
 	InternalKeyKindRangeKeyDelete  = base.InternalKeyKindRangeKeyDelete
 	InternalKeyKindIngestSST       = base.InternalKeyKindIngestSST
+	InternalKeyKindDeleteSized     = base.InternalKeyKindDeleteSized
 	InternalKeyKindInvalid         = base.InternalKeyKindInvalid
 	InternalKeySeqNumBatch         = base.InternalKeySeqNumBatch
 	InternalKeySeqNumMax           = base.InternalKeySeqNumMax

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -24,6 +24,14 @@ const (
 	//InternalKeyKindColumnFamilyDeletion     InternalKeyKind = 4
 	//InternalKeyKindColumnFamilyValue        InternalKeyKind = 5
 	//InternalKeyKindColumnFamilyMerge        InternalKeyKind = 6
+
+	// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
+	// solely for compactions (to reduce write amp and space amp). Readers other
+	// than compactions should treat SINGLEDEL as equivalent to a DEL.
+	// Historically, it was simpler for readers other than compactions to treat
+	// SINGLEDEL as equivalent to DEL, but as of the introduction of
+	// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
+	// correctness.
 	InternalKeyKindSingleDelete InternalKeyKind = 7
 	//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
 	//InternalKeyKindBeginPrepareXID          InternalKeyKind = 9
@@ -66,8 +74,15 @@ const (
 	// batch, or in an sstable.
 	InternalKeyKindIngestSST InternalKeyKind = 22
 
-	// This maximum value isn't part of the file format. It's unlikely,
-	// but future extensions may increase this value.
+	// InternalKeyKindDeleteSized keys behave identically to
+	// InternalKeyKindDelete keys, except that they hold an associated uint64
+	// value indicating the (len(key)+len(value)) of the shadowed entry the
+	// tombstone is expected to delete. This value is used to inform compaction
+	// heuristics, but is not required to be accurate for correctness.
+	InternalKeyKindDeleteSized InternalKeyKind = 23
+
+	// This maximum value isn't part of the file format. Future extensions may
+	// increase this value.
 	//
 	// When constructing an internal key to pass to DB.Seek{GE,LE},
 	// internalKeyComparer sorts decreasing by kind (after sorting increasing by
@@ -75,14 +90,19 @@ const (
 	// which sorts 'less than or equal to' any other valid internalKeyKind, when
 	// searching for any kind of internal key formed by a certain user key and
 	// seqNum.
-	InternalKeyKindMax InternalKeyKind = 22
+	InternalKeyKindMax InternalKeyKind = 23
+
+	// Internal to the sstable format. Not exposed by any sstable iterator.
+	// Declared here to prevent definition of valid key kinds that set this bit.
+	InternalKeyKindSSTableInternalObsoleteBit  InternalKeyKind = 64
+	InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
 
 	// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
 	// zero sequence number.
-	InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid)
+	InternalKeyZeroSeqnumMaxTrailer = uint64(255)
 
 	// A marker for an invalid key.
-	InternalKeyKindInvalid InternalKeyKind = 255
+	InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
 
 	// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
 	// which prevents those entries from being excluded from iteration.
@@ -105,6 +125,9 @@ const (
 	InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
 )
 
+// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
+const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
+
 var internalKeyKindNames = []string{
 	InternalKeyKindDelete:         "DEL",
 	InternalKeyKindSet:            "SET",
@@ -118,6 +141,7 @@ var internalKeyKindNames = []string{
 	InternalKeyKindRangeKeyUnset:  "RANGEKEYUNSET",
 	InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
 	InternalKeyKindIngestSST:      "INGESTSST",
+	InternalKeyKindDeleteSized:    "DELSIZED",
 	InternalKeyKindInvalid:        "INVALID",
 }
 
@@ -204,6 +228,7 @@ var kindsMap = map[string]InternalKeyKind{
 	"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
 	"RANGEKEYDEL":   InternalKeyKindRangeKeyDelete,
 	"INGESTSST":     InternalKeyKindIngestSST,
+	"DELSIZED":      InternalKeyKindDeleteSized,
 }
 
 // ParseInternalKey parses the string representation of an internal key. The

vendor/github.com/cockroachdb/pebble/internal/base/metrics.go

@@ -53,6 +53,15 @@ func (tm *ThroughputMetric) Rate() int64 {
 		float64(time.Second))
 }
 
+// Utilization returns a percent [0, 1.0] indicating the percent of time
+// work was performed.
+func (tm *ThroughputMetric) Utilization() float64 {
+	if tm.WorkDuration == 0 {
+		return 0
+	}
+	return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
+}
+
 // GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
 // accumulating samples of that gauge.
 type GaugeSampleMetric struct {

vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go

@@ -4,6 +4,8 @@
 
 package base
 
+import "fmt"
+
 // This file defines sequence numbers that are reserved for foreign keys i.e.
 // internal keys coming from other Pebble instances and existing in shared
 // storage, as those will "slot below" any internal keys added by our own Pebble
@@ -14,41 +16,41 @@ const (
 	// guarantee there are no keys underneath an internal key.
 	SeqNumZero = uint64(0)
 
-	// SeqNumL6Point is the sequence number reserved for foreign point keys in L6.
-	// This sequence number must be lower than SeqNumL6RangeKey for range key
-	// masking to work correctly.
-	SeqNumL6Point = uint64(1)
+	// SeqNumL6 is the sequence number reserved for foreign keys in L6. This seqnum
+	// will be used to expose any range key sets as well as point keys in L6. Range
+	// deletes do not need to be exposed in L6.
+	SeqNumL6 = uint64(1)
 
-	// SeqNumL6RangeKey is the sequence number reserved for foreign range keys in
-	// L6. Only RangeKeySets are expected at this level.
-	SeqNumL6RangeKey = uint64(2)
+	// SeqNumL5 is the sequence number reserved for foreign keys in L5. This seqnum
+	// needs to be greater than SeqNumL6, so that range deletes in L5 can be
+	// exposed at this level and will correctly delete covering points in L6. Also
+	// range key unsets/dels will be exposed at this seqnum and will need to shadow
+	// overlapping range keys in L6.
+	//
+	// Note that we can use the same sequence number for all exposed keys in L5 as
+	// range dels do not delete points at the same seqnum, and range key
+	// unsets/deletes do not coalesce with range key sets at the same seqnum. Range
+	// key masking does not care about the sequence number of overlapping points
+	// (rather, it applies to points based on suffixes), so we can use this seqnum
+	// for all L5 keys exposed.
+	SeqNumL5 = uint64(2)
 
-	// SeqNumL5RangeDel is the sequence number reserved for foreign range deletes
-	// in L5. These keys could delete L6 points.
-	SeqNumL5RangeDel = uint64(3)
-
-	// SeqNumL5Point is the sequence number reserved for foreign point keys in L5.
-	// Any sst-local range deletions would have already been applied to these keys,
-	// so they can safely get a sequence number higher than SeqNumL5RangeDel.
-	// However they must have a sequence number lower than SeqNumL5RangeKey* for
-	// range key masking to work correctly.
-	SeqNumL5Point = uint64(4)
-
-	// SeqNumL5RangeKeyUnsetDel is the sequence number reserved for foreign
-	// range key unsets/deletes in L5. These operations could apply to L6
-	// RangeKeySets, so this sequence number must be > SeqNumL6RangeKey.
-	SeqNumL5RangeKeyUnsetDel = uint64(5)
-
-	// SeqNumL5RangeKeySet is the sequence number reserved for foreign range key
-	// Sets in L5. These operations could apply to L6 RangeKeySets, so this
-	// sequence number must be > SeqNumL6RangeKey. Any SST-local rangekey
-	// unsets/dels have already been applied to them, so their sequence number must
-	// be > SeqNumL5RangeKeyUnsetDel.
-	SeqNumL5RangeKeySet = uint64(6)
-
-	// Sequence numbers 7-9 are reserved for future use.
+	// Sequence numbers 3-9 are reserved for future use.
 
 	// SeqNumStart is the first sequence number assigned to a key written by
 	// ourselves.
 	SeqNumStart = uint64(10)
 )
+
+// SeqNumForLevel returns the appropriate reserved sequence number for keys in
+// foreign sstables at the specified level.
+func SeqNumForLevel(level int) uint64 {
+	switch level {
+	case 5:
+		return SeqNumL5
+	case 6:
+		return SeqNumL6
+	default:
+		panic(fmt.Sprintf("unexpected foreign sstable at level %d", level))
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -62,7 +62,7 @@ type FragmentIterator interface {
 
 // TableNewSpanIter creates a new iterator for range key spans for the given
 // file.
-type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
+type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
 
 // SpanIterOptions is a subset of IterOptions that are necessary to instantiate
 // per-sstable span iterators.
@@ -70,6 +70,9 @@ type SpanIterOptions struct {
 	// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
 	// when iterating over range keys.
 	RangeKeyFilters []base.BlockPropertyFilter
+	// Level specifies the level where this sstable is being read. Must be
+	// specified for foreign (i.e. shared not-created-by-this-instance) sstables.
+	Level manifest.Level
 }
 
 // Iter is an iterator over a set of fragmented spans.

vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go

@@ -98,7 +98,7 @@ func (l *LevelIter) Init(
 ) {
 	l.err = nil
 	l.level = level
-	l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
+	l.tableOpts = opts
 	l.cmp = cmp
 	l.iterFile = nil
 	l.newIter = newIter
@@ -179,7 +179,7 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur
 		return noFileLoaded
 	}
 	if indicator != fileAlreadyLoaded {
-		l.iter, l.err = l.newIter(file, &l.tableOpts)
+		l.iter, l.err = l.newIter(file, l.tableOpts)
 		indicator = newFileLoaded
 	}
 	if l.err != nil {

vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go

@@ -25,49 +25,6 @@ import (
 // MergingIter implementation, but will require a bit of plumbing to thread the
 // Equal function.
 
-// Transformer defines a transformation to be applied to a Span.
-type Transformer interface {
-	// Transform takes a Span as input and writes the transformed Span to the
-	// provided output *Span pointer. The output Span's Keys slice may be reused
-	// by Transform to reduce allocations.
-	Transform(cmp base.Compare, in Span, out *Span) error
-}
-
-// The TransformerFunc type is an adapter to allow the use of ordinary functions
-// as Transformers. If f is a function with the appropriate signature,
-// TransformerFunc(f) is a Transformer that calls f.
-type TransformerFunc func(base.Compare, Span, *Span) error
-
-// Transform calls f(cmp, in, out).
-func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
-	return tf(cmp, in, out)
-}
-
-var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
-	dst.Start, dst.End = s.Start, s.End
-	dst.Keys = append(dst.Keys[:0], s.Keys...)
-	return nil
-})
-
-// VisibleTransform filters keys that are invisible at the provided snapshot
-// sequence number.
-func VisibleTransform(snapshot uint64) Transformer {
-	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
-		dst.Start, dst.End = s.Start, s.End
-		dst.Keys = dst.Keys[:0]
-		for _, k := range s.Keys {
-			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
-			// because a batch's visible span keys are filtered when they're
-			// fragmented. There's no requirement to enforce visibility at
-			// iteration time.
-			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
-				dst.Keys = append(dst.Keys, k)
-			}
-		}
-		return nil
-	})
-}
-
 // MergingIter merges spans across levels of the LSM, exposing an iterator over
 // spans that yields sets of spans fragmented at unique user key boundaries.
 //

vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go

@@ -368,6 +368,7 @@ func (s Span) String() string {
 
 // Pretty returns a formatter for the span.
 func (s Span) Pretty(f base.FormatKey) fmt.Formatter {
+	// TODO(jackson): Take a base.FormatValue to format Key.Value too.
 	return prettySpan{s, f}
 }
 

vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go

@@ -0,0 +1,149 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Transformer defines a transformation to be applied to a Span.
+type Transformer interface {
+	// Transform takes a Span as input and writes the transformed Span to the
+	// provided output *Span pointer. The output Span's Keys slice may be reused
+	// by Transform to reduce allocations.
+	Transform(cmp base.Compare, in Span, out *Span) error
+}
+
+// The TransformerFunc type is an adapter to allow the use of ordinary functions
+// as Transformers. If f is a function with the appropriate signature,
+// TransformerFunc(f) is a Transformer that calls f.
+type TransformerFunc func(base.Compare, Span, *Span) error
+
+// Transform calls f(cmp, in, out).
+func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
+	return tf(cmp, in, out)
+}
+
+var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+	dst.Start, dst.End = s.Start, s.End
+	dst.Keys = append(dst.Keys[:0], s.Keys...)
+	return nil
+})
+
+// VisibleTransform filters keys that are invisible at the provided snapshot
+// sequence number.
+func VisibleTransform(snapshot uint64) Transformer {
+	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+		dst.Start, dst.End = s.Start, s.End
+		dst.Keys = dst.Keys[:0]
+		for _, k := range s.Keys {
+			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
+			// because a batch's visible span keys are filtered when they're
+			// fragmented. There's no requirement to enforce visibility at
+			// iteration time.
+			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
+				dst.Keys = append(dst.Keys, k)
+			}
+		}
+		return nil
+	})
+}
+
+// TransformerIter is a FragmentIterator that applies a Transformer on all
+// returned keys. Used for when a caller needs to apply a transformer on an
+// iterator but does not otherwise need the mergingiter's merging ability.
+type TransformerIter struct {
+	FragmentIterator
+
+	// Transformer is applied on every Span returned by this iterator.
+	Transformer Transformer
+	// Comparer in use for this keyspace.
+	Compare base.Compare
+
+	span Span
+	err  error
+}
+
+func (t *TransformerIter) applyTransform(span *Span) *Span {
+	t.span = Span{
+		Start: t.span.Start[:0],
+		End:   t.span.End[:0],
+		Keys:  t.span.Keys[:0],
+	}
+	if err := t.Transformer.Transform(t.Compare, *span, &t.span); err != nil {
+		t.err = err
+		return nil
+	}
+	return &t.span
+}
+
+// SeekGE implements the FragmentIterator interface.
+func (t *TransformerIter) SeekGE(key []byte) *Span {
+	span := t.FragmentIterator.SeekGE(key)
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// SeekLT implements the FragmentIterator interface.
+func (t *TransformerIter) SeekLT(key []byte) *Span {
+	span := t.FragmentIterator.SeekLT(key)
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// First implements the FragmentIterator interface.
+func (t *TransformerIter) First() *Span {
+	span := t.FragmentIterator.First()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Last implements the FragmentIterator interface.
+func (t *TransformerIter) Last() *Span {
+	span := t.FragmentIterator.Last()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Next implements the FragmentIterator interface.
+func (t *TransformerIter) Next() *Span {
+	span := t.FragmentIterator.Next()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Prev implements the FragmentIterator interface.
+func (t *TransformerIter) Prev() *Span {
+	span := t.FragmentIterator.Prev()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Error implements the FragmentIterator interface.
+func (t *TransformerIter) Error() error {
+	if t.err != nil {
+		return t.err
+	}
+	return t.FragmentIterator.Error()
+}
+
+// Close implements the FragmentIterator interface.
+func (t *TransformerIter) Close() error {
+	err := t.FragmentIterator.Close()
+	if err != nil {
+		return err
+	}
+	return t.err
+}

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -227,6 +227,12 @@ type FileMetadata struct {
 	// we'd have to write virtual sstable stats to the version edit.
 	Stats TableStats
 
+	// Deleted is set to true if a VersionEdit gets installed that has deleted
+	// this file. Protected by the manifest lock (see versionSet.logLock()).
+	Deleted bool
+
+	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
+	// pick L0 compactions. Only accurate for the most recent Version.
 	SubLevel         int
 	L0Index          int
 	minIntervalIndex int
@@ -235,9 +241,6 @@ type FileMetadata struct {
 	// NB: the alignment of this struct is 8 bytes. We pack all the bools to
 	// ensure an optimal packing.
 
-	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
-	// pick L0 compactions. Only accurate for the most recent Version.
-	//
 	// IsIntraL0Compacting is set to True if this file is part of an intra-L0
 	// compaction. When it's true, IsCompacting must also return true. If
 	// Compacting is true and IsIntraL0Compacting is false for an L0 file, the
@@ -370,6 +373,17 @@ func (m *FileMetadata) InitPhysicalBacking() {
 	}
 }
 
+// InitProviderBacking creates a new FileBacking for a file backed by
+// an objstorage.Provider.
+func (m *FileMetadata) InitProviderBacking(fileNum base.DiskFileNum) {
+	if !m.Virtual {
+		panic("pebble: provider-backed sstables must be virtual")
+	}
+	if m.FileBacking == nil {
+		m.FileBacking = &FileBacking{DiskFileNum: fileNum}
+	}
+}
+
 // ValidateVirtual should be called once the FileMetadata for a virtual sstable
 // is created to verify that the fields of the virtual sstable are sound.
 func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -51,7 +51,9 @@ const (
 	tagMaxColumnFamily  = 203
 
 	// Pebble tags.
-	tagNewFile5 = 104 // Range keys.
+	tagNewFile5            = 104 // Range keys.
+	tagCreatedBackingTable = 105
+	tagRemovedBackingTable = 106
 
 	// The custom tags sub-format used by tagNewFile4 and above.
 	customTagTerminate         = 1
@@ -59,6 +61,7 @@ const (
 	customTagCreationTime      = 6
 	customTagPathID            = 65
 	customTagNonSafeIgnoreMask = 1 << 6
+	customTagVirtual           = 66
 )
 
 // DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -73,6 +76,9 @@ type DeletedFileEntry struct {
 type NewFileEntry struct {
 	Level int
 	Meta  *FileMetadata
+	// BackingFileNum is only set during manifest replay, and only for virtual
+	// sstables.
+	BackingFileNum base.DiskFileNum
 }
 
 // VersionEdit holds the state for an edit to a Version along with other
@@ -143,7 +149,9 @@ type VersionEdit struct {
 
 // Decode decodes an edit from the specified reader.
 //
-// TODO(bananabrick): Support decoding of virtual sstable state.
+// Note that the Decode step will not set the FileBacking for virtual sstables
+// and the responsibility is left to the caller. However, the Decode step will
+// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles.
 func (v *VersionEdit) Decode(r io.Reader) error {
 	br, ok := r.(byteReader)
 	if !ok {
@@ -196,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			// NB: RocksDB does not use compaction pointers anymore.
 
+		case tagRemovedBackingTable:
+			n, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			v.RemovedBackingTables = append(
+				v.RemovedBackingTables, base.FileNum(n).DiskFileNum(),
+			)
+		case tagCreatedBackingTable:
+			dfn, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			size, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			fileBacking := &FileBacking{
+				DiskFileNum: base.FileNum(dfn).DiskFileNum(),
+				Size:        size,
+			}
+			v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking)
 		case tagDeletedFile:
 			level, err := d.readLevel()
 			if err != nil {
@@ -301,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			var markedForCompaction bool
 			var creationTime uint64
+			virtualState := struct {
+				virtual        bool
+				backingFileNum uint64
+			}{}
 			if tag == tagNewFile4 || tag == tagNewFile5 {
 				for {
 					customTag, err := d.readUvarint()
@@ -309,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 					}
 					if customTag == customTagTerminate {
 						break
+					} else if customTag == customTagVirtual {
+						virtualState.virtual = true
+						n, err := d.readUvarint()
+						if err != nil {
+							return err
+						}
+						virtualState.backingFileNum = n
+						continue
 					}
+
 					field, err := d.readBytes()
 					if err != nil {
 						return err
@@ -345,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				SmallestSeqNum:      smallestSeqNum,
 				LargestSeqNum:       largestSeqNum,
 				MarkedForCompaction: markedForCompaction,
+				Virtual:             virtualState.virtual,
 			}
 			if tag != tagNewFile5 { // no range keys present
 				m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
@@ -376,11 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				}
 			}
 			m.boundsSet = true
-			m.InitPhysicalBacking()
-			v.NewFiles = append(v.NewFiles, NewFileEntry{
+			if !virtualState.virtual {
+				m.InitPhysicalBacking()
+			}
+
+			nfe := NewFileEntry{
 				Level: level,
 				Meta:  m,
-			})
+			}
+			if virtualState.virtual {
+				nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum()
+			}
+			v.NewFiles = append(v.NewFiles, nfe)
 
 		case tagPrevLogNumber:
 			n, err := d.readUvarint()
@@ -442,8 +493,6 @@ func (v *VersionEdit) String() string {
 }
 
 // Encode encodes an edit to the specified writer.
-//
-// TODO(bananabrick): Support encoding of virtual sstable state.
 func (v *VersionEdit) Encode(w io.Writer) error {
 	e := versionEditEncoder{new(bytes.Buffer)}
 
@@ -463,6 +512,15 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(tagNextFileNumber)
 		e.writeUvarint(uint64(v.NextFileNum))
 	}
+	for _, dfn := range v.RemovedBackingTables {
+		e.writeUvarint(tagRemovedBackingTable)
+		e.writeUvarint(uint64(dfn.FileNum()))
+	}
+	for _, fileBacking := range v.CreatedBackingTables {
+		e.writeUvarint(tagCreatedBackingTable)
+		e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum()))
+		e.writeUvarint(fileBacking.Size)
+	}
 	// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
 	// even though its value is zero. We detect this by encoding LastSeqNum when
 	// ComparerName is set.
@@ -476,7 +534,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(uint64(x.FileNum))
 	}
 	for _, x := range v.NewFiles {
-		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
+		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
 		var tag uint64
 		switch {
 		case x.Meta.HasRangeKeys:
@@ -529,6 +587,10 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 				e.writeUvarint(customTagNeedsCompaction)
 				e.writeBytes([]byte{1})
 			}
+			if x.Meta.Virtual {
+				e.writeUvarint(customTagVirtual)
+				e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum()))
+			}
 			e.writeUvarint(customTagTerminate)
 		}
 	}
@@ -536,6 +598,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 	return err
 }
 
+// versionEditDecoder should be used to decode version edits.
 type versionEditDecoder struct {
 	byteReader
 }
@@ -634,7 +697,9 @@ type BulkVersionEdit struct {
 	Added   [NumLevels]map[base.FileNum]*FileMetadata
 	Deleted [NumLevels]map[base.FileNum]*FileMetadata
 
-	AddedFileBacking   []*FileBacking
+	// AddedFileBacking is a map to support lookup so that we can populate the
+	// FileBacking of virtual sstables during manifest replay.
+	AddedFileBacking   map[base.DiskFileNum]*FileBacking
 	RemovedFileBacking []base.DiskFileNum
 
 	// AddedByFileNum maps file number to file metadata for all added files
@@ -697,6 +762,16 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		}
 	}
 
+	// Generate state for Added backing files. Note that these must be generated
+	// before we loop through the NewFiles, because we need to populate the
+	// FileBackings which might be used by the NewFiles loop.
+	if b.AddedFileBacking == nil {
+		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
+	}
+	for _, fb := range ve.CreatedBackingTables {
+		b.AddedFileBacking[fb.DiskFileNum] = fb
+	}
+
 	for _, nf := range ve.NewFiles {
 		// A new file should not have been deleted in this or a preceding
 		// VersionEdit at the same level (though files can move across levels).
@@ -705,6 +780,17 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 				return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
 			}
 		}
+		if nf.Meta.Virtual && nf.Meta.FileBacking == nil {
+			// FileBacking for a virtual sstable must only be nil if we're performing
+			// manifest replay.
+			nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum]
+			if nf.Meta.FileBacking == nil {
+				return errors.Errorf("FileBacking for virtual sstable must not be nil")
+			}
+		} else if nf.Meta.FileBacking == nil {
+			return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum)
+		}
+
 		if b.Added[nf.Level] == nil {
 			b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
 		}
@@ -717,9 +803,6 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		}
 	}
 
-	// Generate state for the backing files.
-	b.AddedFileBacking = append(b.AddedFileBacking, ve.CreatedBackingTables...)
-
 	// Since a file can be removed from backing files in exactly one version
 	// edit it is safe to just append without any de-duplication.
 	b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -6,6 +6,7 @@ package rangekey
 
 import (
 	"bytes"
+	"fmt"
 	"math"
 	"sort"
 
@@ -367,3 +368,78 @@ func coalesce(
 	}
 	return nil
 }
+
+// ForeignSSTTransformer implements a keyspan.Transformer for range keys in
+// foreign sstables (i.e. shared sstables not created by us). It is largely
+// similar to the Transform function implemented in UserIteratorConfig in that
+// it calls coalesce to remove range keys shadowed by other range keys, but also
+// retains the range key that does the shadowing. In addition, it elides
+// RangeKey unsets/dels in L6 as they are inapplicable when reading from a
+// different Pebble instance.
+type ForeignSSTTransformer struct {
+	Comparer *base.Comparer
+	Level    int
+	sortBuf  keyspan.KeysBySuffix
+}
+
+// Transform implements the Transformer interface.
+func (f *ForeignSSTTransformer) Transform(
+	cmp base.Compare, s keyspan.Span, dst *keyspan.Span,
+) error {
+	// Apply shadowing of keys.
+	dst.Start = s.Start
+	dst.End = s.End
+	f.sortBuf = keyspan.KeysBySuffix{
+		Cmp:  cmp,
+		Keys: f.sortBuf.Keys[:0],
+	}
+	if err := coalesce(f.Comparer.Equal, &f.sortBuf, math.MaxUint64, s.Keys); err != nil {
+		return err
+	}
+	keys := f.sortBuf.Keys
+	dst.Keys = dst.Keys[:0]
+	for i := range keys {
+		seqNum := keys[i].SeqNum()
+		switch keys[i].Kind() {
+		case base.InternalKeyKindRangeKeySet:
+			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
+				panic("pebble: keys unexpectedly not in ascending suffix order")
+			}
+			switch f.Level {
+			case 5:
+				fallthrough
+			case 6:
+				if seqNum != base.SeqNumForLevel(f.Level) {
+					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
+				}
+			}
+		case base.InternalKeyKindRangeKeyUnset:
+			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
+				panic("pebble: keys unexpectedly not in ascending suffix order")
+			}
+			fallthrough
+		case base.InternalKeyKindRangeKeyDelete:
+			switch f.Level {
+			case 5:
+				// Emit this key.
+				if seqNum != base.SeqNumForLevel(f.Level) {
+					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
+				}
+			case 6:
+				// Skip this key, as foreign sstable in L6 do not need to emit range key
+				// unsets/dels as they do not apply to any other sstables.
+				continue
+			}
+		default:
+			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
+		}
+		dst.Keys = append(dst.Keys, keyspan.Key{
+			Trailer: base.MakeTrailer(seqNum, keys[i].Kind()),
+			Suffix:  keys[i].Suffix,
+			Value:   keys[i].Value,
+		})
+	}
+	// coalesce results in dst.Keys being sorted by Suffix.
+	dst.KeysOrder = keyspan.BySuffixAsc
+	return nil
+}

vendor/github.com/cockroachdb/pebble/iterator.go

@@ -568,7 +568,10 @@ func (i *Iterator) findNextEntry(limit []byte) {
 			i.iterValidityState = IterValid
 			return
 
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
+			// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
+			// only simpler, but is also necessary for correctness due to
+			// InternalKeyKindSSTableInternalObsoleteBit.
 			i.nextUserKey()
 			continue
 
@@ -631,7 +634,10 @@ func (i *Iterator) nextPointCurrentUserKey() bool {
 		i.err = base.CorruptionErrorf("pebble: unexpected range key set mid-user key")
 		return false
 
-	case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+	case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
+		// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
+		// only simpler, but is also necessary for correctness due to
+		// InternalKeyKindSSTableInternalObsoleteBit.
 		return false
 
 	case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -929,7 +935,7 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			// return the key even if the MERGE point key is deleted.
 			rangeKeyBoundary = true
 
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			i.value = LazyValue{}
 			i.iterValidityState = IterExhausted
 			valueMerger = nil
@@ -1092,9 +1098,13 @@ func (i *Iterator) mergeNext(key InternalKey, valueMerger ValueMerger) {
 			return
 		}
 		switch key.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			// We've hit a deletion tombstone. Return everything up to this
 			// point.
+			//
+			// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
+			// only simpler, but is also necessary for correctness due to
+			// InternalKeyKindSSTableInternalObsoleteBit.
 			return
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:

vendor/github.com/cockroachdb/pebble/level_checker.go

@@ -165,7 +165,7 @@ func (m *simpleMergingIter) step() bool {
 		if m.valueMerger != nil {
 			// Ongoing series of MERGE records.
 			switch item.key.Kind() {
-			case InternalKeyKindSingleDelete, InternalKeyKindDelete:
+			case InternalKeyKindSingleDelete, InternalKeyKindDelete, InternalKeyKindDeleteSized:
 				var closer io.Closer
 				_, closer, m.err = m.valueMerger.Finish(true /* includesBase */)
 				if m.err == nil && closer != nil {
@@ -379,7 +379,7 @@ func checkRangeTombstones(c *checkConfig) error {
 			atomicUnit, _ := expandToAtomicUnit(c.cmp, lf.Slice(), true /* disableIsCompacting */)
 			lower, upper := manifest.KeyRange(c.cmp, atomicUnit.Iter())
 			iterToClose, iter, err := c.newIters(
-				context.Background(), lf.FileMetadata, nil, internalIterOpts{})
+				context.Background(), lf.FileMetadata, &IterOptions{level: manifest.Level(lsmLevel)}, internalIterOpts{})
 			if err != nil {
 				return err
 			}

vendor/github.com/cockroachdb/pebble/level_iter.go

@@ -36,9 +36,9 @@ type tableNewIters func(
 // tableNewRangeDelIter takes a tableNewIters and returns a TableNewSpanIter
 // for the rangedel iterator returned by tableNewIters.
 func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter {
-	return func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
+	return func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
 		iter, rangeDelIter, err := newIters(
-			ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters}, internalIterOpts{})
+			ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters, level: iterOptions.Level}, internalIterOpts{})
 		if iter != nil {
 			_ = iter.Close()
 		}
@@ -200,6 +200,11 @@ type levelIter struct {
 	// cache when constructing new table iterators.
 	internalOpts internalIterOpts
 
+	// Scratch space for the obsolete keys filter, when there are no other block
+	// property filters specified. See the performance note where
+	// IterOptions.PointKeyFilters is declared.
+	filtersBuf [1]BlockPropertyFilter
+
 	// Disable invariant checks even if they are otherwise enabled. Used by tests
 	// which construct "impossible" situations (e.g. seeking to a key before the
 	// lower bound).
@@ -267,8 +272,12 @@ func (l *levelIter) init(
 	l.upper = opts.UpperBound
 	l.tableOpts.TableFilter = opts.TableFilter
 	l.tableOpts.PointKeyFilters = opts.PointKeyFilters
+	if len(opts.PointKeyFilters) == 0 {
+		l.tableOpts.PointKeyFilters = l.filtersBuf[:0:1]
+	}
 	l.tableOpts.UseL6Filters = opts.UseL6Filters
 	l.tableOpts.level = l.level
+	l.tableOpts.snapshotForHideObsoletePoints = opts.snapshotForHideObsoletePoints
 	l.cmp = cmp
 	l.split = split
 	l.iterFile = nil

vendor/github.com/cockroachdb/pebble/metrics.go

@@ -184,6 +184,11 @@ type Metrics struct {
 		Duration time.Duration
 	}
 
+	Ingest struct {
+		// The total number of ingestions
+		Count uint64
+	}
+
 	Flush struct {
 		// The total number of flushes.
 		Count           int64
@@ -225,6 +230,9 @@ type Metrics struct {
 		// The approximate count of internal tombstones (DEL, SINGLEDEL and
 		// RANGEDEL key kinds) within the database.
 		TombstoneCount uint64
+		// A cumulative total number of missized DELSIZED keys encountered by
+		// compactions since the database was opened.
+		MissizedTombstonesCount uint64
 	}
 
 	Snapshots struct {
@@ -504,6 +512,9 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 		notApplicable,
 		notApplicable,
 		redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
+	w.Printf(" ingest %9d\n",
+		redact.Safe(m.Ingest.Count),
+	)
 }
 
 func hitRate(hits, misses int64) float64 {

vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go

@@ -27,8 +27,8 @@ func (h *NoopReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error
 // Close is part of the ReadHandle interface.
 func (*NoopReadHandle) Close() error { return nil }
 
-// MaxReadahead is part of the ReadHandle interface.
-func (*NoopReadHandle) MaxReadahead() {}
+// SetupForCompaction is part of the ReadHandle interface.
+func (*NoopReadHandle) SetupForCompaction() {}
 
 // RecordCacheHit is part of the ReadHandle interface.
 func (*NoopReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {}

vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go

@@ -16,6 +16,9 @@ import (
 type Readable interface {
 	// ReadAt reads len(p) bytes into p starting at offset off.
 	//
+	// Does not return partial results; if off + len(p) is past the end of the
+	// object, an error is returned.
+	//
 	// Clients of ReadAt can execute parallel ReadAt calls on the
 	// same Readable.
 	ReadAt(ctx context.Context, p []byte, off int64) error
@@ -39,14 +42,18 @@ type Readable interface {
 type ReadHandle interface {
 	// ReadAt reads len(p) bytes into p starting at offset off.
 	//
+	// Does not return partial results; if off + len(p) is past the end of the
+	// object, an error is returned.
+	//
 	// Parallel ReadAt calls on the same ReadHandle are not allowed.
 	ReadAt(ctx context.Context, p []byte, off int64) error
 
 	Close() error
 
-	// MaxReadahead configures the implementation to expect large sequential
-	// reads. Used to skip any initial read-ahead ramp-up.
-	MaxReadahead()
+	// SetupForCompaction informs the implementation that the read handle will
+	// be used to read data blocks for a compaction. The implementation can expect
+	// sequential reads, and can decide to not retain data in any caches.
+	SetupForCompaction()
 
 	// RecordCacheHit informs the implementation that we were able to retrieve a
 	// block from cache. This is useful for example when the implementation is
@@ -209,6 +216,11 @@ type Provider interface {
 	// Cannot be called if shared storage is not configured for the provider.
 	SetCreatorID(creatorID CreatorID) error
 
+	// IsForeign returns whether this object is owned by a different node. Return
+	// value undefined if creator ID is not set yet, or if this object does not
+	// exist in this provider.
+	IsForeign(meta ObjectMetadata) bool
+
 	// SharedObjectBacking encodes the shared object metadata.
 	SharedObjectBacking(meta *ObjectMetadata) (SharedObjectBackingHandle, error)
 

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go

@@ -16,9 +16,9 @@ const (
 	// RecordCacheHitOp happens when a read is satisfied from the block cache. See
 	// objstorage.ReadHandle.RecordCacheHit().
 	RecordCacheHitOp
-	// MaxReadaheadOp is a "meta operation" that configures a read handle for
-	// large sequential reads. See objstorage.ReadHandle.MaxReadahead().
-	MaxReadaheadOp
+	// SetupForCompactionOp is a "meta operation" that configures a read handle
+	// for large sequential reads. See objstorage.ReadHandle.SetupForCompaction().
+	SetupForCompactionOp
 )
 
 // Reason indicates the higher-level context of the operation.

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go

@@ -209,14 +209,14 @@ func (rh *readHandle) Close() error {
 	return rh.rh.Close()
 }
 
-// MaxReadahead is part of the objstorage.ReadHandle interface.
-func (rh *readHandle) MaxReadahead() {
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) SetupForCompaction() {
 	rh.g.add(context.Background(), Event{
-		Op:       MaxReadaheadOp,
+		Op:       SetupForCompactionOp,
 		FileNum:  rh.fileNum,
 		HandleID: rh.handleID,
 	})
-	rh.rh.MaxReadahead()
+	rh.rh.SetupForCompaction()
 }
 
 // RecordCacheHit is part of the objstorage.ReadHandle interface.

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go

@@ -93,6 +93,21 @@ type Settings struct {
 	// (experimental).
 	Shared struct {
 		Storage shared.Storage
+
+		// CacheSizeBytes is the size of the on-disk block cache for objects
+		// on shared storage. If it is 0, no cache is used.
+		CacheSizeBytes int64
+
+		// CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used.
+		CacheBlockSize int
+
+		// The number of independent shards the cache leverages. Each shard is the same size,
+		// and a hash of filenum & offset map a read to a certain shard. If set to 0,
+		// 2*runtime.GOMAXPROCS is used as the shard count.
+		CacheShardCount int
+
+		// TODO(radu): allow the cache to live on another FS/location (e.g. to use
+		// instance-local SSD).
 	}
 }
 

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go

@@ -11,15 +11,15 @@ const (
 	minFileReadsForReadahead = 2
 	// TODO(bilal): Have the initial size value be a factor of the block size,
 	// as opposed to a hardcoded value.
-	initialReadaheadSize = 64 << 10  /* 64KB */
-	maxReadaheadSize     = 256 << 10 /* 256KB */
+	initialReadaheadSize = 64 << 10 /* 64KB */
 )
 
 // readaheadState contains state variables related to readahead. Updated on
 // file reads.
 type readaheadState struct {
 	// Number of sequential reads.
-	numReads int64
+	numReads         int64
+	maxReadaheadSize int64
 	// Size issued to the next call to Prefetch. Starts at or above
 	// initialReadaheadSize and grows exponentially until maxReadaheadSize.
 	size int64
@@ -32,22 +32,23 @@ type readaheadState struct {
 	limit int64
 }
 
-func makeReadaheadState() readaheadState {
+func makeReadaheadState(maxReadaheadSize int64) readaheadState {
 	return readaheadState{
-		size: initialReadaheadSize,
+		size:             initialReadaheadSize,
+		maxReadaheadSize: maxReadaheadSize,
 	}
 }
 
 func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
 	currentReadEnd := offset + blockLength
 	if rs.numReads >= minFileReadsForReadahead {
-		if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+		if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 			// This is a read that would have resulted in a readahead, had it
 			// not been a cache hit.
 			rs.limit = currentReadEnd
 			return
 		}
-		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
 			// We read too far away from rs.limit to benefit from readahead in
 			// any scenario. Reset all variables.
 			rs.numReads = 1
@@ -61,7 +62,7 @@ func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
 		// readahead.
 		return
 	}
-	if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 		// Blocks are being read sequentially and would benefit from readahead
 		// down the line.
 		rs.numReads++
@@ -89,28 +90,28 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		// as well as the interval where a read would benefit from read ahead:
 		// [rs.limit, rs.limit + rs.size]
 		// We increase the latter interval to
-		// [rs.limit, rs.limit + maxReadaheadSize] to account for cases where
+		// [rs.limit, rs.limit + rs.maxReadaheadSize] to account for cases where
 		// readahead may not be beneficial with a small readahead size, but over
 		// time the readahead size would increase exponentially to make it
 		// beneficial.
-		if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+		if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 			// We are doing a read in the interval ahead of
 			// the last readahead range. In the diagrams below, ++++ is the last
 			// readahead range, ==== is the range represented by
-			// [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range
+			// [rs.limit, rs.limit + rs.maxReadaheadSize], and ---- is the range
 			// being read.
 			//
-			//               rs.limit           rs.limit + maxReadaheadSize
+			//               rs.limit           rs.limit + rs.maxReadaheadSize
 			//         ++++++++++|===========================|
 			//
 			//              |-------------|
 			//            offset       currentReadEnd
 			//
 			// This case is also possible, as are all cases with an overlap
-			// between [rs.limit, rs.limit + maxReadaheadSize] and [offset,
+			// between [rs.limit, rs.limit + rs.maxReadaheadSize] and [offset,
 			// currentReadEnd]:
 			//
-			//               rs.limit           rs.limit + maxReadaheadSize
+			//               rs.limit           rs.limit + rs.maxReadaheadSize
 			//         ++++++++++|===========================|
 			//
 			//                                            |-------------|
@@ -122,19 +123,19 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 			rs.prevSize = rs.size
 			// Increase rs.size for the next read.
 			rs.size *= 2
-			if rs.size > maxReadaheadSize {
-				rs.size = maxReadaheadSize
+			if rs.size > rs.maxReadaheadSize {
+				rs.size = rs.maxReadaheadSize
 			}
 			return rs.prevSize
 		}
-		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
 			// The above conditional has rs.limit > rs.prevSize to confirm that
 			// rs.limit - rs.prevSize would not underflow.
 			// We read too far away from rs.limit to benefit from readahead in
 			// any scenario. Reset all variables.
 			// The case where we read too far ahead:
 			//
-			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + maxReadaheadSize)
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 			//                    |+++++++++++++|=============|
 			//
 			//                                                  |-------------|
@@ -142,7 +143,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 			//
 			// Or too far behind:
 			//
-			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + maxReadaheadSize)
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 			//                    |+++++++++++++|=============|
 			//
 			//    |-------------|
@@ -160,7 +161,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		// readahead, but there's no reason to issue a readahead call at the
 		// moment.
 		//
-		// (rs.limit - rs.prevSize)            (rs.limit + maxReadaheadSize)
+		// (rs.limit - rs.prevSize)            (rs.limit + rs.maxReadaheadSize)
 		//                    |+++++++++++++|===============|
 		//                             (rs.limit)
 		//
@@ -170,11 +171,11 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		rs.numReads++
 		return 0
 	}
-	if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 		// Blocks are being read sequentially and would benefit from readahead
 		// down the line.
 		//
-		//                       (rs.limit)   (rs.limit + maxReadaheadSize)
+		//                       (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 		//                         |=============|
 		//
 		//                    |-------|
@@ -186,7 +187,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 	// We read too far ahead of the last read, or before it. This indicates
 	// a random read, where readahead is not desirable. Reset all variables.
 	//
-	// (rs.limit - maxReadaheadSize)  (rs.limit)   (rs.limit + maxReadaheadSize)
+	// (rs.limit - rs.maxReadaheadSize)  (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 	//                     |+++++++++++++|=============|
 	//
 	//                                                    |-------|

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go

@@ -6,6 +6,7 @@ package objstorageprovider
 
 import (
 	"context"
+	"runtime"
 	"sync"
 	"sync/atomic"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
 	"github.com/cockroachdb/pebble/objstorage/shared"
 )
@@ -21,6 +23,7 @@ import (
 // All fields remain unset if shared storage is not configured.
 type sharedSubsystem struct {
 	catalog *sharedobjcat.Catalog
+	cache   *sharedcache.Cache
 
 	// checkRefsOnOpen controls whether we check the ref marker file when opening
 	// an object. Normally this is true when invariants are enabled (but the provider
@@ -65,6 +68,24 @@ func (p *provider) sharedInit() error {
 		p.st.Logger.Infof("shared storage configured; no creatorID yet")
 	}
 
+	if p.st.Shared.CacheSizeBytes > 0 {
+		const defaultBlockSize = 32 * 1024
+		blockSize := p.st.Shared.CacheBlockSize
+		if blockSize == 0 {
+			blockSize = defaultBlockSize
+		}
+
+		numShards := p.st.Shared.CacheShardCount
+		if numShards == 0 {
+			numShards = 2 * runtime.GOMAXPROCS(0)
+		}
+
+		p.shared.cache, err = sharedcache.Open(p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, p.st.Shared.CacheSizeBytes, numShards)
+		if err != nil {
+			return errors.Wrapf(err, "pebble: could not open shared object cache")
+		}
+	}
+
 	for _, meta := range contents.Objects {
 		o := objstorage.ObjectMetadata{
 			DiskFileNum: meta.FileNum,
@@ -95,6 +116,14 @@ func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
 	return nil
 }
 
+// IsForeign is part of the objstorage.Provider interface.
+func (p *provider) IsForeign(meta objstorage.ObjectMetadata) bool {
+	if !p.shared.initialized.Load() {
+		return false
+	}
+	return meta.IsShared() && p.shared.creatorID != meta.Shared.CreatorID
+}
+
 func (p *provider) sharedCheckInitialized() error {
 	if p.sharedStorage() == nil {
 		return errors.Errorf("shared object support not configured")
@@ -211,7 +240,7 @@ func (p *provider) sharedOpenForReading(
 		}
 		return nil, err
 	}
-	return newSharedReadable(reader, size), nil
+	return p.newSharedReadable(reader, size, meta.DiskFileNum), nil
 }
 
 func (p *provider) sharedSize(meta objstorage.ObjectMetadata) (int64, error) {

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go

@@ -6,28 +6,55 @@ package objstorageprovider
 
 import (
 	"context"
+	"io"
 
+	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
 	"github.com/cockroachdb/pebble/objstorage/shared"
 )
 
+const sharedMaxReadaheadSize = 1024 * 1024 /* 1MB */
+
 // sharedReadable is a very simple implementation of Readable on top of the
 // ReadCloser returned by shared.Storage.CreateObject.
 type sharedReadable struct {
 	objReader shared.ObjectReader
 	size      int64
+	fileNum   base.DiskFileNum
+	provider  *provider
 }
 
 var _ objstorage.Readable = (*sharedReadable)(nil)
 
-func newSharedReadable(objReader shared.ObjectReader, size int64) *sharedReadable {
+func (p *provider) newSharedReadable(
+	objReader shared.ObjectReader, size int64, fileNum base.DiskFileNum,
+) *sharedReadable {
 	return &sharedReadable{
 		objReader: objReader,
 		size:      size,
+		fileNum:   fileNum,
+		provider:  p,
 	}
 }
 
+// ReadAt is part of the objstorage.Readable interface.
 func (r *sharedReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	return r.readInternal(ctx, p, offset, false /* forCompaction */)
+}
+
+// readInternal performs a read for the object, using the cache when
+// appropriate.
+func (r *sharedReadable) readInternal(
+	ctx context.Context, p []byte, offset int64, forCompaction bool,
+) error {
+	if cache := r.provider.shared.cache; cache != nil {
+		flags := sharedcache.ReadFlags{
+			// Don't add data to the cache if this read is for a compaction.
+			ReadOnly: forCompaction,
+		}
+		return r.provider.shared.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
+	}
 	return r.objReader.ReadAt(ctx, p, offset)
 }
 
@@ -42,24 +69,94 @@ func (r *sharedReadable) Size() int64 {
 
 func (r *sharedReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
 	// TODO(radu): use a pool.
-	return &sharedReadHandle{readable: r}
+	rh := &sharedReadHandle{readable: r}
+	rh.readahead.state = makeReadaheadState(sharedMaxReadaheadSize)
+	return rh
 }
 
 type sharedReadHandle struct {
-	readable *sharedReadable
+	readable  *sharedReadable
+	readahead struct {
+		state  readaheadState
+		data   []byte
+		offset int64
+	}
+	forCompaction bool
 }
 
 var _ objstorage.ReadHandle = (*sharedReadHandle)(nil)
 
+// ReadAt is part of the objstorage.ReadHandle interface.
 func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
-	return r.readable.ReadAt(ctx, p, offset)
+	readaheadSize := r.maybeReadahead(offset, len(p))
+
+	// Check if we already have the data from a previous read-ahead.
+	if rhSize := int64(len(r.readahead.data)); rhSize > 0 {
+		if r.readahead.offset <= offset && r.readahead.offset+rhSize > offset {
+			n := copy(p, r.readahead.data[offset-r.readahead.offset:])
+			if n == len(p) {
+				// All data was available.
+				return nil
+			}
+			// Use the data that we had and do a shorter read.
+			offset += int64(n)
+			p = p[n:]
+			readaheadSize -= n
+		}
+	}
+
+	if readaheadSize > len(p) {
+		// Don't try to read past EOF.
+		if offset+int64(readaheadSize) > r.readable.size {
+			readaheadSize = int(r.readable.size - offset)
+			if readaheadSize <= 0 {
+				// This shouldn't happen in practice (Pebble should never try to read
+				// past EOF).
+				return io.EOF
+			}
+		}
+		r.readahead.offset = offset
+		// TODO(radu): we need to somehow account for this memory.
+		if cap(r.readahead.data) >= readaheadSize {
+			r.readahead.data = r.readahead.data[:readaheadSize]
+		} else {
+			r.readahead.data = make([]byte, readaheadSize)
+		}
+
+		if err := r.readable.readInternal(ctx, r.readahead.data, offset, r.forCompaction); err != nil {
+			// Make sure we don't treat the data as valid next time.
+			r.readahead.data = r.readahead.data[:0]
+			return err
+		}
+		copy(p, r.readahead.data)
+		return nil
+	}
+
+	return r.readable.readInternal(ctx, p, offset, r.forCompaction)
 }
 
+func (r *sharedReadHandle) maybeReadahead(offset int64, len int) int {
+	if r.forCompaction {
+		return sharedMaxReadaheadSize
+	}
+	return int(r.readahead.state.maybeReadahead(offset, int64(len)))
+}
+
+// Close is part of the objstorage.ReadHandle interface.
 func (r *sharedReadHandle) Close() error {
 	r.readable = nil
+	r.readahead.data = nil
 	return nil
 }
 
-func (r *sharedReadHandle) MaxReadahead() {}
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (r *sharedReadHandle) SetupForCompaction() {
+	r.forCompaction = true
+}
 
-func (r *sharedReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {}
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (r *sharedReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
+	if !r.forCompaction {
+		r.readahead.state.recordCacheHit(offset, size)
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go

@@ -0,0 +1,570 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sharedcache
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"sync/atomic"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage/shared"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+const (
+	shardingBlockSize = 1024 * 1024
+)
+
+// Cache is a persistent cache backed by a local filesystem. It is intended
+// to cache data that is in slower shared storage (e.g. S3), hence the
+// package name 'sharedcache'.
+type Cache struct {
+	shards []shard
+	logger base.Logger
+
+	blockSize int
+
+	writeBackWaitGroup sync.WaitGroup
+	// TODO(josh): Have a dedicated metrics struct. Right now, this
+	// is just for testing.
+	misses atomic.Int32
+}
+
+// Open opens a cache. If there is no existing cache at fsDir, a new one
+// is created.
+func Open(
+	fs vfs.FS, logger base.Logger, fsDir string, blockSize int, sizeBytes int64, numShards int,
+) (*Cache, error) {
+	min := shardingBlockSize * int64(numShards)
+	if sizeBytes < min {
+		return nil, errors.Errorf("cache size %d lower than min %d", sizeBytes, min)
+	}
+
+	sc := &Cache{
+		logger:    logger,
+		blockSize: blockSize,
+	}
+	sc.shards = make([]shard, numShards)
+	blocksPerShard := sizeBytes / int64(numShards) / int64(blockSize)
+	for i := range sc.shards {
+		if err := sc.shards[i].init(fs, fsDir, i, blocksPerShard, blockSize); err != nil {
+			return nil, err
+		}
+	}
+	return sc, nil
+}
+
+// Close closes the cache. Methods such as ReadAt should not be called after Close is
+// called.
+func (c *Cache) Close() error {
+	c.writeBackWaitGroup.Wait()
+
+	var retErr error
+	for i := range c.shards {
+		if err := c.shards[i].close(); err != nil && retErr == nil {
+			retErr = err
+		}
+	}
+	c.shards = nil
+	return retErr
+}
+
+// ReadFlags contains options for Cache.ReadAt.
+type ReadFlags struct {
+	// ReadOnly instructs ReadAt to not write any new data into the cache; it is
+	// used when the data is unlikely to be used again.
+	ReadOnly bool
+}
+
+// ReadAt performs a read form an object, attempting to use cached data when
+// possible.
+func (c *Cache) ReadAt(
+	ctx context.Context,
+	fileNum base.DiskFileNum,
+	p []byte,
+	ofs int64,
+	objReader shared.ObjectReader,
+	objSize int64,
+	flags ReadFlags,
+) error {
+	if ofs >= objSize {
+		if invariants.Enabled {
+			panic("invalid ReadAt offset")
+		}
+		return io.EOF
+	}
+	// TODO(radu): for compaction reads, we may not want to read from the cache at
+	// all.
+	{
+		n, err := c.get(fileNum, p, ofs)
+		if err != nil {
+			return err
+		}
+		if n == len(p) {
+			// Everything was in cache!
+			return nil
+		}
+
+		// Note this. The below code does not need the original ofs, as with the earlier
+		// reading from the cache done, the relevant offset is ofs + int64(n). Same with p.
+		ofs += int64(n)
+		p = p[n:]
+
+		if invariants.Enabled {
+			if n != 0 && ofs%int64(c.blockSize) != 0 {
+				panic(fmt.Sprintf("after non-zero read from cache, ofs is not block-aligned: %v %v", ofs, n))
+			}
+		}
+	}
+
+	c.misses.Add(1)
+
+	if flags.ReadOnly {
+		return objReader.ReadAt(ctx, p, ofs)
+	}
+
+	// We must do reads with offset & size that are multiples of the block size. Else
+	// later cache hits may return incorrect zeroed results from the cache.
+	firstBlockInd := ofs / int64(c.blockSize)
+	adjustedOfs := firstBlockInd * int64(c.blockSize)
+
+	// Take the length of what is left to read plus the length of the adjustment of
+	// the offset plus the size of a block minus one and divide by the size of a block
+	// to get the number of blocks to read from the object.
+	sizeOfOffAdjustment := int(ofs - adjustedOfs)
+	numBlocksToRead := ((len(p) + sizeOfOffAdjustment) + (c.blockSize - 1)) / c.blockSize
+	adjustedLen := numBlocksToRead * c.blockSize
+	adjustedP := make([]byte, adjustedLen)
+
+	// Read the rest from the object. We may need to cap the length to avoid past EOF reads.
+	eofCap := int64(adjustedLen)
+	if adjustedOfs+eofCap > objSize {
+		eofCap = objSize - adjustedOfs
+	}
+	if err := objReader.ReadAt(ctx, adjustedP[:eofCap], adjustedOfs); err != nil {
+		return err
+	}
+	copy(p, adjustedP[sizeOfOffAdjustment:])
+
+	// TODO(josh): For writing back to the cache, we may want a concurrency-limited approach
+	// that does batching, instead of what is below. I think it's reasonable though to
+	// run a production experiment with what is below, before making adjustments. Note that
+	// the writes being done are in multiples of the filesystem block size. The filesystem
+	// might do okay with them.
+	c.writeBackWaitGroup.Add(1)
+	go func() {
+		defer c.writeBackWaitGroup.Done()
+		if err := c.set(fileNum, adjustedP, adjustedOfs); err != nil {
+			// TODO(josh): Would like to log at error severity, but base.Logger doesn't
+			// have error severity.
+			c.logger.Infof("writing back to cache after miss failed: %v", err)
+		}
+	}()
+	return nil
+}
+
+// get attempts to read the requested data from the cache, if it is already
+// there.
+//
+// If all data is available, returns n = len(p).
+//
+// If data is partially available, a prefix of the data is read; returns n < len(p)
+// and no error. If no prefix is available, returns n = 0 and no error.
+func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
+	// The data extent might cross shard boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	for {
+		shard := c.getShard(fileNum, ofs+int64(n))
+		cappedLen := len(p[n:])
+		if toBoundary := int(shardingBlockSize - ((ofs + int64(n)) % shardingBlockSize)); cappedLen > toBoundary {
+			cappedLen = toBoundary
+		}
+		numRead, err := shard.get(fileNum, p[n:n+cappedLen], ofs+int64(n))
+		if err != nil {
+			return n, err
+		}
+		n += numRead
+		if numRead < cappedLen {
+			// We only read a prefix from this shard.
+			return n, nil
+		}
+		if n == len(p) {
+			// We are done.
+			return n, nil
+		}
+		// Data extent crosses shard boundary, continue with next shard.
+	}
+}
+
+// set attempts to write the requested data to the cache. Both ofs & len(p) must
+// be multiples of the block size.
+//
+// If all of p is not written to the shard, set returns a non-nil error.
+func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
+	if invariants.Enabled {
+		if ofs%int64(c.blockSize) != 0 || len(p)%c.blockSize != 0 {
+			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
+		}
+	}
+
+	// The data extent might cross shard boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	n := 0
+	for {
+		shard := c.getShard(fileNum, ofs+int64(n))
+		cappedLen := len(p[n:])
+		if toBoundary := int(shardingBlockSize - ((ofs + int64(n)) % shardingBlockSize)); cappedLen > toBoundary {
+			cappedLen = toBoundary
+		}
+		err := shard.set(fileNum, p[n:n+cappedLen], ofs+int64(n))
+		if err != nil {
+			return err
+		}
+		// set returns an error if cappedLen bytes aren't written to the shard.
+		n += cappedLen
+		if n == len(p) {
+			// We are done.
+			return nil
+		}
+		// Data extent crosses shard boundary, continue with next shard.
+	}
+}
+
+func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard {
+	const prime64 = 1099511628211
+	hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs)/shardingBlockSize
+	// TODO(josh): Instance change ops are often run in production. Such an operation
+	// updates len(c.shards); see openSharedCache. As a result, the behavior of this
+	// function changes, and the cache empties out at restart time. We may want a better
+	// story here eventually.
+	return &c.shards[hash%uint64(len(c.shards))]
+}
+
+type shard struct {
+	file         vfs.File
+	sizeInBlocks int64
+	blockSize    int
+	mu           struct {
+		sync.Mutex
+		// TODO(josh): None of these datastructures are space-efficient.
+		// Focusing on correctness to start.
+		where whereMap
+		locks []lockState
+		free  []cacheBlockIndex
+	}
+}
+
+// Maps a logical block in an SST to an index of the cache block with the
+// file contents (to the "cache block index").
+type whereMap map[logicalBlockID]cacheBlockIndex
+
+type logicalBlockID struct {
+	filenum base.DiskFileNum
+	// Offset into the file, in units of shard.BlockSize.
+	offsetInUnitsOfBlocks int64
+}
+
+type cacheBlockIndex int64
+
+type lockState int64
+
+const (
+	unlocked lockState = 0
+	// >0 lockState tracks the number of distinct readers of some cache block / logical block
+	// which is in the secondary cache. It is used to ensure that a cache block is not evicted
+	// and overwritten, while there are active readers.
+	readLockTakenInc = 1
+	// -1 lockState indicates that some cache block is currently being populated with data from
+	// blob storage. It is used to ensure that a cache block is not read or evicted again, while
+	// it is being populated.
+	writeLockTaken = -1
+)
+
+func (s *shard) init(
+	fs vfs.FS, fsDir string, shardIdx int, sizeInBlocks int64, blockSize int,
+) error {
+	*s = shard{
+		sizeInBlocks: sizeInBlocks,
+	}
+	if blockSize < 1024 || shardingBlockSize%blockSize != 0 {
+		return errors.Newf("invalid block size %d (must divide %d)", blockSize, shardingBlockSize)
+	}
+	s.blockSize = blockSize
+	file, err := fs.OpenReadWrite(fs.PathJoin(fsDir, fmt.Sprintf("SHARED-CACHE-%03d", shardIdx)))
+	if err != nil {
+		return err
+	}
+	// TODO(radu): truncate file if necessary (especially important if we restart
+	// with more shards).
+	if err := file.Preallocate(0, int64(blockSize)*sizeInBlocks); err != nil {
+		return err
+	}
+	s.file = file
+
+	// TODO(josh): Right now, the secondary cache is not persistent. All existing
+	// cache contents will be over-written, since all metadata is only stored in
+	// memory.
+	s.mu.where = make(whereMap)
+	for i := int64(0); i < sizeInBlocks; i++ {
+		s.mu.locks = append(s.mu.locks, unlocked)
+		s.mu.free = append(s.mu.free, cacheBlockIndex(i))
+	}
+
+	return nil
+}
+
+func (s *shard) close() error {
+	defer func() {
+		s.file = nil
+	}()
+	return s.file.Close()
+}
+
+// get attempts to read the requested data from the shard. The data must not
+// cross a shard boundary.
+//
+// If all data is available, returns n = len(p).
+//
+// If data is partially available, a prefix of the data is read; returns n < len(p)
+// and no error. If no prefix is available, returns n = 0 and no error.
+//
+// TODO(josh): Today, if there are two cache blocks needed to satisfy a read, and the
+// first block is not in the cache and the second one is, we will read both from
+// blob storage. We should fix this. This is not an unlikely scenario if we are doing
+// a reverse scan, since those iterate over sstable blocks in reverse order and due to
+// cache block aligned reads will have read the suffix of the sstable block that will
+// be needed next.
+func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
+	if invariants.Enabled {
+		if ofs/shardingBlockSize != (ofs+int64(len(p))-1)/shardingBlockSize {
+			panic(fmt.Sprintf("get crosses shard boundary: %v %v", ofs, len(p)))
+		}
+		s.assertShardStateIsConsistent()
+	}
+
+	// The data extent might cross cache block boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	for {
+		k := logicalBlockID{
+			filenum:               fileNum,
+			offsetInUnitsOfBlocks: (ofs + int64(n)) / int64(s.blockSize),
+		}
+		s.mu.Lock()
+		cacheBlockInd, ok := s.mu.where[k]
+		// TODO(josh): Multiple reads within the same few milliseconds (anything that is smaller
+		// than blob storage read latency) that miss on the same logical block ID will not necessarily
+		// be rare. We may want to do only one read, with the later readers blocking on the first read
+		// completing. This could be implemented either here or in the primary block cache. See
+		// https://github.com/cockroachdb/pebble/pull/2586 for additional discussion.
+		if !ok {
+			s.mu.Unlock()
+			return n, nil
+		}
+		if s.mu.locks[cacheBlockInd] == writeLockTaken {
+			// In practice, if we have two reads of the same SST block in close succession, we
+			// would expect the second to hit in the in-memory block cache. So it's not worth
+			// optimizing this case here.
+			s.mu.Unlock()
+			return n, nil
+		}
+		s.mu.locks[cacheBlockInd] += readLockTakenInc
+		s.mu.Unlock()
+
+		readAt := int64(cacheBlockInd) * int64(s.blockSize)
+		if n == 0 { // if first read
+			readAt += ofs % int64(s.blockSize)
+		}
+		readSize := s.blockSize
+		if n == 0 { // if first read
+			// Cast to int safe since ofs is modded by block size.
+			readSize -= int(ofs % int64(s.blockSize))
+		}
+
+		if len(p[n:]) <= readSize {
+			numRead, err := s.file.ReadAt(p[n:], readAt)
+			s.dropReadLock(cacheBlockInd)
+			return n + numRead, err
+		}
+		numRead, err := s.file.ReadAt(p[n:n+readSize], readAt)
+		s.dropReadLock(cacheBlockInd)
+		if err != nil {
+			return 0, err
+		}
+
+		// Note that numRead == readSize, since we checked for an error above.
+		n += numRead
+	}
+}
+
+// set attempts to write the requested data to the shard. The data must not
+// cross a shard boundary, and both ofs & len(p) must be multiples of the
+// block size.
+//
+// If all of p is not written to the shard, set returns a non-nil error.
+func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
+	if invariants.Enabled {
+		if ofs/shardingBlockSize != (ofs+int64(len(p))-1)/shardingBlockSize {
+			panic(fmt.Sprintf("set crosses shard boundary: %v %v", ofs, len(p)))
+		}
+		if ofs%int64(s.blockSize) != 0 || len(p)%s.blockSize != 0 {
+			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
+		}
+		s.assertShardStateIsConsistent()
+	}
+
+	// The data extent might cross cache block boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	n := 0
+	for {
+		if n == len(p) {
+			return nil
+		}
+		if invariants.Enabled {
+			if n > len(p) {
+				panic(fmt.Sprintf("set with n greater than len(p): %v %v", n, len(p)))
+			}
+		}
+
+		// If the logical block is already in the cache, we should skip doing a set.
+		k := logicalBlockID{
+			filenum:               fileNum,
+			offsetInUnitsOfBlocks: (ofs + int64(n)) / int64(s.blockSize),
+		}
+		s.mu.Lock()
+		if _, ok := s.mu.where[k]; ok {
+			s.mu.Unlock()
+			n += s.blockSize
+			continue
+		}
+
+		// Determine cache block index by looking at the free list and randomly evicting if
+		// it is empty.
+		var cacheBlockInd cacheBlockIndex
+		if len(s.mu.free) == 0 {
+			// TODO(josh): Right now, we do random eviction. Eventually, we will do something
+			// more sophisticated, e.g. leverage ClockPro.
+			foundBlockToEvict := false
+			var k logicalBlockID
+			for k1, v := range s.mu.where {
+				// This implies that hot blocks will not be evicted as often, meaning the
+				// eviction scheme is not quite random. I think that's fine for now, since we plan
+				// to leverage a more sophisticated eviction scheme eventually anyway.
+				lock := s.mu.locks[v]
+				if lock >= readLockTakenInc || lock == writeLockTaken {
+					continue
+				}
+				foundBlockToEvict = true
+				cacheBlockInd = v
+				k = k1
+				break
+			}
+			// TODO(josh): We may want to block until a block frees up, instead of returning
+			// an error here. But I think we can do that later on, e.g. after running some production
+			// experiments.
+			if !foundBlockToEvict {
+				s.mu.Unlock()
+				return errors.New("no block to evict so skipping write to cache")
+			}
+			delete(s.mu.where, k)
+		} else {
+			cacheBlockInd = s.mu.free[len(s.mu.free)-1]
+			s.mu.free = s.mu.free[:len(s.mu.free)-1]
+		}
+
+		s.mu.where[k] = cacheBlockInd
+		s.mu.locks[cacheBlockInd] = writeLockTaken
+		s.mu.Unlock()
+
+		writeAt := int64(cacheBlockInd) * int64(s.blockSize)
+
+		writeSize := s.blockSize
+		if len(p[n:]) <= writeSize {
+			writeSize = len(p[n:])
+		}
+
+		_, err := s.file.WriteAt(p[n:n+writeSize], writeAt)
+		if err != nil {
+			s.freeBlock(k, cacheBlockInd)
+			return err
+		}
+		s.dropWriteLock(cacheBlockInd)
+
+		n += writeSize
+	}
+}
+
+// Doesn't inline currently. This might be okay, but something to keep in mind.
+func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) {
+	s.mu.Lock()
+	s.mu.locks[cacheBlockInd] -= readLockTakenInc
+	if invariants.Enabled {
+		if s.mu.locks[cacheBlockInd] < 0 {
+			panic(fmt.Sprintf("unexpected lock state %v in dropReadLock: %v", s.mu.locks[cacheBlockInd], s.mu.locks))
+		}
+	}
+	s.mu.Unlock()
+}
+
+// Doesn't inline currently. This might be okay, but something to keep in mind.
+func (s *shard) dropWriteLock(cacheBlockInd cacheBlockIndex) {
+	s.mu.Lock()
+	if invariants.Enabled {
+		if s.mu.locks[cacheBlockInd] != writeLockTaken {
+			panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock: %v", s.mu.locks[cacheBlockInd], s.mu.locks))
+		}
+	}
+	s.mu.locks[cacheBlockInd] = unlocked
+	s.mu.Unlock()
+}
+
+// Doesn't inline currently. This might be okay, but something to keep in mind.
+func (s *shard) freeBlock(k logicalBlockID, cacheBlockInd cacheBlockIndex) {
+	s.mu.Lock()
+	delete(s.mu.where, k)
+	s.mu.locks[cacheBlockInd] = unlocked
+	s.mu.free = append(s.mu.free, cacheBlockInd)
+	s.mu.Unlock()
+}
+
+func (s *shard) assertShardStateIsConsistent() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	cacheBlockInds := make(map[cacheBlockIndex]bool)
+	for _, v := range s.mu.where {
+		if cacheBlockInds[v] {
+			panic(fmt.Sprintf("repeated cache block index (where): %v %v %v %v", v, cacheBlockInds, s.mu.where, s.mu.free))
+		}
+		cacheBlockInds[v] = true
+	}
+	for _, v := range s.mu.free {
+		if cacheBlockInds[v] {
+			panic(fmt.Sprintf("repeated cache block index (free): %v %v %v %v", v, cacheBlockInds, s.mu.where, s.mu.free))
+		}
+		cacheBlockInds[v] = true
+	}
+	for i := int64(0); i < s.sizeInBlocks; i++ {
+		if !cacheBlockInds[cacheBlockIndex(i)] {
+			panic(fmt.Sprintf("missing cache block index: %v %v %v %v", i, cacheBlockInds, s.mu.where, s.mu.free))
+		}
+	}
+	if int64(len(s.mu.locks)) != s.sizeInBlocks {
+		panic(fmt.Sprintf("lock table isn't correct size: %v %v", len(s.mu.locks), s.sizeInBlocks))
+	}
+	for _, ls := range s.mu.locks {
+		if ls < writeLockTaken {
+			panic(fmt.Sprintf("lock state %v is not allowed: %v", ls, s.mu.locks))
+		}
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go

@@ -15,6 +15,8 @@ import (
 	"github.com/cockroachdb/pebble/vfs"
 )
 
+const fileMaxReadaheadSize = 256 * 1024 /* 256KB */
+
 // fileReadable implements objstorage.Readable on top of a vfs.File.
 //
 // The implementation might use Prealloc and might reopen the file with
@@ -75,7 +77,7 @@ func (r *fileReadable) Size() int64 {
 func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
 	rh := readHandlePool.Get().(*vfsReadHandle)
 	rh.r = r
-	rh.rs = makeReadaheadState()
+	rh.rs = makeReadaheadState(fileMaxReadaheadSize)
 	return rh
 }
 
@@ -126,10 +128,10 @@ func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error
 		n, err = rh.sequentialFile.ReadAt(p, offset)
 	} else {
 		if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
-			if readaheadSize >= maxReadaheadSize {
+			if readaheadSize >= fileMaxReadaheadSize {
 				// We've reached the maximum readahead size. Beyond this point, rely on
 				// OS-level readahead.
-				rh.MaxReadahead()
+				rh.switchToOSReadahead()
 			} else {
 				_ = rh.r.file.Prefetch(offset, readaheadSize)
 			}
@@ -142,8 +144,12 @@ func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error
 	return err
 }
 
-// MaxReadahead is part of the objstorage.ReadHandle interface.
-func (rh *vfsReadHandle) MaxReadahead() {
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) SetupForCompaction() {
+	rh.switchToOSReadahead()
+}
+
+func (rh *vfsReadHandle) switchToOSReadahead() {
 	if rh.sequentialFile != nil {
 		return
 	}

vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go

@@ -66,8 +66,11 @@ type Storage interface {
 type ObjectReader interface {
 	// ReadAt reads len(p) bytes into p starting at offset off.
 	//
-	// Clients of ReadAt can execute parallel ReadAt calls on the
-	// same ObjectReader.
+	// Does not return partial results; if offset + len(p) is past the end of the
+	// object, an error is returned.
+	//
+	// Clients of ReadAt can execute parallel ReadAt calls on the same
+	// ObjectReader.
 	ReadAt(ctx context.Context, p []byte, offset int64) error
 
 	Close() error

vendor/github.com/cockroachdb/pebble/open.go

@@ -210,8 +210,8 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		write:         d.commitWrite,
 	})
 	d.deletionLimiter = rate.NewLimiter(
-		rate.Limit(d.opts.Experimental.MinDeletionRate),
-		d.opts.Experimental.MinDeletionRate)
+		rate.Limit(d.opts.TargetByteDeletionRate),
+		d.opts.TargetByteDeletionRate)
 	d.mu.nextJobID = 1
 	d.mu.mem.nextSize = opts.MemTableSize
 	if d.mu.mem.nextSize > initialMemTableSize {
@@ -512,7 +512,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 	}
 	d.mu.tableStats.cond.L = &d.mu.Mutex
 	d.mu.tableValidation.cond.L = &d.mu.Mutex
-	if !d.opts.ReadOnly && !d.opts.private.disableTableStats {
+	if !d.opts.ReadOnly {
 		d.maybeCollectTableStatsLocked()
 	}
 	d.calculateDiskAvailableBytes()
@@ -804,13 +804,12 @@ func (d *DB) replayWAL(
 					paths[i] = base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
 				}
 
-				var meta []*manifest.FileMetadata
-				meta, _, err = ingestLoad(
-					d.opts, d.mu.formatVers.vers, paths, d.cacheID, fileNums,
-				)
+				var lr ingestLoadResult
+				lr, err = ingestLoad(d.opts, d.mu.formatVers.vers, paths, nil, d.cacheID, fileNums)
 				if err != nil {
 					return nil, 0, err
 				}
+				meta := lr.localMeta
 
 				if uint32(len(meta)) != b.Count() {
 					panic("pebble: couldn't load all files in WAL entry.")

vendor/github.com/cockroachdb/pebble/options.go

@@ -17,6 +17,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/objstorage/shared"
 	"github.com/cockroachdb/pebble/sstable"
@@ -118,10 +119,13 @@ type IterOptions struct {
 	// function can be used by multiple iterators, if the iterator is cloned.
 	TableFilter func(userProps map[string]string) bool
 	// PointKeyFilters can be used to avoid scanning tables and blocks in tables
-	// when iterating over point keys. It is requires that this slice is sorted in
-	// increasing order of the BlockPropertyFilter.ShortID. This slice represents
-	// an intersection across all filters, i.e., all filters must indicate that the
-	// block is relevant.
+	// when iterating over point keys. This slice represents an intersection
+	// across all filters, i.e., all filters must indicate that the block is
+	// relevant.
+	//
+	// Performance note: When len(PointKeyFilters) > 0, the caller should ensure
+	// that cap(PointKeyFilters) is at least len(PointKeyFilters)+1. This helps
+	// avoid allocations in Pebble internal code that mutates the slice.
 	PointKeyFilters []BlockPropertyFilter
 	// RangeKeyFilters can be usefd to avoid scanning tables and blocks in tables
 	// when iterating over range keys. The same requirements that apply to
@@ -180,6 +184,10 @@ type IterOptions struct {
 	level manifest.Level
 	// disableLazyCombinedIteration is an internal testing option.
 	disableLazyCombinedIteration bool
+	// snapshotForHideObsoletePoints is specified for/by levelIter when opening
+	// files and is used to decide whether to hide obsolete points. A value of 0
+	// implies obsolete points should not be hidden.
+	snapshotForHideObsoletePoints uint64
 
 	// NB: If adding new Options, you must account for them in iterator
 	// construction and Iterator.SetOptions.
@@ -222,6 +230,17 @@ func (o *IterOptions) getLogger() Logger {
 	return o.logger
 }
 
+// SpanIterOptions creates a SpanIterOptions from this IterOptions.
+func (o *IterOptions) SpanIterOptions(level manifest.Level) keyspan.SpanIterOptions {
+	if o == nil {
+		return keyspan.SpanIterOptions{Level: level}
+	}
+	return keyspan.SpanIterOptions{
+		RangeKeyFilters: o.RangeKeyFilters,
+		Level:           level,
+	}
+}
+
 // scanInternalOptions is similar to IterOptions, meant for use with
 // scanInternalIterator.
 type scanInternalOptions struct {
@@ -509,17 +528,6 @@ type Options struct {
 		// concurrency slots as determined by the two options is chosen.
 		CompactionDebtConcurrency int
 
-		// MinDeletionRate is the minimum number of bytes per second that would
-		// be deleted. Deletion pacing is used to slow down deletions when
-		// compactions finish up or readers close, and newly-obsolete files need
-		// cleaning up. Deleting lots of files at once can cause disk latency to
-		// go up on some SSDs, which this functionality guards against. This is a
-		// minimum as the maximum is theoretically unlimited; pacing is disabled
-		// when there are too many obsolete files relative to live bytes, or
-		// there isn't enough disk space available. Setting this to 0 disables
-		// deletion pacing, which is also the default.
-		MinDeletionRate int
-
 		// ReadCompactionRate controls the frequency of read triggered
 		// compactions by adjusting `AllowedSeeks` in manifest.FileMetadata:
 		//
@@ -606,12 +614,6 @@ type Options struct {
 		// for CPUWorkPermissionGranter for more details.
 		CPUWorkPermissionGranter CPUWorkPermissionGranter
 
-		// PointTombstoneWeight is a float in the range [0, +inf) used to weight the
-		// point tombstone heuristics during compaction picking.
-		//
-		// The default value is 1, which results in no scaling of point tombstones.
-		PointTombstoneWeight float64
-
 		// EnableValueBlocks is used to decide whether to enable writing
 		// TableFormatPebblev3 sstables. This setting is only respected by a
 		// specific subset of format major versions: FormatSSTableValueBlocks,
@@ -626,18 +628,24 @@ type Options struct {
 		ShortAttributeExtractor ShortAttributeExtractor
 
 		// RequiredInPlaceValueBound specifies an optional span of user key
-		// prefixes for which the values must be stored with the key. This is
-		// useful for statically known exclusions to value separation. In
-		// CockroachDB, this will be used for the lock table key space that has
-		// non-empty suffixes, but those locks don't represent actual MVCC
-		// versions (the suffix ordering is arbitrary). We will also need to add
-		// support for dynamically configured exclusions (we want the default to
-		// be to allow Pebble to decide whether to separate the value or not,
-		// hence this is structured as exclusions), for example, for users of
-		// CockroachDB to dynamically exclude certain tables.
+		// prefixes that are not-MVCC, but have a suffix. For these the values
+		// must be stored with the key, since the concept of "older versions" is
+		// not defined. It is also useful for statically known exclusions to value
+		// separation. In CockroachDB, this will be used for the lock table key
+		// space that has non-empty suffixes, but those locks don't represent
+		// actual MVCC versions (the suffix ordering is arbitrary). We will also
+		// need to add support for dynamically configured exclusions (we want the
+		// default to be to allow Pebble to decide whether to separate the value
+		// or not, hence this is structured as exclusions), for example, for users
+		// of CockroachDB to dynamically exclude certain tables.
 		//
 		// Any change in exclusion behavior takes effect only on future written
 		// sstables, and does not start rewriting existing sstables.
+		//
+		// Even ignoring changes in this setting, exclusions are interpreted as a
+		// guidance by Pebble, and not necessarily honored. Specifically, user
+		// keys with multiple Pebble-versions *may* have the older versions stored
+		// in value blocks.
 		RequiredInPlaceValueBound UserKeyPrefixBound
 
 		// DisableIngestAsFlushable disables lazy ingestion of sstables through
@@ -858,6 +866,19 @@ type Options struct {
 	// changing options dynamically?
 	WALMinSyncInterval func() time.Duration
 
+	// TargetByteDeletionRate is the rate (in bytes per second) at which sstable file
+	// deletions are limited to (under normal circumstances).
+	//
+	// Deletion pacing is used to slow down deletions when compactions finish up
+	// or readers close and newly-obsolete files need cleaning up. Deleting lots
+	// of files at once can cause disk latency to go up on some SSDs, which this
+	// functionality guards against. This is only a best-effort target; pacing is
+	// disabled when there are too many obsolete files relative to live bytes, or
+	// there isn't enough disk space available.
+	//
+	// Setting this to 0 disables deletion pacing, which is also the default.
+	TargetByteDeletionRate int
+
 	// private options are only used by internal tests or are used internally
 	// for facilitating upgrade paths of unconfigurable functionality.
 	private struct {
@@ -1043,10 +1064,6 @@ func (o *Options) EnsureDefaults() *Options {
 	if o.Experimental.CPUWorkPermissionGranter == nil {
 		o.Experimental.CPUWorkPermissionGranter = defaultCPUWorkGranter{}
 	}
-	if o.Experimental.PointTombstoneWeight == 0 {
-		o.Experimental.PointTombstoneWeight = 1
-	}
-
 	if o.Experimental.MultiLevelCompactionHueristic == nil {
 		o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{}
 	}
@@ -1167,9 +1184,8 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  max_open_files=%d\n", o.MaxOpenFiles)
 	fmt.Fprintf(&buf, "  mem_table_size=%d\n", o.MemTableSize)
 	fmt.Fprintf(&buf, "  mem_table_stop_writes_threshold=%d\n", o.MemTableStopWritesThreshold)
-	fmt.Fprintf(&buf, "  min_deletion_rate=%d\n", o.Experimental.MinDeletionRate)
+	fmt.Fprintf(&buf, "  min_deletion_rate=%d\n", o.TargetByteDeletionRate)
 	fmt.Fprintf(&buf, "  merger=%s\n", o.Merger.Name)
-	fmt.Fprintf(&buf, "  point_tombstone_weight=%f\n", o.Experimental.PointTombstoneWeight)
 	fmt.Fprintf(&buf, "  read_compaction_rate=%d\n", o.Experimental.ReadCompactionRate)
 	fmt.Fprintf(&buf, "  read_sampling_multiplier=%d\n", o.Experimental.ReadSamplingMultiplier)
 	fmt.Fprintf(&buf, "  strict_wal_tail=%t\n", o.private.strictWALTail)
@@ -1374,7 +1390,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				// version is valid right here.
 				var v uint64
 				v, err = strconv.ParseUint(value, 10, 64)
-				if vers := FormatMajorVersion(v); vers > FormatNewest || vers == FormatDefault {
+				if vers := FormatMajorVersion(v); vers > internalFormatNewest || vers == FormatDefault {
 					err = errors.Newf("unknown format major version %d", o.FormatMajorVersion)
 				}
 				if err == nil {
@@ -1414,12 +1430,12 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				// Do nothing; option existed in older versions of pebble, and
 				// may be meaningful again eventually.
 			case "min_deletion_rate":
-				o.Experimental.MinDeletionRate, err = strconv.Atoi(value)
+				o.TargetByteDeletionRate, err = strconv.Atoi(value)
 			case "min_flush_rate":
 				// Do nothing; option existed in older versions of pebble, and
 				// may be meaningful again eventually.
 			case "point_tombstone_weight":
-				o.Experimental.PointTombstoneWeight, err = strconv.ParseFloat(value, 64)
+				// Do nothing; deprecated.
 			case "strict_wal_tail":
 				o.private.strictWALTail, err = strconv.ParseBool(value)
 			case "merger":
@@ -1594,9 +1610,9 @@ func (o *Options) Validate() error {
 		fmt.Fprintf(&buf, "MemTableStopWritesThreshold (%d) must be >= 2\n",
 			o.MemTableStopWritesThreshold)
 	}
-	if o.FormatMajorVersion > FormatNewest {
+	if o.FormatMajorVersion > internalFormatNewest {
 		fmt.Fprintf(&buf, "FormatMajorVersion (%d) must be <= %d\n",
-			o.FormatMajorVersion, FormatNewest)
+			o.FormatMajorVersion, internalFormatNewest)
 	}
 	if o.TableCache != nil && o.Cache != o.TableCache.cache {
 		fmt.Fprintf(&buf, "underlying cache in the TableCache and the Cache dont match\n")
@@ -1616,6 +1632,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions {
 		readerOpts.Comparer = o.Comparer
 		readerOpts.Filters = o.Filters
 		if o.Merger != nil {
+			readerOpts.Merge = o.Merger.Merge
 			readerOpts.MergerName = o.Merger.Name
 		}
 		readerOpts.LoggerAndTracer = o.LoggerAndTracer
@@ -1640,6 +1657,9 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab
 	if format >= sstable.TableFormatPebblev3 {
 		writerOpts.ShortAttributeExtractor = o.Experimental.ShortAttributeExtractor
 		writerOpts.RequiredInPlaceValueBound = o.Experimental.RequiredInPlaceValueBound
+		if format >= sstable.TableFormatPebblev4 && level == numLevels-1 {
+			writerOpts.WritingToLowestLevel = true
+		}
 	}
 	levelOpts := o.Level(level)
 	writerOpts.BlockRestartInterval = levelOpts.BlockRestartInterval

vendor/github.com/cockroachdb/pebble/pacer.go

@@ -106,7 +106,7 @@ func (p *deletionPacer) limit(amount uint64, info deletionPacerInfo) error {
 }
 
 // maybeThrottle slows down a deletion of this file if it's faster than
-// opts.Experimental.MinDeletionRate.
+// opts.TargetByteDeletionRate.
 func (p *deletionPacer) maybeThrottle(bytesToDelete uint64) error {
 	return p.limit(bytesToDelete, p.getInfo())
 }

vendor/github.com/cockroachdb/pebble/range_keys.go

@@ -63,8 +63,7 @@ func (i *Iterator) constructRangeKeyIter() {
 	// around Key Trailer order.
 	iter := current.RangeKeyLevels[0].Iter()
 	for f := iter.Last(); f != nil; f = iter.Prev() {
-		spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
-		spanIter, err := i.newIterRangeKey(f, spanIterOpts)
+		spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions(manifest.Level(0)))
 		if err != nil {
 			i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
 			continue
@@ -78,7 +77,7 @@ func (i *Iterator) constructRangeKeyIter() {
 			continue
 		}
 		li := i.rangeKey.iterConfig.NewLevelIter()
-		spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
+		spanIterOpts := i.opts.SpanIterOptions(manifest.Level(level))
 		li.Init(spanIterOpts, i.cmp, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
 			manifest.Level(level), manifest.KeyTypeRange)
 		i.rangeKey.iterConfig.AddLevel(li)

vendor/github.com/cockroachdb/pebble/scan_internal.go

@@ -7,12 +7,15 @@ package pebble
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/sstable"
 )
 
 const (
@@ -92,8 +95,41 @@ type pcIterPos int
 const (
 	pcIterPosCur pcIterPos = iota
 	pcIterPosNext
+	pcIterPosPrev
 )
 
+// pointCollapsingSSTIterator implements sstable.Iterator while composing
+// pointCollapsingIterator.
+type pointCollapsingSSTIterator struct {
+	pointCollapsingIterator
+	childIter sstable.Iterator
+}
+
+var pcSSTIterPool = sync.Pool{
+	New: func() interface{} {
+		return &pointCollapsingSSTIterator{}
+	},
+}
+
+// MaybeFilteredKeys implements the sstable.Iterator interface.
+func (p *pointCollapsingSSTIterator) MaybeFilteredKeys() bool {
+	return p.childIter.MaybeFilteredKeys()
+}
+
+// SetCloseHook implements the sstable.Iterator interface.
+func (p *pointCollapsingSSTIterator) SetCloseHook(fn func(i sstable.Iterator) error) {
+	p.childIter.SetCloseHook(fn)
+}
+
+// Close implements the sstable.Iterator interface.
+func (p *pointCollapsingSSTIterator) Close() error {
+	err := p.pointCollapsingIterator.Close()
+	p.pointCollapsingIterator = pointCollapsingIterator{}
+	p.childIter = nil
+	pcSSTIterPool.Put(p)
+	return err
+}
+
 // pointCollapsingIterator is an internalIterator that collapses point keys and
 // returns at most one point internal key for each user key. Merges are merged,
 // sets are emitted as-is, and SingleDeletes are collapsed with the next point
@@ -112,32 +148,52 @@ type pointCollapsingIterator struct {
 	merge    base.Merge
 	err      error
 	seqNum   uint64
-	// The current position of `iter`. Could be backed by savedKey (i.e. iterKey ==
-	// &savedKey) or could be owned by `iter`. findNextEntry and similar methods
-	// are expected to save the current value of this to savedKey if they're
-	// iterating away from the current key but still need to retain it. See
-	// comments in findNextEntry on how this field is used.
+	// The current position of `iter`. Always owned by the underlying iter.
+	iterKey *InternalKey
+	// The last saved key. findNextEntry and similar methods are expected to save
+	// the current value of iterKey to savedKey if they're iterating away from the
+	// current key but still need to retain it. See comments in findNextEntry on
+	// how this field is used.
 	//
 	// At the end of a positioning call:
 	//  - if pos == pcIterPosNext, iterKey is pointing to the next user key owned
 	//    by `iter` while savedKey is holding a copy to our current key.
-	//  - If pos == pcIterPosCur, iterKey is pointing to either a savedKey-backed
-	//    copy of the current key, or an `iter`-owned current key in which case
-	//    the value of savedKey is undefined.
-	iterKey  *InternalKey
-	savedKey InternalKey
+	//  - If pos == pcIterPosCur, iterKey is pointing to an `iter`-owned current
+	//    key, and savedKey is either undefined or pointing to a version of the
+	//    current key owned by this iterator (i.e. backed by savedKeyBuf).
+	//  - if pos == pcIterPosPrev, iterKey is pointing to the key before
+	//    p.savedKey. p.savedKey is treated as the current key and is owned by
+	//    this iterator, while p.iterKey is the previous key that is the current
+	//    position of the child iterator.
+	savedKey    InternalKey
+	savedKeyBuf []byte
+	// elideRangeDeletes ignores range deletes returned by the interleaving
+	// iterator if true.
+	elideRangeDeletes bool
 	// Value at the current iterator position, at iterKey.
-	value base.LazyValue
+	iterValue base.LazyValue
+	// Saved value backed by valueBuf, if set. Used in reverse iteration, and
+	// for merges.
+	savedValue base.LazyValue
 	// Used for Merge keys only.
 	valueMerger ValueMerger
 	valueBuf    []byte
+	// If fixedSeqNum is non-zero, all emitted points are verified to have this
+	// fixed sequence number.
+	fixedSeqNum uint64
 }
 
 // SeekPrefixGE implements the InternalIterator interface.
 func (p *pointCollapsingIterator) SeekPrefixGE(
 	prefix, key []byte, flags base.SeekGEFlags,
 ) (*base.InternalKey, base.LazyValue) {
-	panic("unimplemented")
+	p.resetKey()
+	p.iterKey, p.iterValue = p.iter.SeekPrefixGE(prefix, key, flags)
+	p.pos = pcIterPosCur
+	if p.iterKey == nil {
+		return nil, base.LazyValue{}
+	}
+	return p.findNextEntry()
 }
 
 // SeekGE implements the InternalIterator interface.
@@ -145,7 +201,7 @@ func (p *pointCollapsingIterator) SeekGE(
 	key []byte, flags base.SeekGEFlags,
 ) (*base.InternalKey, base.LazyValue) {
 	p.resetKey()
-	p.iterKey, p.value = p.iter.SeekGE(key, flags)
+	p.iterKey, p.iterValue = p.iter.SeekGE(key, flags)
 	p.pos = pcIterPosCur
 	if p.iterKey == nil {
 		return nil, base.LazyValue{}
@@ -157,11 +213,17 @@ func (p *pointCollapsingIterator) SeekGE(
 func (p *pointCollapsingIterator) SeekLT(
 	key []byte, flags base.SeekLTFlags,
 ) (*base.InternalKey, base.LazyValue) {
-	panic("unimplemented")
+	p.resetKey()
+	p.iterKey, p.iterValue = p.iter.SeekLT(key, flags)
+	p.pos = pcIterPosCur
+	if p.iterKey == nil {
+		return nil, base.LazyValue{}
+	}
+	return p.findPrevEntry()
 }
 
 func (p *pointCollapsingIterator) resetKey() {
-	p.savedKey.UserKey = p.savedKey.UserKey[:0]
+	p.savedKey.UserKey = p.savedKeyBuf[:0]
 	p.savedKey.Trailer = 0
 	p.valueMerger = nil
 	p.valueBuf = p.valueBuf[:0]
@@ -169,59 +231,73 @@ func (p *pointCollapsingIterator) resetKey() {
 	p.pos = pcIterPosCur
 }
 
-// findNextEntry is called to return the next key. p.iter must be positioned at
-// the start of the first user key we are interested in.
-func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyValue) {
-	finishAndReturnMerge := func() (*base.InternalKey, base.LazyValue) {
-		value, closer, err := p.valueMerger.Finish(true /* includesBase */)
-		if err != nil {
-			p.err = err
-			return nil, base.LazyValue{}
-		}
-		p.valueBuf = append(p.valueBuf[:0], value...)
-		if closer != nil {
-			_ = closer.Close()
-		}
-		p.valueMerger = nil
-		newValue := base.MakeInPlaceValue(value)
-		return &p.savedKey, newValue
+func (p *pointCollapsingIterator) verifySeqNum(key *base.InternalKey) *base.InternalKey {
+	if !invariants.Enabled {
+		return key
 	}
+	if p.fixedSeqNum == 0 || key == nil || key.Kind() == InternalKeyKindRangeDelete {
+		return key
+	}
+	if key.SeqNum() != p.fixedSeqNum {
+		panic(fmt.Sprintf("expected foreign point key to have seqnum %d, got %d", p.fixedSeqNum, key.SeqNum()))
+	}
+	return key
+}
 
-	// saveKey sets p.iterKey (if not-nil) to &p.savedKey. We can use this equality
-	// as a proxy to determine if we're at the first internal key for a user key.
+// finishAndReturnMerge finishes off the valueMerger and returns the saved key.
+func (p *pointCollapsingIterator) finishAndReturnMerge() (*base.InternalKey, base.LazyValue) {
+	value, closer, err := p.valueMerger.Finish(true /* includesBase */)
+	if err != nil {
+		p.err = err
+		return nil, base.LazyValue{}
+	}
+	p.valueBuf = append(p.valueBuf[:0], value...)
+	if closer != nil {
+		_ = closer.Close()
+	}
+	p.valueMerger = nil
+	val := base.MakeInPlaceValue(p.valueBuf)
+	return p.verifySeqNum(&p.savedKey), val
+}
+
+// findNextEntry is called to return the next key. p.iter must be positioned at the
+// start of the first user key we are interested in.
+func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyValue) {
 	p.saveKey()
+	// Saves a comparison in the fast path
+	firstIteration := true
 	for p.iterKey != nil {
-		// NB: p.savedKey is either the current key (iff p.iterKey == &p.savedKey),
+		// NB: p.savedKey is either the current key (iff p.iterKey == firstKey),
 		// or the previous key.
-		if p.iterKey != &p.savedKey && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
+		if !firstIteration && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
 			if p.valueMerger != nil {
 				if p.savedKey.Kind() != InternalKeyKindMerge {
 					panic(fmt.Sprintf("expected key %s to have MERGE kind", p.iterKey))
 				}
 				p.pos = pcIterPosNext
-				return finishAndReturnMerge()
+				return p.finishAndReturnMerge()
 			}
 			p.saveKey()
 			continue
 		}
+		firstIteration = false
 		if s := p.iter.Span(); s != nil && s.CoversAt(p.seqNum, p.iterKey.SeqNum()) {
 			// All future keys for this user key must be deleted.
 			if p.valueMerger != nil {
-				return finishAndReturnMerge()
+				return p.finishAndReturnMerge()
 			} else if p.savedKey.Kind() == InternalKeyKindSingleDelete {
 				panic("cannot process singledel key in point collapsing iterator")
 			}
 			// Fast forward to the next user key.
 			p.saveKey()
-			p.iterKey, p.value = p.iter.Next()
+			p.iterKey, p.iterValue = p.iter.Next()
 			for p.iterKey != nil && p.savedKey.SeqNum() >= p.iterKey.SeqNum() && p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
-				p.iterKey, p.value = p.iter.Next()
+				p.iterKey, p.iterValue = p.iter.Next()
 			}
 			continue
 		}
 		switch p.savedKey.Kind() {
-		case InternalKeyKindSet, InternalKeyKindDelete, InternalKeyKindSetWithDelete:
-			p.saveKey()
+		case InternalKeyKindSet, InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
 			// Note that we return SETs directly, even if they would otherwise get
 			// compacted into a Del to turn into a SetWithDelete. This is a fast
 			// path optimization that can break SINGLEDEL determinism. To lead to
@@ -240,14 +316,14 @@ func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyV
 			// of blocks and can determine user key changes without doing key saves
 			// or comparisons.
 			p.pos = pcIterPosCur
-			return p.iterKey, p.value
+			return p.verifySeqNum(p.iterKey), p.iterValue
 		case InternalKeyKindSingleDelete:
 			// Panic, as this iterator is not expected to observe single deletes.
 			panic("cannot process singledel key in point collapsing iterator")
 		case InternalKeyKindMerge:
 			if p.valueMerger == nil {
 				// Set up merger. This is the first Merge key encountered.
-				value, callerOwned, err := p.value.Value(p.valueBuf[:0])
+				value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
 				if err != nil {
 					p.err = err
 					return nil, base.LazyValue{}
@@ -263,13 +339,13 @@ func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyV
 					return nil, base.LazyValue{}
 				}
 				p.saveKey()
-				p.iterKey, p.value = p.iter.Next()
+				p.iterKey, p.iterValue = p.iter.Next()
 				continue
 			}
 			switch p.iterKey.Kind() {
 			case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
 				// Merge into key.
-				value, callerOwned, err := p.value.Value(p.valueBuf[:0])
+				value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
 				if err != nil {
 					p.err = err
 					return nil, base.LazyValue{}
@@ -287,31 +363,144 @@ func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyV
 			if p.iterKey.Kind() != InternalKeyKindMerge {
 				p.savedKey.SetKind(p.iterKey.Kind())
 				p.pos = pcIterPosCur
-				return finishAndReturnMerge()
+				return p.finishAndReturnMerge()
 			}
-			p.iterKey, p.value = p.iter.Next()
+			p.iterKey, p.iterValue = p.iter.Next()
 		case InternalKeyKindRangeDelete:
+			if p.elideRangeDeletes {
+				// Skip this range delete, and process any point after it.
+				p.iterKey, p.iterValue = p.iter.Next()
+				p.saveKey()
+				continue
+			}
 			// These are interleaved by the interleaving iterator ahead of all points.
 			// We should pass them as-is, but also account for any points ahead of
 			// them.
 			p.pos = pcIterPosCur
-			return p.iterKey, p.value
+			return p.verifySeqNum(p.iterKey), p.iterValue
 		default:
 			panic(fmt.Sprintf("unexpected kind: %d", p.iterKey.Kind()))
 		}
 	}
 	if p.valueMerger != nil {
 		p.pos = pcIterPosNext
-		return finishAndReturnMerge()
+		return p.finishAndReturnMerge()
 	}
 	p.resetKey()
 	return nil, base.LazyValue{}
 }
 
+// findPrevEntry finds the relevant point key to return for the previous user key
+// (i.e. in reverse iteration). Requires that the iterator is already positioned
+// at the first-in-reverse (i.e. rightmost / largest) internal key encountered
+// for that user key.
+func (p *pointCollapsingIterator) findPrevEntry() (*base.InternalKey, base.LazyValue) {
+	if p.iterKey == nil {
+		p.pos = pcIterPosCur
+		return nil, base.LazyValue{}
+	}
+
+	p.saveKey()
+	firstIteration := true
+	for p.iterKey != nil {
+		if !firstIteration && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
+			p.pos = pcIterPosPrev
+			return p.verifySeqNum(&p.savedKey), p.savedValue
+		}
+		firstIteration = false
+		if s := p.iter.Span(); s != nil && s.CoversAt(p.seqNum, p.iterKey.SeqNum()) {
+			// Skip this key.
+			p.iterKey, p.iterValue = p.iter.Prev()
+			p.saveKey()
+			continue
+		}
+		switch p.iterKey.Kind() {
+		case InternalKeyKindSet, InternalKeyKindDelete, InternalKeyKindSetWithDelete:
+			// Instead of calling saveKey(), we take advantage of the invariant that
+			// p.savedKey.UserKey == p.iterKey.UserKey (otherwise we'd have gone into
+			// the !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) case at
+			// the top of this for loop). That allows us to just save the trailer
+			// and move on.
+			if invariants.Enabled && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
+				panic("unexpected inequality between p.iterKey and p.savedKey")
+			}
+			p.savedKey.Trailer = p.iterKey.Trailer
+			// Copy value into p.savedValue.
+			value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
+			if err != nil {
+				p.err = err
+				return nil, base.LazyValue{}
+			}
+			if !callerOwned {
+				p.valueBuf = append(p.valueBuf[:0], value...)
+			} else {
+				p.valueBuf = value
+			}
+			p.valueMerger = nil
+			p.savedValue = base.MakeInPlaceValue(p.valueBuf)
+			p.iterKey, p.iterValue = p.iter.Prev()
+			continue
+		case InternalKeyKindSingleDelete:
+			// Panic, as this iterator is not expected to observe single deletes.
+			panic("cannot process singledel key in point collapsing iterator")
+		case InternalKeyKindMerge:
+			panic("cannot process merge key in point collapsing iterator in reverse iteration")
+		case InternalKeyKindRangeDelete:
+			if p.elideRangeDeletes {
+				// Skip this range delete, and process any point before it.
+				p.iterKey, p.iterValue = p.iter.Prev()
+				continue
+			}
+			// These are interleaved by the interleaving iterator behind all points.
+			if p.savedKey.Kind() != InternalKeyKindRangeDelete {
+				// If the previous key was not a rangedel, we need to return it. Pretend that we're at the
+				// previous user key (i.e. with p.pos = pcIterPosPrev) even if we're not, so on the next
+				// Prev() we encounter and return this rangedel. For now return the point ahead of
+				// this range del (if any).
+				p.pos = pcIterPosPrev
+				return p.verifySeqNum(&p.savedKey), p.savedValue
+			}
+			// We take advantage of the fact that a Prev() *on* a RangeDel iterKey
+			// always takes us to a different user key, so on the next iteration
+			// we will fall into the !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey)
+			// case.
+			//
+			// Instead of calling saveKey(), we take advantage of the invariant that
+			// p.savedKey.UserKey == p.iterKey.UserKey (otherwise we'd have gone into
+			// the !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) case at
+			// the top of this for loop). That allows us to just save the trailer
+			// and move on.
+			if invariants.Enabled && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
+				panic("unexpected inequality between p.iterKey and p.savedKey")
+			}
+			p.savedKey.Trailer = p.iterKey.Trailer
+			// Copy value into p.savedValue.
+			value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
+			if err != nil {
+				p.err = err
+				return nil, base.LazyValue{}
+			}
+			if !callerOwned {
+				p.valueBuf = append(p.valueBuf[:0], value...)
+			} else {
+				p.valueBuf = value
+			}
+			p.valueMerger = nil
+			p.savedValue = base.MakeInPlaceValue(p.valueBuf)
+			p.iterKey, p.iterValue = p.iter.Prev()
+			continue
+		default:
+			panic(fmt.Sprintf("unexpected kind: %d", p.iterKey.Kind()))
+		}
+	}
+	p.pos = pcIterPosPrev
+	return p.verifySeqNum(&p.savedKey), p.savedValue
+}
+
 // First implements the InternalIterator interface.
 func (p *pointCollapsingIterator) First() (*base.InternalKey, base.LazyValue) {
 	p.resetKey()
-	p.iterKey, p.value = p.iter.First()
+	p.iterKey, p.iterValue = p.iter.First()
 	p.pos = pcIterPosCur
 	if p.iterKey == nil {
 		return nil, base.LazyValue{}
@@ -321,41 +510,67 @@ func (p *pointCollapsingIterator) First() (*base.InternalKey, base.LazyValue) {
 
 // Last implements the InternalIterator interface.
 func (p *pointCollapsingIterator) Last() (*base.InternalKey, base.LazyValue) {
-	panic("unimplemented")
+	p.resetKey()
+	p.iterKey, p.iterValue = p.iter.Last()
+	p.pos = pcIterPosCur
+	if p.iterKey == nil {
+		return nil, base.LazyValue{}
+	}
+	return p.findPrevEntry()
 }
 
 func (p *pointCollapsingIterator) saveKey() {
 	if p.iterKey == nil {
-		p.savedKey = InternalKey{UserKey: p.savedKey.UserKey[:0]}
-		return
-	} else if p.iterKey == &p.savedKey {
-		// Do nothing.
+		p.savedKey = InternalKey{UserKey: p.savedKeyBuf[:0]}
 		return
 	}
-	p.savedKey.CopyFrom(*p.iterKey)
-	p.iterKey = &p.savedKey
+	p.savedKeyBuf = append(p.savedKeyBuf[:0], p.iterKey.UserKey...)
+	p.savedKey = InternalKey{UserKey: p.savedKeyBuf, Trailer: p.iterKey.Trailer}
 }
 
 // Next implements the InternalIterator interface.
 func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) {
 	switch p.pos {
+	case pcIterPosPrev:
+		p.saveKey()
+		if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
+			p.iterKey, p.iterValue = p.iter.Next()
+			p.pos = pcIterPosCur
+		} else {
+			// Fast forward to the next user key.
+			key, val := p.iter.Next()
+			// p.iterKey.SeqNum() >= key.SeqNum() is an optimization that allows us to
+			// use p.iterKey.SeqNum() < key.SeqNum() as a sign that the user key has
+			// changed, without needing to do the full key comparison.
+			for key != nil && p.savedKey.SeqNum() >= key.SeqNum() &&
+				p.comparer.Equal(p.savedKey.UserKey, key.UserKey) {
+				key, val = p.iter.Next()
+			}
+			if key == nil {
+				// There are no keys to return.
+				p.resetKey()
+				return nil, base.LazyValue{}
+			}
+			p.iterKey, p.iterValue = key, val
+			p.pos = pcIterPosCur
+		}
+		fallthrough
 	case pcIterPosCur:
 		p.saveKey()
-		if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete {
+		if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
 			// Step over the interleaved range delete and process the very next
 			// internal key, even if it's at the same user key. This is because a
 			// point for that user key has not been returned yet.
-			p.iterKey, p.value = p.iter.Next()
+			p.iterKey, p.iterValue = p.iter.Next()
 			break
 		}
-		// Fast forward to the next user key. p.iterKey stayed at p.savedKey for
-		// this loop.
+		// Fast forward to the next user key.
 		key, val := p.iter.Next()
 		// p.iterKey.SeqNum() >= key.SeqNum() is an optimization that allows us to
 		// use p.iterKey.SeqNum() < key.SeqNum() as a sign that the user key has
 		// changed, without needing to do the full key comparison.
-		for p.iterKey != nil && key != nil && p.iterKey.SeqNum() >= key.SeqNum() &&
-			p.comparer.Equal(p.iterKey.UserKey, key.UserKey) {
+		for key != nil && p.savedKey.SeqNum() >= key.SeqNum() &&
+			p.comparer.Equal(p.savedKey.UserKey, key.UserKey) {
 			key, val = p.iter.Next()
 		}
 		if key == nil {
@@ -363,7 +578,7 @@ func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) {
 			p.resetKey()
 			return nil, base.LazyValue{}
 		}
-		p.iterKey, p.value = key, val
+		p.iterKey, p.iterValue = key, val
 	case pcIterPosNext:
 		p.pos = pcIterPosCur
 	}
@@ -376,12 +591,56 @@ func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) {
 
 // NextPrefix implements the InternalIterator interface.
 func (p *pointCollapsingIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	// TODO(bilal): Implement this. It'll be similar to SeekGE, except we'll call
+	// the child iterator's NextPrefix, and have some special logic in case pos
+	// is pcIterPosNext.
 	panic("unimplemented")
 }
 
 // Prev implements the InternalIterator interface.
 func (p *pointCollapsingIterator) Prev() (*base.InternalKey, base.LazyValue) {
-	panic("unimplemented")
+	switch p.pos {
+	case pcIterPosNext:
+		// Rewind backwards to the previous iter key.
+		p.saveKey()
+		key, val := p.iter.Prev()
+		for key != nil && p.savedKey.SeqNum() <= key.SeqNum() &&
+			p.comparer.Equal(p.savedKey.UserKey, key.UserKey) {
+			if key.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
+				// We need to pause at this range delete and return it as-is, as "cur"
+				// is referencing the point key after it, not the range delete.
+				break
+			}
+			key, val = p.iter.Prev()
+		}
+		p.iterKey = key
+		p.iterValue = val
+		p.pos = pcIterPosCur
+		fallthrough
+	case pcIterPosCur:
+		p.saveKey()
+		key, val := p.iter.Prev()
+		for key != nil && p.savedKey.SeqNum() <= key.SeqNum() &&
+			p.comparer.Equal(p.savedKey.UserKey, key.UserKey) {
+			if key.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
+				// We need to pause at this range delete and return it as-is, as "cur"
+				// is referencing the point key after it, not the range delete.
+				break
+			}
+			key, val = p.iter.Prev()
+		}
+		p.iterKey = key
+		p.iterValue = val
+		p.pos = pcIterPosCur
+	case pcIterPosPrev:
+		// Do nothing.
+		p.pos = pcIterPosCur
+	}
+	if p.iterKey == nil {
+		p.resetKey()
+		return nil, base.LazyValue{}
+	}
+	return p.findPrevEntry()
 }
 
 // Error implements the InternalIterator interface.
@@ -477,10 +736,6 @@ func (d *DB) truncateSharedFile(
 
 	// We will need to truncate file bounds in at least one direction. Open all
 	// relevant iterators.
-	//
-	// TODO(bilal): Once virtual sstables go in, verify that the constraining of
-	// bounds to virtual sstable bounds happens below this method, so we aren't
-	// unintentionally exposing keys we shouldn't be exposing.
 	iter, rangeDelIter, err := d.newIters(ctx, file, &IterOptions{
 		LowerBound: lower,
 		UpperBound: upper,
@@ -497,7 +752,7 @@ func (d *DB) truncateSharedFile(
 		)
 		defer rangeDelIter.Close()
 	}
-	rangeKeyIter, err := d.tableNewRangeKeyIter(file, nil /* spanIterOptions */)
+	rangeKeyIter, err := d.tableNewRangeKeyIter(file, keyspan.SpanIterOptions{Level: manifest.Level(level)})
 	if err != nil {
 		return nil, false, err
 	}
@@ -584,6 +839,10 @@ func (d *DB) truncateSharedFile(
 	if len(sst.Smallest.UserKey) == 0 {
 		return nil, true, nil
 	}
+	sst.Size, err = d.tableCache.estimateSize(file, sst.Smallest.UserKey, sst.Largest.UserKey)
+	if err != nil {
+		return nil, false, err
+	}
 	return sst, false, nil
 }
 
@@ -616,7 +875,7 @@ func scanInternalImpl(
 			for f := files.SeekGE(cmp, lower); f != nil && cmp(f.Smallest.UserKey, upper) < 0; f = files.Next() {
 				var objMeta objstorage.ObjectMetadata
 				var err error
-				objMeta, err = provider.Lookup(fileTypeTable, f.FileNum.DiskFileNum())
+				objMeta, err = provider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
 				if err != nil {
 					return err
 				}
@@ -718,6 +977,7 @@ func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *
 	mlevels = mlevels[:numMergingLevels]
 	levels = levels[:numLevelIters]
 	rangeDelLevels = rangeDelLevels[:numLevelIters]
+	i.opts.IterOptions.snapshotForHideObsoletePoints = i.seqNum
 	addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) {
 		li := &levels[levelsIndex]
 		rli := &rangeDelLevels[levelsIndex]
@@ -804,8 +1064,7 @@ func (i *scanInternalIterator) constructRangeKeyIter() {
 	// around Key Trailer order.
 	iter := current.RangeKeyLevels[0].Iter()
 	for f := iter.Last(); f != nil; f = iter.Prev() {
-		spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
-		spanIter, err := i.newIterRangeKey(f, spanIterOpts)
+		spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions(manifest.Level(0)))
 		if err != nil {
 			i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
 			continue
@@ -822,7 +1081,7 @@ func (i *scanInternalIterator) constructRangeKeyIter() {
 			continue
 		}
 		li := i.rangeKey.iterConfig.NewLevelIter()
-		spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
+		spanIterOpts := i.opts.SpanIterOptions(manifest.Level(level))
 		li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
 			manifest.Level(level), manifest.KeyTypeRange)
 		i.rangeKey.iterConfig.AddLevel(li)

vendor/github.com/cockroachdb/pebble/sstable/block.go

@@ -47,7 +47,10 @@ type blockWriter struct {
 	// ensuring that any of the keys with the same prefix can be used to
 	// assemble the full key when the prefix does change.
 	restarts []uint32
-	curKey   []byte
+	// Do not read curKey directly from outside blockWriter since it can have
+	// the InternalKeyKindSSTableInternalObsoleteBit set. Use getCurKey() or
+	// getCurUserKey() instead.
+	curKey []byte
 	// curValue excludes the optional prefix provided to
 	// storeWithOptionalValuePrefix.
 	curValue []byte
@@ -79,6 +82,20 @@ const setHasSameKeyPrefixRestartMask uint32 = 1 << 31
 const restartMaskLittleEndianHighByteWithoutSetHasSamePrefix byte = 0b0111_1111
 const restartMaskLittleEndianHighByteOnlySetHasSamePrefix byte = 0b1000_0000
 
+func (w *blockWriter) getCurKey() InternalKey {
+	k := base.DecodeInternalKey(w.curKey)
+	k.Trailer = k.Trailer & trailerObsoleteMask
+	return k
+}
+
+func (w *blockWriter) getCurUserKey() []byte {
+	n := len(w.curKey) - base.InternalTrailerLen
+	if n < 0 {
+		panic(errors.AssertionFailedf("corrupt key in blockWriter buffer"))
+	}
+	return w.curKey[:n:n]
+}
+
 // If !addValuePrefix, the valuePrefix is ignored.
 func (w *blockWriter) storeWithOptionalValuePrefix(
 	keySize int,
@@ -191,12 +208,21 @@ func (w *blockWriter) storeWithOptionalValuePrefix(
 
 func (w *blockWriter) add(key InternalKey, value []byte) {
 	w.addWithOptionalValuePrefix(
-		key, value, len(key.UserKey), false, 0, false)
+		key, false, value, len(key.UserKey), false, 0, false)
 }
 
 // Callers that always set addValuePrefix to false should use add() instead.
+//
+// isObsolete indicates whether this key-value pair is obsolete in this
+// sstable (only applicable when writing data blocks) -- see the comment in
+// table.go and the longer one in format.go. addValuePrefix adds a 1 byte
+// prefix to the value, specified in valuePrefix -- this is used for data
+// blocks in TableFormatPebblev3 onwards for SETs (see the comment in
+// format.go, with more details in value_block.go). setHasSameKeyPrefix is
+// also used in TableFormatPebblev3 onwards for SETs.
 func (w *blockWriter) addWithOptionalValuePrefix(
 	key InternalKey,
+	isObsolete bool,
 	value []byte,
 	maxSharedKeyLen int,
 	addValuePrefix bool,
@@ -210,6 +236,9 @@ func (w *blockWriter) addWithOptionalValuePrefix(
 		w.curKey = make([]byte, 0, size*2)
 	}
 	w.curKey = w.curKey[:size]
+	if isObsolete {
+		key.Trailer = key.Trailer | trailerObsoleteBit
+	}
 	key.Encode(w.curKey)
 
 	w.storeWithOptionalValuePrefix(
@@ -374,13 +403,14 @@ type blockIter struct {
 	cached      []blockEntry
 	cachedBuf   []byte
 	cacheHandle cache.Handle
-	// The first key in the block. This is used by the caller to set bounds
+	// The first user key in the block. This is used by the caller to set bounds
 	// for block iteration for already loaded blocks.
-	firstKey          InternalKey
+	firstUserKey      []byte
 	lazyValueHandling struct {
 		vbr            *valueBlockReader
 		hasValuePrefix bool
 	}
+	hideObsoletePoints bool
 }
 
 // blockIter implements the base.InternalIterator interface.
@@ -388,14 +418,16 @@ var _ base.InternalIterator = (*blockIter)(nil)
 
 func newBlockIter(cmp Compare, block block) (*blockIter, error) {
 	i := &blockIter{}
-	return i, i.init(cmp, block, 0)
+	return i, i.init(cmp, block, 0, false)
 }
 
 func (i *blockIter) String() string {
 	return "block"
 }
 
-func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error {
+func (i *blockIter) init(
+	cmp Compare, block block, globalSeqNum uint64, hideObsoletePoints bool,
+) error {
 	numRestarts := int32(binary.LittleEndian.Uint32(block[len(block)-4:]))
 	if numRestarts == 0 {
 		return base.CorruptionErrorf("pebble/table: invalid table (block has no restart points)")
@@ -408,6 +440,7 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error {
 	i.data = block
 	i.fullKey = i.fullKey[:0]
 	i.val = nil
+	i.hideObsoletePoints = hideObsoletePoints
 	i.clearCache()
 	if i.restarts > 0 {
 		if err := i.readFirstKey(); err != nil {
@@ -415,15 +448,21 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error {
 		}
 	} else {
 		// Block is empty.
-		i.firstKey = InternalKey{}
+		i.firstUserKey = nil
 	}
 	return nil
 }
 
-func (i *blockIter) initHandle(cmp Compare, block cache.Handle, globalSeqNum uint64) error {
+// NB: two cases of hideObsoletePoints:
+//   - Local sstable iteration: globalSeqNum will be set iff the sstable was
+//     ingested.
+//   - Foreign sstable iteration: globalSeqNum is always set.
+func (i *blockIter) initHandle(
+	cmp Compare, block cache.Handle, globalSeqNum uint64, hideObsoletePoints bool,
+) error {
 	i.cacheHandle.Release()
 	i.cacheHandle = block
-	return i.init(cmp, block.Get(), globalSeqNum)
+	return i.init(cmp, block.Get(), globalSeqNum, hideObsoletePoints)
 }
 
 func (i *blockIter) invalidate() {
@@ -590,24 +629,27 @@ func (i *blockIter) readFirstKey() error {
 	// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
 	// BlockIter benchmarks.
 	if n := len(firstKey) - 8; n >= 0 {
-		i.firstKey.Trailer = binary.LittleEndian.Uint64(firstKey[n:])
-		i.firstKey.UserKey = firstKey[:n:n]
-		if i.globalSeqNum != 0 {
-			i.firstKey.SetSeqNum(i.globalSeqNum)
-		}
+		i.firstUserKey = firstKey[:n:n]
 	} else {
-		i.firstKey.Trailer = uint64(InternalKeyKindInvalid)
-		i.firstKey.UserKey = nil
+		i.firstUserKey = nil
 		return base.CorruptionErrorf("pebble/table: invalid firstKey in block")
 	}
 	return nil
 }
 
-func (i *blockIter) decodeInternalKey(key []byte) {
+// The sstable internal obsolete bit is set when writing a block and unset by
+// blockIter, so no code outside block writing/reading code ever sees it.
+const trailerObsoleteBit = uint64(base.InternalKeyKindSSTableInternalObsoleteBit)
+const trailerObsoleteMask = (InternalKeySeqNumMax << 8) | uint64(base.InternalKeyKindSSTableInternalObsoleteMask)
+
+func (i *blockIter) decodeInternalKey(key []byte) (hiddenPoint bool) {
 	// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
 	// BlockIter benchmarks.
 	if n := len(key) - 8; n >= 0 {
-		i.ikey.Trailer = binary.LittleEndian.Uint64(key[n:])
+		trailer := binary.LittleEndian.Uint64(key[n:])
+		hiddenPoint = i.hideObsoletePoints &&
+			(trailer&trailerObsoleteBit != 0)
+		i.ikey.Trailer = trailer & trailerObsoleteMask
 		i.ikey.UserKey = key[:n:n]
 		if i.globalSeqNum != 0 {
 			i.ikey.SetSeqNum(i.globalSeqNum)
@@ -616,6 +658,7 @@ func (i *blockIter) decodeInternalKey(key []byte) {
 		i.ikey.Trailer = uint64(InternalKeyKindInvalid)
 		i.ikey.UserKey = nil
 	}
+	return hiddenPoint
 }
 
 func (i *blockIter) clearCache() {
@@ -640,13 +683,15 @@ func (i *blockIter) cacheEntry() {
 	i.cachedBuf = append(i.cachedBuf, i.key...)
 }
 
+func (i *blockIter) getFirstUserKey() []byte {
+	return i.firstUserKey
+}
+
 // SeekGE implements internalIterator.SeekGE, as documented in the pebble
 // package.
 func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) {
 	i.clearCache()
 
-	ikey := base.MakeSearchKey(key)
-
 	// Find the index of the smallest restart point whose key is > the key
 	// sought; index will be numRestarts if there is no such restart point.
 	i.offset = 0
@@ -700,23 +745,23 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
 				ptr = unsafe.Pointer(uintptr(ptr) + 5)
 			}
 
-			// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
-			// BlockIter benchmarks.
+			// Manually inlining part of base.DecodeInternalKey provides a 5-10%
+			// speedup on BlockIter benchmarks.
 			s := getBytes(ptr, int(v1))
-			var k InternalKey
+			var k []byte
 			if n := len(s) - 8; n >= 0 {
-				k.Trailer = binary.LittleEndian.Uint64(s[n:])
-				k.UserKey = s[:n:n]
-				// NB: We can't have duplicate keys if the globalSeqNum != 0, so we
-				// leave the seqnum on this key as 0 as it won't affect our search
-				// since ikey has the maximum seqnum.
-			} else {
-				k.Trailer = uint64(InternalKeyKindInvalid)
+				k = s[:n:n]
 			}
+			// Else k is invalid, and left as nil
 
-			if base.InternalCompare(i.cmp, ikey, k) >= 0 {
+			if i.cmp(key, k) > 0 {
+				// The search key is greater than the user key at this restart point.
+				// Search beyond this restart point, since we are trying to find the
+				// first restart point with a user key >= the search key.
 				index = h + 1 // preserves f(i-1) == false
 			} else {
+				// k >= search key, so prune everything after index (since index
+				// satisfies the property we are looking for).
 				upper = h // preserves f(j) == true
 			}
 		}
@@ -724,21 +769,25 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
 		// => answer is index.
 	}
 
-	// Since keys are strictly increasing, if index > 0 then the restart point at
-	// index-1 will be the largest whose key is <= the key sought.  If index ==
-	// 0, then all keys in this block are larger than the key sought, and offset
-	// remains at zero.
+	// index is the first restart point with key >= search key. Define the keys
+	// between a restart point and the next restart point as belonging to that
+	// restart point.
+	//
+	// Since keys are strictly increasing, if index > 0 then the restart point
+	// at index-1 will be the first one that has some keys belonging to it that
+	// could be equal to the search key.  If index == 0, then all keys in this
+	// block are larger than the key sought, and offset remains at zero.
 	if index > 0 {
 		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 	}
 	i.readEntry()
-	i.decodeInternalKey(i.key)
+	hiddenPoint := i.decodeInternalKey(i.key)
 
 	// Iterate from that restart point to somewhere >= the key sought.
 	if !i.valid() {
 		return nil, base.LazyValue{}
 	}
-	if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
+	if !hiddenPoint && i.cmp(i.ikey.UserKey, key) >= 0 {
 		// Initialize i.lazyValue
 		if !i.lazyValueHandling.hasValuePrefix ||
 			base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
@@ -751,7 +800,7 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
 		return &i.ikey, i.lazyValue
 	}
 	for i.Next(); i.valid(); i.Next() {
-		if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
+		if i.cmp(i.ikey.UserKey, key) >= 0 {
 			// i.Next() has already initialized i.lazyValue.
 			return &i.ikey, i.lazyValue
 		}
@@ -773,8 +822,6 @@ func (i *blockIter) SeekPrefixGE(
 func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, base.LazyValue) {
 	i.clearCache()
 
-	ikey := base.MakeSearchKey(key)
-
 	// Find the index of the smallest restart point whose key is >= the key
 	// sought; index will be numRestarts if there is no such restart point.
 	i.offset = 0
@@ -828,23 +875,23 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 				ptr = unsafe.Pointer(uintptr(ptr) + 5)
 			}
 
-			// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
-			// BlockIter benchmarks.
+			// Manually inlining part of base.DecodeInternalKey provides a 5-10%
+			// speedup on BlockIter benchmarks.
 			s := getBytes(ptr, int(v1))
-			var k InternalKey
+			var k []byte
 			if n := len(s) - 8; n >= 0 {
-				k.Trailer = binary.LittleEndian.Uint64(s[n:])
-				k.UserKey = s[:n:n]
-				// NB: We can't have duplicate keys if the globalSeqNum != 0, so we
-				// leave the seqnum on this key as 0 as it won't affect our search
-				// since ikey has the maximum seqnum.
-			} else {
-				k.Trailer = uint64(InternalKeyKindInvalid)
+				k = s[:n:n]
 			}
+			// Else k is invalid, and left as nil
 
-			if base.InternalCompare(i.cmp, ikey, k) > 0 {
+			if i.cmp(key, k) > 0 {
+				// The search key is greater than the user key at this restart point.
+				// Search beyond this restart point, since we are trying to find the
+				// first restart point with a user key >= the search key.
 				index = h + 1 // preserves f(i-1) == false
 			} else {
+				// k >= search key, so prune everything after index (since index
+				// satisfies the property we are looking for).
 				upper = h // preserves f(j) == true
 			}
 		}
@@ -852,8 +899,15 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 		// => answer is index.
 	}
 
-	// Since keys are strictly increasing, if index > 0 then the restart point at
-	// index-1 will be the largest whose key is < the key sought.
+	// index is the first restart point with key >= search key. Define the keys
+	// between a restart point and the next restart point as belonging to that
+	// restart point. Note that index could be equal to i.numRestarts, i.e., we
+	// are past the last restart.
+	//
+	// Since keys are strictly increasing, if index > 0 then the restart point
+	// at index-1 will be the first one that has some keys belonging to it that
+	// are less than the search key.  If index == 0, then all keys in this block
+	// are larger than the search key, so there is no match.
 	targetOffset := i.restarts
 	if index > 0 {
 		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
@@ -876,24 +930,41 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 	for {
 		i.offset = i.nextOffset
 		i.readEntry()
-		i.decodeInternalKey(i.key)
+		// When hidden keys are common, there is additional optimization possible
+		// by not caching entries that are hidden (note that some calls to
+		// cacheEntry don't decode the internal key before caching, but checking
+		// whether a key is hidden does not require full decoding). However, we do
+		// need to use the blockEntry.offset in the cache for the first entry at
+		// the reset point to do the binary search when the cache is empty -- so
+		// we would need to cache that first entry (though not the key) even if
+		// was hidden. Our current assumption is that if there are large numbers
+		// of hidden keys we will be able to skip whole blocks (using block
+		// property filters) so we don't bother optimizing.
+		hiddenPoint := i.decodeInternalKey(i.key)
 
-		if i.cmp(i.ikey.UserKey, ikey.UserKey) >= 0 {
+		// NB: we don't use the hiddenPoint return value of decodeInternalKey
+		// since we want to stop as soon as we reach a key >= ikey.UserKey, so
+		// that we can reverse.
+		if i.cmp(i.ikey.UserKey, key) >= 0 {
 			// The current key is greater than or equal to our search key. Back up to
 			// the previous key which was less than our search key. Note that this for
 			// loop will execute at least once with this if-block not being true, so
 			// the key we are backing up to is the last one this loop cached.
-			i.Prev()
-			// i.Prev() has already initialized i.lazyValue.
-			return &i.ikey, i.lazyValue
+			return i.Prev()
 		}
 
 		if i.nextOffset >= targetOffset {
-			// We've reached the end of the current restart block. Return the current
-			// key. When the restart interval is 1, the first iteration of the for
-			// loop will bring us here. In that case ikey is backed by the block so
-			// we get the desired key stability guarantee for the lifetime of the
-			// blockIter.
+			// We've reached the end of the current restart block. Return the
+			// current key if not hidden, else call Prev().
+			//
+			// When the restart interval is 1, the first iteration of the for loop
+			// will bring us here. In that case ikey is backed by the block so we
+			// get the desired key stability guarantee for the lifetime of the
+			// blockIter. That is, we never cache anything and therefore never
+			// return a key backed by cachedBuf.
+			if hiddenPoint {
+				return i.Prev()
+			}
 			break
 		}
 
@@ -923,7 +994,10 @@ func (i *blockIter) First() (*InternalKey, base.LazyValue) {
 	}
 	i.clearCache()
 	i.readEntry()
-	i.decodeInternalKey(i.key)
+	hiddenPoint := i.decodeInternalKey(i.key)
+	if hiddenPoint {
+		return i.Next()
+	}
 	if !i.lazyValueHandling.hasValuePrefix ||
 		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
 		i.lazyValue = base.MakeInPlaceValue(i.val)
@@ -958,7 +1032,10 @@ func (i *blockIter) Last() (*InternalKey, base.LazyValue) {
 		i.readEntry()
 	}
 
-	i.decodeInternalKey(i.key)
+	hiddenPoint := i.decodeInternalKey(i.key)
+	if hiddenPoint {
+		return i.Prev()
+	}
 	if !i.lazyValueHandling.hasValuePrefix ||
 		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
 		i.lazyValue = base.MakeInPlaceValue(i.val)
@@ -988,6 +1065,7 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
 		i.clearCache()
 	}
 
+start:
 	i.offset = i.nextOffset
 	if !i.valid() {
 		return nil, base.LazyValue{}
@@ -995,11 +1073,17 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
 	i.readEntry()
 	// Manually inlined version of i.decodeInternalKey(i.key).
 	if n := len(i.key) - 8; n >= 0 {
-		i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
+		trailer := binary.LittleEndian.Uint64(i.key[n:])
+		hiddenPoint := i.hideObsoletePoints &&
+			(trailer&trailerObsoleteBit != 0)
+		i.ikey.Trailer = trailer & trailerObsoleteMask
 		i.ikey.UserKey = i.key[:n:n]
 		if i.globalSeqNum != 0 {
 			i.ikey.SetSeqNum(i.globalSeqNum)
 		}
+		if hiddenPoint {
+			goto start
+		}
 	} else {
 		i.ikey.Trailer = uint64(InternalKeyKindInvalid)
 		i.ikey.UserKey = nil
@@ -1015,8 +1099,8 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
 	return &i.ikey, i.lazyValue
 }
 
-// nextPrefix is used for implementing NPrefix.
-func (i *blockIter) nextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
+// NextPrefix implements (base.InternalIterator).NextPrefix.
+func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
 	if i.lazyValueHandling.hasValuePrefix {
 		return i.nextPrefixV3(succKey)
 	}
@@ -1141,6 +1225,7 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
 		// The trailer is written in little endian, so the key kind is the first
 		// byte in the trailer that is encoded in the slice [unshared-8:unshared].
 		keyKind := InternalKeyKind((*[manual.MaxArrayLen]byte)(ptr)[unshared-8])
+		keyKind = keyKind & base.InternalKeyKindSSTableInternalObsoleteMask
 		prefixChanged := false
 		if keyKind == InternalKeyKindSet {
 			if invariants.Enabled && value == 0 {
@@ -1256,8 +1341,12 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
 			i.key = i.fullKey
 		}
 		// Manually inlined version of i.decodeInternalKey(i.key).
+		hiddenPoint := false
 		if n := len(i.key) - 8; n >= 0 {
-			i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
+			trailer := binary.LittleEndian.Uint64(i.key[n:])
+			hiddenPoint = i.hideObsoletePoints &&
+				(trailer&trailerObsoleteBit != 0)
+			i.ikey.Trailer = trailer & trailerObsoleteMask
 			i.ikey.UserKey = i.key[:n:n]
 			if i.globalSeqNum != 0 {
 				i.ikey.SetSeqNum(i.globalSeqNum)
@@ -1273,6 +1362,9 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
 		}
 		if prefixChanged || i.cmp(i.ikey.UserKey, succKey) >= 0 {
 			// Prefix has changed.
+			if hiddenPoint {
+				return i.Next()
+			}
 			if invariants.Enabled && !i.lazyValueHandling.hasValuePrefix {
 				panic(errors.AssertionFailedf("nextPrefixV3 being run for non-v3 sstable"))
 			}
@@ -1294,23 +1386,11 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
 	return i.SeekGE(succKey, base.SeekGEFlagsNone)
 }
 
-// NextPrefix implements (base.InternalIterator).NextPrefix
-func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
-	const nextsBeforeSeek = 3
-	k, v := i.Next()
-	for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ {
-		if j >= nextsBeforeSeek {
-			return i.SeekGE(succKey, base.SeekGEFlagsNone)
-		}
-		k, v = i.Next()
-	}
-	return k, v
-}
-
 // Prev implements internalIterator.Prev, as documented in the pebble
 // package.
 func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
-	if n := len(i.cached) - 1; n >= 0 {
+start:
+	for n := len(i.cached) - 1; n >= 0; n-- {
 		i.nextOffset = i.offset
 		e := &i.cached[n]
 		i.offset = e.offset
@@ -1318,7 +1398,13 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 		// Manually inlined version of i.decodeInternalKey(i.key).
 		i.key = i.cachedBuf[e.keyStart:e.keyEnd]
 		if n := len(i.key) - 8; n >= 0 {
-			i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
+			trailer := binary.LittleEndian.Uint64(i.key[n:])
+			hiddenPoint := i.hideObsoletePoints &&
+				(trailer&trailerObsoleteBit != 0)
+			if hiddenPoint {
+				continue
+			}
+			i.ikey.Trailer = trailer & trailerObsoleteMask
 			i.ikey.UserKey = i.key[:n:n]
 			if i.globalSeqNum != 0 {
 				i.ikey.SetSeqNum(i.globalSeqNum)
@@ -1360,6 +1446,8 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 			// index ≤ h < upper
 			offset := decodeRestart(i.data[i.restarts+4*h:])
 			if offset < targetOffset {
+				// Looking for the first restart that has offset >= targetOffset, so
+				// ignore h and earlier.
 				index = h + 1 // preserves f(i-1) == false
 			} else {
 				upper = h // preserves f(j) == true
@@ -1369,20 +1457,35 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 		// => answer is index.
 	}
 
+	// index is first restart with offset >= targetOffset. Note that
+	// targetOffset may not be at a restart point since one can call Prev()
+	// after Next() (so the cache was not populated) and targetOffset refers to
+	// the current entry. index-1 must have an offset < targetOffset (it can't
+	// be equal to targetOffset since the binary search would have selected that
+	// as the index).
 	i.offset = 0
 	if index > 0 {
 		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 	}
+	// TODO(sumeer): why is the else case not an error given targetOffset is a
+	// valid offset.
 
 	i.readEntry()
 
+	// We stop when i.nextOffset == targetOffset since the targetOffset is the
+	// entry we are stepping back from, and we don't need to cache the entry
+	// before it, since it is the candidate to return.
 	for i.nextOffset < targetOffset {
 		i.cacheEntry()
 		i.offset = i.nextOffset
 		i.readEntry()
 	}
 
-	i.decodeInternalKey(i.key)
+	hiddenPoint := i.decodeInternalKey(i.key)
+	if hiddenPoint {
+		// Use the cache.
+		goto start
+	}
 	if !i.lazyValueHandling.hasValuePrefix ||
 		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
 		i.lazyValue = base.MakeInPlaceValue(i.val)
@@ -1455,6 +1558,10 @@ type fragmentBlockIter struct {
 	err       error
 	dir       int8
 	closeHook func(i keyspan.FragmentIterator) error
+
+	// elideSameSeqnum, if true, returns only the first-occurring (in forward
+	// order) Key for each sequence number.
+	elideSameSeqnum bool
 }
 
 func (i *fragmentBlockIter) resetForReuse() fragmentBlockIter {
@@ -1483,6 +1590,24 @@ func (i *fragmentBlockIter) decodeSpanKeys(k *InternalKey, internalValue []byte)
 	}
 }
 
+func (i *fragmentBlockIter) elideKeysOfSameSeqNum() {
+	if invariants.Enabled {
+		if !i.elideSameSeqnum || len(i.span.Keys) == 0 {
+			panic("elideKeysOfSameSeqNum called when it should not be")
+		}
+	}
+	lastSeqNum := i.span.Keys[0].SeqNum()
+	k := 1
+	for j := 1; j < len(i.span.Keys); j++ {
+		if lastSeqNum != i.span.Keys[j].SeqNum() {
+			lastSeqNum = i.span.Keys[j].SeqNum()
+			i.span.Keys[k] = i.span.Keys[j]
+			k++
+		}
+	}
+	i.span.Keys = i.span.Keys[:k]
+}
+
 // gatherForward gathers internal keys with identical bounds. Keys defined over
 // spans of the keyspace are fragmented such that any overlapping key spans have
 // identical bounds. When these spans are persisted to a range deletion or range
@@ -1532,6 +1657,9 @@ func (i *fragmentBlockIter) gatherForward(k *InternalKey, lazyValue base.LazyVal
 		k, lazyValue = i.blockIter.Next()
 		internalValue = lazyValue.InPlaceValue()
 	}
+	if i.elideSameSeqnum && len(i.span.Keys) > 0 {
+		i.elideKeysOfSameSeqNum()
+	}
 	// i.blockIter is positioned over the first internal key for the next span.
 	return &i.span
 }
@@ -1591,6 +1719,9 @@ func (i *fragmentBlockIter) gatherBackward(k *InternalKey, lazyValue base.LazyVa
 	// Backwards iteration encounters internal keys in the wrong order.
 	keyspan.SortKeysByTrailer(&i.span.Keys)
 
+	if i.elideSameSeqnum && len(i.span.Keys) > 0 {
+		i.elideKeysOfSameSeqNum()
+	}
 	return &i.span
 }
 

vendor/github.com/cockroachdb/pebble/sstable/block_property.go

@@ -117,7 +117,7 @@ type BlockPropertyCollector interface {
 }
 
 // SuffixReplaceableBlockCollector is an extension to the BlockPropertyCollector
-// interface that allows a block property collector to indicate the it supports
+// interface that allows a block property collector to indicate that it supports
 // being *updated* during suffix replacement, i.e. when an existing SST in which
 // all keys have the same key suffix is updated to have a new suffix.
 //

vendor/github.com/cockroachdb/pebble/sstable/format.go

@@ -24,11 +24,161 @@ const (
 	TableFormatPebblev1 // Block properties.
 	TableFormatPebblev2 // Range keys.
 	TableFormatPebblev3 // Value blocks.
-	TableFormatPebblev4 // Reserved.
+	TableFormatPebblev4 // DELSIZED tombstones.
 
 	TableFormatMax = TableFormatPebblev4
 )
 
+// TableFormatPebblev4, in addition to DELSIZED, introduces the use of
+// InternalKeyKindSSTableInternalObsoleteBit.
+//
+// 1. Motivation
+//
+// We have various related problems caused by Pebble snapshots:
+//
+// - P1: RANGEDELs that delete points in the same sstable, but the points
+//   happen to not get deleted during compactions because of an open snapshot.
+//   This causes very expensive iteration, that has been observed in
+//   production deployments
+//
+// - P2: When iterating over a foreign sstable (in disaggregated storage), we
+//   need to do (a) point collapsing to expose at most one point per user key,
+//   (b) apply RANGEDELs in the sstable to hide deleted points in the same
+//   sstable. This per-sstable point collapsing iteration needs to be very
+//   efficient (ideally as efficient from a CPU perspective as iteration over
+//   regular sstables) since foreign sstables can be very long-lived -- one of
+//   the goals of disaggregated storage is to scale compute and disk bandwidth
+//   resources as a function of the hot (from a write perspective) data and
+//   not the whole data, so we don't want to have to rewrite foreign sstables
+//   solely to improve read performance.
+//
+// The ideal solution for P2 would allow user-facing reads to utilize the
+// existing SST iterators (with slight modifications) and with no loss of
+// efficiency. And for P1 and P2 we would like to skip whole blocks of
+// overwritten/deleted points. Even when we can't skip whole blocks, avoiding
+// key comparisons at iteration time to discover what points are deleted is
+// very desirable, since keys can be long.
+//
+// We observe that:
+//
+// - Reads:
+//   - All user-facing reads in CockroachDB use iterators over the DB, hence
+//     have a higher read seqnum than all sstables (there are some rare cases
+//     that can violate this, but those are not important from a performance
+//     optimization perspective).
+//
+//   - Certain internal-facing reads in CockroachDB use snapshots, but the
+//     snapshots are shortlived enough that most L5 and L6 sstables will have
+//     all seqnums lower than the snapshot seqnum.
+//
+// - Writes:
+//   - We already do key comparisons between points when writing the sstable
+//     to ensure that the sstable invariant (monotonically increasing internal
+//     keys) is not violated. So we know which points share the same userkey,
+//     and thereby which points are obsolete because there is a more recent
+//     point in the same sstable.
+//
+//   - The compactionIter knows which point id deleted by a RANGEDEL even if
+//     the point does need to be written because of a snapshot.
+//
+//   So this known information can be encoded in the sstable at write time and
+//   utilized for optimized reading.
+//
+// 2. Solution
+//
+// We primarily scope the solution to the following point kinds: SET,
+// SETWITHDEL, DEL, DELSIZED, SINGLEDEL. These are the ones marked locally
+// obsolete, i.e., obsolete within the sstable, and we can guarantee that at
+// most one point will be exposed per user key. MERGE keys create more
+// complexity: MERGE followed by MERGE causes multiple keys to not be
+// obsolete. Same applies for MERGE followed by SET/SETWITHDEL. Note that:
+//
+// - For regular sst iteration, the obsolete marking is a performance
+//   optimization, and multiple keys for the same userkey can be handled by
+//   higher layers in the iterator tree (specifically pebble.Iterator).
+//
+// - For foreign sst iteration, we disallow MERGEs to be written to such
+//   shared ssts (details below).
+//
+// The key kinds are marked with an obsolete bit
+// (InternalKeyKindSSTableInternalObsoleteBit) when the key-value pair is
+// obsolete. This marking is done within blockWriter, based on information
+// passed to it by Writer. In turn, Writer uses a combination of key
+// comparisons, and information provided by compactionIter to decide whether a
+// key-value pair is obsolete. Additionally, a Pebble-internal
+// BlockPropertyCollector (obsoleteKeyBlockPropertyCollector) is used to mark
+// blocks where all key-value pairs are obsolete. Since the common case is
+// non-obsolete blocks, this block property collector uses the empty byte
+// slice to represent a non-obsolete block, which consumes no space in
+// BlockHandleWithProperties.Props.
+//
+// At read time, the obsolete bit is only visible to the blockIter, which can
+// be optionally configured to hide obsolete points. This hiding is only
+// configured for data block iterators for sstables being read by user-facing
+// iterators at a seqnum greater than the max seqnum in the sstable.
+// Additionally, when this hiding is configured, a Pebble-internal block
+// property filter (obsoleteKeyBlockPropertyFilter), is used to skip whole
+// blocks that are obsolete.
+//
+// 2.1 Correctness
+//
+// Due to the level invariant, the sequence of seqnums for a user key in a
+// sstable represents a contiguous subsequence of the seqnums for the userkey
+// across the whole LSM, and is more recent than the seqnums in a sstable in a
+// lower level. So exposing exactly one point from a sstable for a userkey
+// will also mask the points for the userkey in lower levels. If we expose no
+// point, because of RANGEDELs, that RANGEDEL will also mask the points in
+// lower levels.
+//
+// Note that we do not need to do anything special at write time for
+// SETWITHDEL and SINGLEDEL. This is because these key kinds are treated
+// specially only by compactions, which do not hide obsolete points. For
+// regular reads, SETWITHDEL behaves the same as SET and SINGLEDEL behaves the
+// same as DEL.
+//
+// 2.2 Strictness and MERGE
+//
+// Setting the obsolete bit on point keys is advanced usage, so we support two
+// modes, both of which must be truthful when setting the obsolete bit, but
+// vary in when they don't set the obsolete bit.
+//
+// - Non-strict: In this mode, the bit does not need to be set for keys that
+//   are obsolete. Additionally, any sstable containing MERGE keys can only
+//   use this mode. An iterator over such an sstable, when configured to
+//   hideObsoletePoints, can expose multiple internal keys per user key, and
+//   can expose keys that are deleted by rangedels in the same sstable. This
+//   is the mode that non-advanced users should use. Pebble without
+//   disaggregated storage will also use this mode and will best-effort set
+//   the obsolete bit, to optimize iteration when snapshots have retained many
+//   obsolete keys.
+//
+// - Strict: In this mode, every obsolete key must have the obsolete bit set,
+//   and no MERGE keys are permitted. An iterator over such an sstable, when
+//   configured to hideObsoletePoints satisfies two properties:
+//   - S1: will expose at most one internal key per user key, which is the
+//     most recent one.
+//   - S2: will never expose keys that are deleted by rangedels in the same
+//     sstable.
+//
+//   This is the mode for two use cases in disaggregated storage (which will
+//   exclude parts of the key space that has MERGEs), for levels that contain
+//   sstables that can become foreign sstables:
+//   - Pebble compaction output to these levels that can become foreign
+//     sstables.
+//
+//   - CockroachDB ingest operations that can ingest into the levels that can
+//     become foreign sstables. Note, these are not sstables corresponding to
+//     copied data for CockroachDB range snapshots. This case occurs for
+//     operations like index backfills: these trivially satisfy the strictness
+//     criteria since they only write one key per userkey.
+//
+//     TODO(sumeer): this latter case is not currently supported, since only
+//     Writer.AddWithForceObsolete calls are permitted for writing strict
+//     obsolete sstables. This is done to reduce the likelihood of bugs. One
+//     simple way to lift this limitation would be to disallow adding any
+//     RANGEDELs when a Pebble-external writer is trying to construct a strict
+//     obsolete sstable.
+
 // ParseTableFormat parses the given magic bytes and version into its
 // corresponding internal TableFormat.
 func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) {

vendor/github.com/cockroachdb/pebble/sstable/internal.go

@@ -23,6 +23,8 @@ const (
 	InternalKeyKindLogData         = base.InternalKeyKindLogData
 	InternalKeyKindSingleDelete    = base.InternalKeyKindSingleDelete
 	InternalKeyKindRangeDelete     = base.InternalKeyKindRangeDelete
+	InternalKeyKindSetWithDelete   = base.InternalKeyKindSetWithDelete
+	InternalKeyKindDeleteSized     = base.InternalKeyKindDeleteSized
 	InternalKeyKindMax             = base.InternalKeyKindMax
 	InternalKeyKindInvalid         = base.InternalKeyKindInvalid
 	InternalKeySeqNumBatch         = base.InternalKeySeqNumBatch

vendor/github.com/cockroachdb/pebble/sstable/options.go

@@ -21,6 +21,17 @@ const (
 	NCompression
 )
 
+var ignoredInternalProperties = map[string]struct{}{
+	"rocksdb.column.family.id":             {},
+	"rocksdb.fixed.key.length":             {},
+	"rocksdb.index.key.is.user.key":        {},
+	"rocksdb.index.value.is.delta.encoded": {},
+	"rocksdb.oldest.key.time":              {},
+	"rocksdb.creation.time":                {},
+	"rocksdb.file.creation.time":           {},
+	"rocksdb.format.version":               {},
+}
+
 func (c Compression) String() string {
 	switch c {
 	case DefaultCompression:
@@ -100,6 +111,9 @@ type ReaderOptions struct {
 	// The default cache size is a zero-size cache.
 	Cache *cache.Cache
 
+	// User properties specified in this map will not be added to sst.Properties.UserProperties.
+	DeniedUserProperties map[string]struct{}
+
 	// Comparer defines a total ordering over the space of []byte keys: a 'less
 	// than' relationship. The same comparison algorithm must be used for reads
 	// and writes over the lifetime of the DB.
@@ -107,6 +121,9 @@ type ReaderOptions struct {
 	// The default value uses the same ordering as bytes.Compare.
 	Comparer *Comparer
 
+	// Merge defines the Merge function in use for this keyspace.
+	Merge base.Merge
+
 	// Filters is a map from filter policy name to filter policy. It is used for
 	// debugging tools which may be used on multiple databases configured with
 	// different filter policies. It is not necessary to populate this filters
@@ -126,12 +143,18 @@ func (o ReaderOptions) ensureDefaults() ReaderOptions {
 	if o.Comparer == nil {
 		o.Comparer = base.DefaultComparer
 	}
+	if o.Merge == nil {
+		o.Merge = base.DefaultMerger.Merge
+	}
 	if o.MergerName == "" {
 		o.MergerName = base.DefaultMerger.Name
 	}
 	if o.LoggerAndTracer == nil {
 		o.LoggerAndTracer = base.NoopLoggerAndTracer{}
 	}
+	if o.DeniedUserProperties == nil {
+		o.DeniedUserProperties = ignoredInternalProperties
+	}
 	return o
 }
 
@@ -210,6 +233,17 @@ type WriterOptions struct {
 	// by a wider range of tools and libraries.
 	TableFormat TableFormat
 
+	// IsStrictObsolete is only relevant for >= TableFormatPebblev4. See comment
+	// in format.go. Must be false if format < TableFormatPebblev4.
+	//
+	// TODO(bilal): set this when writing shared ssts.
+	IsStrictObsolete bool
+
+	// WritingToLowestLevel is only relevant for >= TableFormatPebblev4. It is
+	// used to set the obsolete bit on DEL/DELSIZED/SINGLEDEL if they are the
+	// youngest for a userkey.
+	WritingToLowestLevel bool
+
 	// TablePropertyCollectors is a list of TablePropertyCollector creation
 	// functions. A new TablePropertyCollector is created for each sstable built
 	// and lives for the lifetime of the table.

vendor/github.com/cockroachdb/pebble/sstable/properties.go

@@ -23,14 +23,6 @@ var propTagMap = make(map[string]reflect.StructField)
 var propBoolTrue = []byte{'1'}
 var propBoolFalse = []byte{'0'}
 
-var columnFamilyIDField = func() reflect.StructField {
-	f, ok := reflect.TypeOf(Properties{}).FieldByName("ColumnFamilyID")
-	if !ok {
-		panic("Properties.ColumnFamilyID field not found")
-	}
-	return f
-}()
-
 var propOffsetTagMap = make(map[uintptr]string)
 
 func init() {
@@ -56,51 +48,35 @@ func init() {
 // automatically populated during sstable creation and load from the properties
 // meta block when an sstable is opened.
 type Properties struct {
-	// ID of column family for this SST file, corresponding to the CF identified
-	// by column_family_name.
-	ColumnFamilyID uint64 `prop:"rocksdb.column.family.id"`
-	// Name of the column family with which this SST file is associated. Empty if
-	// the column family is unknown.
-	ColumnFamilyName string `prop:"rocksdb.column.family.name"`
 	// The name of the comparer used in this table.
 	ComparerName string `prop:"rocksdb.comparator"`
 	// The compression algorithm used to compress blocks.
 	CompressionName string `prop:"rocksdb.compression"`
 	// The compression options used to compress blocks.
 	CompressionOptions string `prop:"rocksdb.compression_options"`
-	// The time when the SST file was created. Since SST files are immutable,
-	// this is equivalent to last modified time.
-	CreationTime uint64 `prop:"rocksdb.creation.time"`
 	// The total size of all data blocks.
 	DataSize uint64 `prop:"rocksdb.data.size"`
 	// The external sstable version format. Version 2 is the one RocksDB has been
 	// using since 5.13. RocksDB only uses the global sequence number for an
 	// sstable if this property has been set.
 	ExternalFormatVersion uint32 `prop:"rocksdb.external_sst_file.version"`
-	// Actual SST file creation time. 0 means unknown.
-	FileCreationTime uint64 `prop:"rocksdb.file.creation.time"`
 	// The name of the filter policy used in this table. Empty if no filter
 	// policy is used.
 	FilterPolicyName string `prop:"rocksdb.filter.policy"`
 	// The size of filter block.
 	FilterSize uint64 `prop:"rocksdb.filter.size"`
-	// If 0, key is variable length. Otherwise number of bytes for each key.
-	FixedKeyLen uint64 `prop:"rocksdb.fixed.key.length"`
-	// Format version, reserved for backward compatibility.
-	FormatVersion uint64 `prop:"rocksdb.format.version"`
 	// The global sequence number to use for all entries in the table. Present if
 	// the table was created externally and ingested whole.
 	GlobalSeqNum uint64 `prop:"rocksdb.external_sst_file.global_seqno"`
-	// Whether the index key is user key or an internal key.
-	IndexKeyIsUserKey uint64 `prop:"rocksdb.index.key.is.user.key"`
 	// Total number of index partitions if kTwoLevelIndexSearch is used.
 	IndexPartitions uint64 `prop:"rocksdb.index.partitions"`
 	// The size of index block.
 	IndexSize uint64 `prop:"rocksdb.index.size"`
 	// The index type. TODO(peter): add a more detailed description.
 	IndexType uint32 `prop:"rocksdb.block.based.table.index.type"`
-	// Whether delta encoding is used to encode the index values.
-	IndexValueIsDeltaEncoded uint64 `prop:"rocksdb.index.value.is.delta.encoded"`
+	// For formats >= TableFormatPebblev4, this is set to true if the obsolete
+	// bit is strict for all the point keys.
+	IsStrictObsolete bool `prop:"pebble.obsolete.is_strict"`
 	// The name of the merger used in this table. Empty if no merger is used.
 	MergerName string `prop:"rocksdb.merge.operator"`
 	// The number of blocks in this table.
@@ -108,6 +84,9 @@ type Properties struct {
 	// The number of deletion entries in this table, including both point and
 	// range deletions.
 	NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
+	// The number of point deletion entries ("tombstones") in this table that
+	// carry a size hint indicating the size of the value the tombstone deletes.
+	NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
 	// The number of entries in this table.
 	NumEntries uint64 `prop:"rocksdb.num.entries"`
 	// The number of merge operands in the table.
@@ -124,8 +103,6 @@ type Properties struct {
 	NumValueBlocks uint64 `prop:"pebble.num.value-blocks"`
 	// The number of values stored in value blocks. Only serialized if > 0.
 	NumValuesInValueBlocks uint64 `prop:"pebble.num.values.in.value-blocks"`
-	// Timestamp of the earliest key. 0 if unknown.
-	OldestKeyTime uint64 `prop:"rocksdb.oldest.key.time"`
 	// The name of the prefix extractor used in this table. Empty if no prefix
 	// extractor is used.
 	PrefixExtractorName string `prop:"rocksdb.prefix.extractor.name"`
@@ -139,6 +116,10 @@ type Properties struct {
 	// Total raw key size of point deletion tombstones. This value is comparable
 	// to RawKeySize.
 	RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
+	// Sum of the raw value sizes carried by point deletion tombstones
+	// containing size estimates. See the DeleteSized key kind. This value is
+	// comparable to Raw{Key,Value}Size.
+	RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
 	// Total raw rangekey key size.
 	RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"`
 	// Total raw rangekey value size.
@@ -206,12 +187,7 @@ func (p *Properties) String() string {
 		case reflect.Uint32:
 			fmt.Fprintf(&buf, "%d\n", f.Uint())
 		case reflect.Uint64:
-			u := f.Uint()
-			if ft.Offset == columnFamilyIDField.Offset && u == math.MaxInt32 {
-				fmt.Fprintf(&buf, "-\n")
-			} else {
-				fmt.Fprintf(&buf, "%d\n", f.Uint())
-			}
+			fmt.Fprintf(&buf, "%d\n", f.Uint())
 		case reflect.String:
 			fmt.Fprintf(&buf, "%s\n", f.String())
 		default:
@@ -229,7 +205,9 @@ func (p *Properties) String() string {
 	return buf.String()
 }
 
-func (p *Properties) load(b block, blockOffset uint64) error {
+func (p *Properties) load(
+	b block, blockOffset uint64, deniedUserProperties map[string]struct{},
+) error {
 	i, err := newRawBlockIter(bytes.Compare, b)
 	if err != nil {
 		return err
@@ -237,8 +215,7 @@ func (p *Properties) load(b block, blockOffset uint64) error {
 	p.Loaded = make(map[uintptr]struct{})
 	v := reflect.ValueOf(p).Elem()
 	for valid := i.First(); valid; valid = i.Next() {
-		tag := intern.Bytes(i.Key().UserKey)
-		if f, ok := propTagMap[tag]; ok {
+		if f, ok := propTagMap[string(i.Key().UserKey)]; ok {
 			p.Loaded[f.Offset] = struct{}{}
 			field := v.FieldByIndex(f.Index)
 			switch f.Type.Kind() {
@@ -248,7 +225,7 @@ func (p *Properties) load(b block, blockOffset uint64) error {
 				field.SetUint(uint64(binary.LittleEndian.Uint32(i.Value())))
 			case reflect.Uint64:
 				var n uint64
-				if tag == propGlobalSeqnumName {
+				if string(i.Key().UserKey) == propGlobalSeqnumName {
 					n = binary.LittleEndian.Uint64(i.Value())
 				} else {
 					n, _ = binary.Uvarint(i.Value())
@@ -264,7 +241,10 @@ func (p *Properties) load(b block, blockOffset uint64) error {
 		if p.UserProperties == nil {
 			p.UserProperties = make(map[string]string)
 		}
-		p.UserProperties[tag] = string(i.Value())
+
+		if _, denied := deniedUserProperties[string(i.Key().UserKey)]; !denied {
+			p.UserProperties[intern.Bytes(i.Key().UserKey)] = string(i.Value())
+		}
 	}
 	return nil
 }
@@ -306,10 +286,6 @@ func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) {
 		m[k] = []byte(v)
 	}
 
-	p.saveUvarint(m, unsafe.Offsetof(p.ColumnFamilyID), p.ColumnFamilyID)
-	if p.ColumnFamilyName != "" {
-		p.saveString(m, unsafe.Offsetof(p.ColumnFamilyName), p.ColumnFamilyName)
-	}
 	if p.ComparerName != "" {
 		p.saveString(m, unsafe.Offsetof(p.ComparerName), p.ComparerName)
 	}
@@ -319,44 +295,45 @@ func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) {
 	if p.CompressionOptions != "" {
 		p.saveString(m, unsafe.Offsetof(p.CompressionOptions), p.CompressionOptions)
 	}
-	p.saveUvarint(m, unsafe.Offsetof(p.CreationTime), p.CreationTime)
 	p.saveUvarint(m, unsafe.Offsetof(p.DataSize), p.DataSize)
 	if p.ExternalFormatVersion != 0 {
 		p.saveUint32(m, unsafe.Offsetof(p.ExternalFormatVersion), p.ExternalFormatVersion)
 		p.saveUint64(m, unsafe.Offsetof(p.GlobalSeqNum), p.GlobalSeqNum)
 	}
-	if p.FileCreationTime > 0 {
-		p.saveUvarint(m, unsafe.Offsetof(p.FileCreationTime), p.FileCreationTime)
-	}
 	if p.FilterPolicyName != "" {
 		p.saveString(m, unsafe.Offsetof(p.FilterPolicyName), p.FilterPolicyName)
 	}
 	p.saveUvarint(m, unsafe.Offsetof(p.FilterSize), p.FilterSize)
-	p.saveUvarint(m, unsafe.Offsetof(p.FixedKeyLen), p.FixedKeyLen)
-	p.saveUvarint(m, unsafe.Offsetof(p.FormatVersion), p.FormatVersion)
-	p.saveUvarint(m, unsafe.Offsetof(p.IndexKeyIsUserKey), p.IndexKeyIsUserKey)
 	if p.IndexPartitions != 0 {
 		p.saveUvarint(m, unsafe.Offsetof(p.IndexPartitions), p.IndexPartitions)
 		p.saveUvarint(m, unsafe.Offsetof(p.TopLevelIndexSize), p.TopLevelIndexSize)
 	}
 	p.saveUvarint(m, unsafe.Offsetof(p.IndexSize), p.IndexSize)
 	p.saveUint32(m, unsafe.Offsetof(p.IndexType), p.IndexType)
-	p.saveUvarint(m, unsafe.Offsetof(p.IndexValueIsDeltaEncoded), p.IndexValueIsDeltaEncoded)
+	if p.IsStrictObsolete {
+		p.saveBool(m, unsafe.Offsetof(p.IsStrictObsolete), p.IsStrictObsolete)
+	}
 	if p.MergerName != "" {
 		p.saveString(m, unsafe.Offsetof(p.MergerName), p.MergerName)
 	}
 	p.saveUvarint(m, unsafe.Offsetof(p.NumDataBlocks), p.NumDataBlocks)
 	p.saveUvarint(m, unsafe.Offsetof(p.NumEntries), p.NumEntries)
 	p.saveUvarint(m, unsafe.Offsetof(p.NumDeletions), p.NumDeletions)
+	if p.NumSizedDeletions > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.NumSizedDeletions), p.NumSizedDeletions)
+	}
 	p.saveUvarint(m, unsafe.Offsetof(p.NumMergeOperands), p.NumMergeOperands)
 	p.saveUvarint(m, unsafe.Offsetof(p.NumRangeDeletions), p.NumRangeDeletions)
-	// NB: We only write out the RawTombstoneKeySize for Pebble formats. This
-	// isn't strictly necessary because unrecognized properties are interpreted
-	// as user-defined properties, however writing them prevents byte-for-byte
+	// NB: We only write out some properties for Pebble formats. This isn't
+	// strictly necessary because unrecognized properties are interpreted as
+	// user-defined properties, however writing them prevents byte-for-byte
 	// equivalence with RocksDB files that some of our testing requires.
 	if p.RawPointTombstoneKeySize > 0 && tblFormat >= TableFormatPebblev1 {
 		p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneKeySize), p.RawPointTombstoneKeySize)
 	}
+	if p.RawPointTombstoneValueSize > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneValueSize), p.RawPointTombstoneValueSize)
+	}
 	if p.NumRangeKeys() > 0 {
 		p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeyDels), p.NumRangeKeyDels)
 		p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeySets), p.NumRangeKeySets)
@@ -370,7 +347,6 @@ func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) {
 	if p.NumValuesInValueBlocks > 0 {
 		p.saveUvarint(m, unsafe.Offsetof(p.NumValuesInValueBlocks), p.NumValuesInValueBlocks)
 	}
-	p.saveUvarint(m, unsafe.Offsetof(p.OldestKeyTime), p.OldestKeyTime)
 	if p.PrefixExtractorName != "" {
 		p.saveString(m, unsafe.Offsetof(p.PrefixExtractorName), p.PrefixExtractorName)
 	}
@@ -390,6 +366,16 @@ func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) {
 	}
 	p.saveBool(m, unsafe.Offsetof(p.WholeKeyFiltering), p.WholeKeyFiltering)
 
+	if tblFormat < TableFormatPebblev1 {
+		m["rocksdb.column.family.id"] = binary.AppendUvarint([]byte(nil), math.MaxInt32)
+		m["rocksdb.fixed.key.length"] = []byte{0x00}
+		m["rocksdb.index.key.is.user.key"] = []byte{0x00}
+		m["rocksdb.index.value.is.delta.encoded"] = []byte{0x00}
+		m["rocksdb.oldest.key.time"] = []byte{0x00}
+		m["rocksdb.creation.time"] = []byte{0x00}
+		m["rocksdb.format.version"] = []byte{0x00}
+	}
+
 	keys := make([]string, 0, len(m))
 	for key := range m {
 		keys = append(keys, key)

vendor/github.com/cockroachdb/pebble/sstable/reader.go

@@ -323,6 +323,8 @@ type singleLevelIterator struct {
 	// is high).
 	useFilter              bool
 	lastBloomFilterMatched bool
+
+	hideObsoletePoints bool
 }
 
 // singleLevelIterator implements the base.InternalIterator interface.
@@ -404,7 +406,7 @@ func (i *singleLevelIterator) init(
 	v *virtualState,
 	lower, upper []byte,
 	filterer *BlockPropertiesFilterer,
-	useFilter bool,
+	useFilter, hideObsoletePoints bool,
 	stats *base.InternalIteratorStats,
 	rp ReaderProvider,
 ) error {
@@ -428,7 +430,8 @@ func (i *singleLevelIterator) init(
 	i.reader = r
 	i.cmp = r.Compare
 	i.stats = stats
-	err = i.index.initHandle(i.cmp, indexH, r.Properties.GlobalSeqNum)
+	i.hideObsoletePoints = hideObsoletePoints
+	err = i.index.initHandle(i.cmp, indexH, r.Properties.GlobalSeqNum, false)
 	if err != nil {
 		// blockIter.Close releases indexH and always returns a nil error
 		_ = i.index.Close()
@@ -464,9 +467,9 @@ func (i *singleLevelIterator) init(
 // setupForCompaction sets up the singleLevelIterator for use with compactionIter.
 // Currently, it skips readahead ramp-up. It should be called after init is called.
 func (i *singleLevelIterator) setupForCompaction() {
-	i.dataRH.MaxReadahead()
+	i.dataRH.SetupForCompaction()
 	if i.vbRH != nil {
-		i.vbRH.MaxReadahead()
+		i.vbRH.SetupForCompaction()
 	}
 }
 
@@ -566,7 +569,7 @@ func (i *singleLevelIterator) loadBlock(dir int8) loadBlockResult {
 		i.err = err
 		return loadBlockFailed
 	}
-	i.err = i.data.initHandle(i.cmp, block, i.reader.Properties.GlobalSeqNum)
+	i.err = i.data.initHandle(i.cmp, block, i.reader.Properties.GlobalSeqNum, i.hideObsoletePoints)
 	if i.err != nil {
 		// The block is partially loaded, and we don't want it to appear valid.
 		i.data.invalidate()
@@ -653,12 +656,13 @@ func (i *singleLevelIterator) resolveMaybeExcluded(dir int8) intersectsResult {
 }
 
 func (i *singleLevelIterator) initBoundsForAlreadyLoadedBlock() {
-	if i.data.firstKey.UserKey == nil {
+	if i.data.getFirstUserKey() == nil {
 		panic("initBoundsForAlreadyLoadedBlock must not be called on empty or corrupted block")
 	}
 	i.blockLower = i.lower
 	if i.blockLower != nil {
-		if i.data.firstKey.UserKey != nil && i.cmp(i.blockLower, i.data.firstKey.UserKey) < 0 {
+		firstUserKey := i.data.getFirstUserKey()
+		if firstUserKey != nil && i.cmp(i.blockLower, firstUserKey) < 0 {
 			// The lower-bound is less than the first key in the block. No need
 			// to check the lower-bound again for this block.
 			i.blockLower = nil
@@ -1066,7 +1070,7 @@ func (i *singleLevelIterator) SeekLT(
 
 	var dontSeekWithinBlock bool
 	if !i.data.isDataInvalidated() && !i.index.isDataInvalidated() && i.data.valid() && i.index.valid() &&
-		boundsCmp < 0 && i.cmp(i.data.firstKey.UserKey, key) < 0 {
+		boundsCmp < 0 && i.cmp(i.data.getFirstUserKey(), key) < 0 {
 		// Fast-path: The bounds have moved backward, and this SeekLT is
 		// respecting the upper bound (guaranteed by Iterator). We know that
 		// the iterator must already be positioned within or just outside the
@@ -1314,7 +1318,7 @@ func (i *singleLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.Laz
 	if i.err != nil {
 		return nil, base.LazyValue{}
 	}
-	if key, val := i.data.nextPrefix(succKey); key != nil {
+	if key, val := i.data.NextPrefix(succKey); key != nil {
 		if i.blockUpper != nil {
 			cmp := i.cmp(key.UserKey, i.blockUpper)
 			if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 {
@@ -1763,8 +1767,7 @@ func (i *twoLevelIterator) loadIndex(dir int8) loadBlockResult {
 		i.err = err
 		return loadBlockFailed
 	}
-	if i.err = i.index.initHandle(
-		i.cmp, indexBlock, i.reader.Properties.GlobalSeqNum); i.err == nil {
+	if i.err = i.index.initHandle(i.cmp, indexBlock, i.reader.Properties.GlobalSeqNum, false); i.err == nil {
 		return loadBlockOK
 	}
 	return loadBlockFailed
@@ -1842,7 +1845,7 @@ func (i *twoLevelIterator) init(
 	v *virtualState,
 	lower, upper []byte,
 	filterer *BlockPropertiesFilterer,
-	useFilter bool,
+	useFilter, hideObsoletePoints bool,
 	stats *base.InternalIteratorStats,
 	rp ReaderProvider,
 ) error {
@@ -1867,7 +1870,8 @@ func (i *twoLevelIterator) init(
 	i.reader = r
 	i.cmp = r.Compare
 	i.stats = stats
-	err = i.topLevelIndex.initHandle(i.cmp, topLevelIndexH, r.Properties.GlobalSeqNum)
+	i.hideObsoletePoints = hideObsoletePoints
+	err = i.topLevelIndex.initHandle(i.cmp, topLevelIndexH, r.Properties.GlobalSeqNum, false)
 	if err != nil {
 		// blockIter.Close releases topLevelIndexH and always returns a nil error
 		_ = i.topLevelIndex.Close()
@@ -2907,21 +2911,20 @@ func (v *VirtualReader) NewCompactionIter(
 	return v.reader.newCompactionIter(bytesIterated, rp, &v.vState)
 }
 
-// NewIterWithBlockPropertyFiltersAndContext wraps
+// NewIterWithBlockPropertyFiltersAndContextEtc wraps
 // Reader.NewIterWithBlockPropertyFiltersAndContext. We assume that the passed
 // in [lower, upper) bounds will have at least some overlap with the virtual
 // sstable bounds. No overlap is not currently supported in the iterator.
-func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContext(
+func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContextEtc(
 	ctx context.Context,
 	lower, upper []byte,
 	filterer *BlockPropertiesFilterer,
-	useFilterBlock bool,
+	hideObsoletePoints, useFilterBlock bool,
 	stats *base.InternalIteratorStats,
 	rp ReaderProvider,
 ) (Iterator, error) {
 	return v.reader.newIterWithBlockPropertyFiltersAndContext(
-		ctx,
-		lower, upper, filterer, useFilterBlock, stats, rp, &v.vState,
+		ctx, lower, upper, filterer, hideObsoletePoints, useFilterBlock, stats, rp, &v.vState,
 	)
 }
 
@@ -3060,31 +3063,51 @@ func (r *Reader) NewIterWithBlockPropertyFilters(
 ) (Iterator, error) {
 	return r.newIterWithBlockPropertyFiltersAndContext(
 		context.Background(),
-		lower, upper, filterer, useFilterBlock, stats, rp, nil,
+		lower, upper, filterer, false, useFilterBlock, stats, rp, nil,
 	)
 }
 
-// NewIterWithBlockPropertyFiltersAndContext is similar to
+// NewIterWithBlockPropertyFiltersAndContextEtc is similar to
 // NewIterWithBlockPropertyFilters and additionally accepts a context for
 // tracing.
-func (r *Reader) NewIterWithBlockPropertyFiltersAndContext(
+//
+// If hideObsoletePoints, the callee assumes that filterer already includes
+// obsoleteKeyBlockPropertyFilter. The caller can satisfy this contract by
+// first calling TryAddBlockPropertyFilterForHideObsoletePoints.
+func (r *Reader) NewIterWithBlockPropertyFiltersAndContextEtc(
 	ctx context.Context,
 	lower, upper []byte,
 	filterer *BlockPropertiesFilterer,
-	useFilterBlock bool,
+	hideObsoletePoints, useFilterBlock bool,
 	stats *base.InternalIteratorStats,
 	rp ReaderProvider,
 ) (Iterator, error) {
 	return r.newIterWithBlockPropertyFiltersAndContext(
-		ctx,
-		lower, upper, filterer, useFilterBlock, stats, rp, nil,
+		ctx, lower, upper, filterer, hideObsoletePoints, useFilterBlock, stats, rp, nil,
 	)
 }
 
+// TryAddBlockPropertyFilterForHideObsoletePoints is expected to be called
+// before the call to NewIterWithBlockPropertyFiltersAndContextEtc, to get the
+// value of hideObsoletePoints and potentially add a block property filter.
+func (r *Reader) TryAddBlockPropertyFilterForHideObsoletePoints(
+	snapshotForHideObsoletePoints uint64,
+	fileLargestSeqNum uint64,
+	pointKeyFilters []BlockPropertyFilter,
+) (hideObsoletePoints bool, filters []BlockPropertyFilter) {
+	hideObsoletePoints = r.tableFormat >= TableFormatPebblev4 &&
+		snapshotForHideObsoletePoints > fileLargestSeqNum
+	if hideObsoletePoints {
+		pointKeyFilters = append(pointKeyFilters, obsoleteKeyBlockPropertyFilter{})
+	}
+	return hideObsoletePoints, pointKeyFilters
+}
+
 func (r *Reader) newIterWithBlockPropertyFiltersAndContext(
 	ctx context.Context,
 	lower, upper []byte,
 	filterer *BlockPropertiesFilterer,
+	hideObsoletePoints bool,
 	useFilterBlock bool,
 	stats *base.InternalIteratorStats,
 	rp ReaderProvider,
@@ -3095,7 +3118,7 @@ func (r *Reader) newIterWithBlockPropertyFiltersAndContext(
 	// until the final iterator closes.
 	if r.Properties.IndexType == twoLevelIndex {
 		i := twoLevelIterPool.Get().(*twoLevelIterator)
-		err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, stats, rp)
+		err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, hideObsoletePoints, stats, rp)
 		if err != nil {
 			return nil, err
 		}
@@ -3103,7 +3126,7 @@ func (r *Reader) newIterWithBlockPropertyFiltersAndContext(
 	}
 
 	i := singleLevelIterPool.Get().(*singleLevelIterator)
-	err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, stats, rp)
+	err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, hideObsoletePoints, stats, rp)
 	if err != nil {
 		return nil, err
 	}
@@ -3135,7 +3158,8 @@ func (r *Reader) newCompactionIter(
 		err := i.init(
 			context.Background(),
 			r, v, nil /* lower */, nil /* upper */, nil,
-			false /* useFilter */, nil /* stats */, rp,
+			false /* useFilter */, false, /* hideObsoletePoints */
+			nil /* stats */, rp,
 		)
 		if err != nil {
 			return nil, err
@@ -3149,7 +3173,8 @@ func (r *Reader) newCompactionIter(
 	i := singleLevelIterPool.Get().(*singleLevelIterator)
 	err := i.init(
 		context.Background(), r, v, nil /* lower */, nil, /* upper */
-		nil, false /* useFilter */, nil /* stats */, rp,
+		nil, false /* useFilter */, false, /* hideObsoletePoints */
+		nil /* stats */, rp,
 	)
 	if err != nil {
 		return nil, err
@@ -3175,8 +3200,8 @@ func (r *Reader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) {
 	if err != nil {
 		return nil, err
 	}
-	i := &fragmentBlockIter{}
-	if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum); err != nil {
+	i := &fragmentBlockIter{elideSameSeqnum: true}
+	if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum, false); err != nil {
 		return nil, err
 	}
 	return i, nil
@@ -3197,7 +3222,7 @@ func (r *Reader) NewRawRangeKeyIter() (keyspan.FragmentIterator, error) {
 		return nil, err
 	}
 	i := rangeKeyFragmentBlockIterPool.Get().(*rangeKeyFragmentBlockIter)
-	if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum); err != nil {
+	if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum, false); err != nil {
 		return nil, err
 	}
 	return i, nil
@@ -3357,7 +3382,7 @@ func (r *Reader) transformRangeDelV1(b []byte) ([]byte, error) {
 	// tombstones. We need properly fragmented and sorted range tombstones in
 	// order to serve from them directly.
 	iter := &blockIter{}
-	if err := iter.init(r.Compare, b, r.Properties.GlobalSeqNum); err != nil {
+	if err := iter.init(r.Compare, b, r.Properties.GlobalSeqNum, false); err != nil {
 		return nil, err
 	}
 	var tombstones []keyspan.Span
@@ -3444,7 +3469,7 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error {
 			return err
 		}
 		r.propertiesBH = bh
-		err := r.Properties.load(b.Get(), bh.Offset)
+		err := r.Properties.load(b.Get(), bh.Offset, r.opts.DeniedUserProperties)
 		b.Release()
 		if err != nil {
 			return err
@@ -3549,7 +3574,8 @@ func (r *Reader) Layout() (*Layout, error) {
 			if err != nil {
 				return nil, err
 			}
-			if err := iter.init(r.Compare, subIndex.Get(), 0 /* globalSeqNum */); err != nil {
+			if err := iter.init(r.Compare, subIndex.Get(), 0, /* globalSeqNum */
+				false /* hideObsoletePoints */); err != nil {
 				return nil, err
 			}
 			for key, value := iter.First(); key != nil; key, value = iter.Next() {
@@ -3867,6 +3893,7 @@ func NewReader(f objstorage.Readable, o ReaderOptions, extraOpts ...ReaderOption
 	if r.err != nil {
 		return nil, r.Close()
 	}
+
 	return r, nil
 }
 

vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go

@@ -43,6 +43,9 @@ func RewriteKeySuffixes(
 // modification as they are not affected by the suffix, while block and table
 // properties are only minimally recomputed.
 //
+// TODO(sumeer): document limitations, if any, due to this limited
+// re-computation of properties (is there any loss of fidelity?).
+//
 // Any block and table property collectors configured in the WriterOptions must
 // implement SuffixReplaceableTableCollector/SuffixReplaceableBlockCollector.
 //
@@ -50,6 +53,26 @@ func RewriteKeySuffixes(
 // same TableFormat as the input, which is returned in case the caller wants
 // to do some error checking. Suffix rewriting is meant to be efficient, and
 // allowing changes in the TableFormat detracts from that efficiency.
+//
+// Any obsolete bits that key-value pairs may be annotated with are ignored
+// and lost during the rewrite. Additionally, the output sstable has the
+// pebble.obsolete.is_strict property set to false. These limitations could be
+// removed if needed. The current use case for
+// RewriteKeySuffixesAndReturnFormat in CockroachDB is for MVCC-compliant file
+// ingestion, where these files do not contain RANGEDELs and have one
+// key-value pair per userkey -- so they trivially satisfy the strict
+// criteria, and we don't need the obsolete bit as a performance optimization.
+// For disaggregated storage, strict obsolete sstables are needed for L5 and
+// L6, but at the time of writing, we expect such MVCC-compliant file
+// ingestion to only ingest into levels L4 and higher. If this changes, we can
+// do one of two things to get rid of this limitation:
+//   - Validate that there are no duplicate userkeys and no RANGEDELs/MERGEs
+//     in the sstable to be rewritten. Validating no duplicate userkeys is
+//     non-trivial when rewriting blocks in parallel, so we could encode the
+//     pre-existing condition in the (existing) SnapshotPinnedKeys property --
+//     we need to update the external sst writer to calculate and encode this
+//     property.
+//   - Preserve the obsolete bit (with changes to the blockIter).
 func RewriteKeySuffixesAndReturnFormat(
 	sst []byte,
 	rOpts ReaderOptions,
@@ -184,7 +207,7 @@ func rewriteBlocks(
 		if err != nil {
 			return err
 		}
-		if err := iter.init(r.Compare, inputBlock, r.Properties.GlobalSeqNum); err != nil {
+		if err := iter.init(r.Compare, inputBlock, r.Properties.GlobalSeqNum, false); err != nil {
 			return err
 		}
 
@@ -433,8 +456,13 @@ func (c copyFilterWriter) policyName() string      { return c.origPolicyName }
 // RewriteKeySuffixesViaWriter is similar to RewriteKeySuffixes but uses just a
 // single loop over the Reader that writes each key to the Writer with the new
 // suffix. The is significantly slower than the parallelized rewriter, and does
-// more work to rederive filters, props, etc, however re-doing that work makes
-// it less restrictive -- props no longer need to
+// more work to rederive filters, props, etc.
+//
+// Any obsolete bits that key-value pairs may be annotated with are ignored
+// and lost during the rewrite. Some of the obsolete bits may be recreated --
+// specifically when there are multiple keys with the same user key.
+// Additionally, the output sstable has the pebble.obsolete.is_strict property
+// set to false. See the longer comment at RewriteKeySuffixesAndReturnFormat.
 func RewriteKeySuffixesViaWriter(
 	r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte,
 ) (*WriterMetadata, error) {
@@ -442,6 +470,7 @@ func RewriteKeySuffixesViaWriter(
 		return nil, errors.New("a valid splitter is required to rewrite suffixes")
 	}
 
+	o.IsStrictObsolete = false
 	w := NewWriter(out, o)
 	defer func() {
 		if w != nil {
@@ -472,7 +501,7 @@ func RewriteKeySuffixesViaWriter(
 		if err != nil {
 			return nil, err
 		}
-		if w.addPoint(scratch, val); err != nil {
+		if w.addPoint(scratch, val, false); err != nil {
 			return nil, err
 		}
 		k, v = i.Next()

vendor/github.com/cockroachdb/pebble/sstable/table.go

@@ -160,8 +160,27 @@ The metaindex block also contains block handles as values, with keys being
 the names of the meta blocks.
 
 For a description of value blocks and the meta value index block, see
-value_block.go
+value_block.go.
 
+Data blocks have some additional features:
+- For TableFormatPebblev3 onwards:
+  - For SETs, the value has a 1 byte value prefix, which indicates whether the
+    value is inline, or in a separate value block, and indicates whether the
+    prefix of the userkey (as defined by split) has changed or not. See
+    value_block.go for details.
+  - The most significant bit of the restart points is used to indicate whether
+    userkey prefix has changed since the last restart point. See the detailed
+    comment in blockWriter.
+  - The maximum length of the "shared prefix" when encoding the key, is the
+    length of the prefix of the userkey (as defined by split) of the previous
+    key.
+
+- For TableFormatPebblev4 onwards:
+  - The key kinds may be altered to set the
+    InternalKeyKindSSTableInternalObsoleteBit if the key-value pair is obsolete
+    in the context of that sstable (for a reader that reads at a higher seqnum
+    than the highest seqnum in the sstable). For details, see the comment in
+    format.go.
 */
 
 const (

vendor/github.com/cockroachdb/pebble/sstable/writer.go

@@ -132,6 +132,8 @@ type Writer struct {
 	separator               Separator
 	successor               Successor
 	tableFormat             TableFormat
+	isStrictObsolete        bool
+	writingToLowestLevel    bool
 	cache                   *cache.Cache
 	restartInterval         int
 	checksumType            ChecksumType
@@ -166,6 +168,7 @@ type Writer struct {
 	props               Properties
 	propCollectors      []TablePropertyCollector
 	blockPropCollectors []BlockPropertyCollector
+	obsoleteCollector   obsoleteKeyBlockPropertyCollector
 	blockPropsEncoder   blockPropertiesEncoder
 	// filter accumulates the filter block. If populated, the filter ingests
 	// either the output of w.split (i.e. a prefix extractor) if w.split is not
@@ -215,6 +218,8 @@ type pointKeyInfo struct {
 	// prefixLen uses w.split, if not nil. Only computed when w.valueBlockWriter
 	// is not nil.
 	prefixLen int
+	// True iff the point was marked obsolete.
+	isObsolete bool
 }
 
 type coordinationState struct {
@@ -673,7 +678,12 @@ func (w *Writer) Set(key, value []byte) error {
 	if w.err != nil {
 		return w.err
 	}
-	return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindSet), value)
+	if w.isStrictObsolete {
+		return errors.Errorf("use AddWithForceObsolete")
+	}
+	// forceObsolete is false based on the assumption that no RANGEDELs in the
+	// sstable delete the added points.
+	return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindSet), value, false)
 }
 
 // Delete deletes the value for the given key. The sequence number is set to
@@ -685,7 +695,12 @@ func (w *Writer) Delete(key []byte) error {
 	if w.err != nil {
 		return w.err
 	}
-	return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindDelete), nil)
+	if w.isStrictObsolete {
+		return errors.Errorf("use AddWithForceObsolete")
+	}
+	// forceObsolete is false based on the assumption that no RANGEDELs in the
+	// sstable delete the added points.
+	return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindDelete), nil, false)
 }
 
 // DeleteRange deletes all of the keys (and values) in the range [start,end)
@@ -711,7 +726,13 @@ func (w *Writer) Merge(key, value []byte) error {
 	if w.err != nil {
 		return w.err
 	}
-	return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindMerge), value)
+	if w.isStrictObsolete {
+		return errors.Errorf("use AddWithForceObsolete")
+	}
+	// forceObsolete is false based on the assumption that no RANGEDELs in the
+	// sstable that delete the added points. If the user configured this writer
+	// to be strict-obsolete, addPoint will reject the addition of this MERGE.
+	return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindMerge), value, false)
 }
 
 // Add adds a key/value pair to the table being written. For a given Writer,
@@ -721,6 +742,24 @@ func (w *Writer) Merge(key, value []byte) error {
 // point entries. Additionally, range deletion tombstones must be fragmented
 // (i.e. by keyspan.Fragmenter).
 func (w *Writer) Add(key InternalKey, value []byte) error {
+	if w.isStrictObsolete {
+		return errors.Errorf("use AddWithForceObsolete")
+	}
+	return w.AddWithForceObsolete(key, value, false)
+}
+
+// AddWithForceObsolete must be used when writing a strict-obsolete sstable.
+//
+// forceObsolete indicates whether the caller has determined that this key is
+// obsolete even though it may be the latest point key for this userkey. This
+// should be set to true for keys obsoleted by RANGEDELs, and is required for
+// strict-obsolete sstables.
+//
+// Note that there are two properties, S1 and S2 (see comment in format.go)
+// that strict-obsolete ssts must satisfy. S2, due to RANGEDELs, is solely the
+// responsibility of the caller. S1 is solely the responsibility of the
+// callee.
+func (w *Writer) AddWithForceObsolete(key InternalKey, value []byte, forceObsolete bool) error {
 	if w.err != nil {
 		return w.err
 	}
@@ -735,7 +774,7 @@ func (w *Writer) Add(key InternalKey, value []byte) error {
 			"pebble: range keys must be added via one of the RangeKey* functions")
 		return w.err
 	}
-	return w.addPoint(key, value)
+	return w.addPoint(key, value, forceObsolete)
 }
 
 func (w *Writer) makeAddPointDecisionV2(key InternalKey) error {
@@ -745,12 +784,7 @@ func (w *Writer) makeAddPointDecisionV2(key InternalKey) error {
 		return nil
 	}
 	if !w.disableKeyOrderChecks {
-		prevPointUserKey := w.dataBlockBuf.dataBlock.curKey
-		n := len(prevPointUserKey) - base.InternalTrailerLen
-		if n < 0 {
-			panic("corrupt key in blockWriter buffer")
-		}
-		prevPointUserKey = prevPointUserKey[:n:n]
+		prevPointUserKey := w.dataBlockBuf.dataBlock.getCurUserKey()
 		cmpUser := w.compare(prevPointUserKey, key.UserKey)
 		if cmpUser > 0 || (cmpUser == 0 && prevTrailer <= key.Trailer) {
 			return errors.Errorf(
@@ -762,9 +796,17 @@ func (w *Writer) makeAddPointDecisionV2(key InternalKey) error {
 	return nil
 }
 
+// REQUIRES: at least one point has been written to the Writer.
+func (w *Writer) getLastPointUserKey() []byte {
+	if w.dataBlockBuf.dataBlock.nEntries == 0 {
+		panic(errors.AssertionFailedf("no point keys added to writer"))
+	}
+	return w.dataBlockBuf.dataBlock.getCurUserKey()
+}
+
 func (w *Writer) makeAddPointDecisionV3(
 	key InternalKey, valueLen int,
-) (setHasSamePrefix bool, writeToValueBlock bool, err error) {
+) (setHasSamePrefix bool, writeToValueBlock bool, isObsolete bool, err error) {
 	prevPointKeyInfo := w.lastPointKeyInfo
 	w.lastPointKeyInfo.userKeyLen = len(key.UserKey)
 	w.lastPointKeyInfo.prefixLen = w.lastPointKeyInfo.userKeyLen
@@ -772,18 +814,16 @@ func (w *Writer) makeAddPointDecisionV3(
 		w.lastPointKeyInfo.prefixLen = w.split(key.UserKey)
 	}
 	w.lastPointKeyInfo.trailer = key.Trailer
-	if w.dataBlockBuf.dataBlock.nEntries == 0 {
-		return false, false, nil
+	w.lastPointKeyInfo.isObsolete = false
+	if !w.meta.HasPointKeys {
+		return false, false, false, nil
 	}
-	prevPointUserKey := w.dataBlockBuf.dataBlock.curKey
-	n := len(prevPointUserKey) - base.InternalTrailerLen
-	if n < 0 {
-		panic("corrupt key in blockWriter buffer")
-	}
-	prevPointUserKey = prevPointUserKey[:n:n]
+	keyKind := base.TrailerKind(key.Trailer)
+	prevPointUserKey := w.getLastPointUserKey()
 	prevPointKey := InternalKey{UserKey: prevPointUserKey, Trailer: prevPointKeyInfo.trailer}
-	considerWriteToValueBlock := base.TrailerKind(prevPointKeyInfo.trailer) == InternalKeyKindSet &&
-		base.TrailerKind(key.Trailer) == InternalKeyKindSet
+	prevKeyKind := base.TrailerKind(prevPointKeyInfo.trailer)
+	considerWriteToValueBlock := prevKeyKind == InternalKeyKindSet &&
+		keyKind == InternalKeyKindSet
 	if considerWriteToValueBlock && !w.requiredInPlaceValueBound.IsEmpty() {
 		keyPrefix := key.UserKey[:w.lastPointKeyInfo.prefixLen]
 		cmpUpper := w.compare(
@@ -812,14 +852,59 @@ func (w *Writer) makeAddPointDecisionV3(
 	} else {
 		cmpUser = w.compare(prevPointUserKey, key.UserKey)
 	}
+	keyKindIsSetOrMerge := keyKind == InternalKeyKindSet || keyKind == InternalKeyKindSetWithDelete ||
+		keyKind == InternalKeyKindMerge
+	keyKindIsPointDelete := keyKind == InternalKeyKindDelete ||
+		keyKind == InternalKeyKindSingleDelete || keyKind == InternalKeyKindDeleteSized
+	if !keyKindIsSetOrMerge && !keyKindIsPointDelete {
+		panic(errors.AssertionFailedf("unexpected key kind %s", keyKind.String()))
+	}
+	// If same user key, then the current key is obsolete if any of the
+	// following is true:
+	// C1 The prev key was obsolete.
+	// C2 The prev key was not a MERGE
+	// C3 The current key is not a SET or SETWITHDELETE or MERGE, then it is
+	//    obsolete. This rule excludes SET, SETWITHDELETE, MERGE, since they
+	//    will be merged with the previous key.
+	//
+	// Regardless of whether it is the same user key or not
+	// C4 The current key is some kind of point delete, and we are writing to
+	//    the lowest level, then it is also obsolete.
+	//
+	// C1 ensures that for a user key there is at most one transition from
+	// !obsolete to obsolete. Consider a user key k, for which the first n keys
+	// are not obsolete. We consider the various value of n:
+	//
+	// n = 0: This happens due to forceObsolete being set by the caller, or due
+	// to C4. forceObsolete must only be set due a RANGEDEL, and that RANGEDEL
+	// must also delete all the lower seqnums for the same user key. C4 triggers
+	// due to a point delete and that deletes all the lower seqnums for the same
+	// user key.
+	//
+	// n = 1: This is the common case. It happens when the first key is not a
+	// MERGE, or the current key is some kind of point delete.
+	//
+	// n > 1: This is due to a sequence of MERGE keys, potentially followed by a
+	// single SET or SETWITHDELETE.
+	isObsolete = (cmpUser == 0 &&
+		(prevKeyKind != InternalKeyKindMerge || prevPointKeyInfo.isObsolete || !keyKindIsSetOrMerge)) ||
+		(w.writingToLowestLevel &&
+			(keyKind == InternalKeyKindDelete || keyKind == InternalKeyKindSingleDelete ||
+				keyKind == InternalKeyKindDeleteSized))
+	// TODO(sumeer): storing isObsolete SET and SETWITHDEL in value blocks is
+	// possible, but requires some care in documenting and checking invariants.
+	// There is code that assumes nothing in value blocks because of single MVCC
+	// version (those should be ok). We have to ensure setHasSamePrefix is
+	// correctly initialized here etc.
+
 	if !w.disableKeyOrderChecks &&
 		(cmpUser > 0 || (cmpUser == 0 && prevPointKeyInfo.trailer <= key.Trailer)) {
-		return false, false, errors.Errorf(
+		return false, false, false, errors.Errorf(
 			"pebble: keys must be added in strictly increasing order: %s, %s",
 			prevPointKey.Pretty(w.formatKey), key.Pretty(w.formatKey))
 	}
 	if !considerWriteToValueBlock {
-		return false, false, nil
+		return false, false, isObsolete, nil
 	}
 	// NB: it is possible that cmpUser == 0, i.e., these two SETs have identical
 	// user keys (because of an open snapshot). This should be the rare case.
@@ -832,19 +917,24 @@ func (w *Writer) makeAddPointDecisionV3(
 	if considerWriteToValueBlock && valueLen <= tinyValueThreshold {
 		considerWriteToValueBlock = false
 	}
-	return setHasSamePrefix, considerWriteToValueBlock, nil
+	return setHasSamePrefix, considerWriteToValueBlock, isObsolete, nil
 }
 
-func (w *Writer) addPoint(key InternalKey, value []byte) error {
+func (w *Writer) addPoint(key InternalKey, value []byte, forceObsolete bool) error {
+	if w.isStrictObsolete && key.Kind() == InternalKeyKindMerge {
+		return errors.Errorf("MERGE not supported in a strict-obsolete sstable")
+	}
 	var err error
 	var setHasSameKeyPrefix, writeToValueBlock, addPrefixToValueStoredWithKey bool
+	var isObsolete bool
 	maxSharedKeyLen := len(key.UserKey)
 	if w.valueBlockWriter != nil {
 		// maxSharedKeyLen is limited to the prefix of the preceding key. If the
 		// preceding key was in a different block, then the blockWriter will
 		// ignore this maxSharedKeyLen.
 		maxSharedKeyLen = w.lastPointKeyInfo.prefixLen
-		setHasSameKeyPrefix, writeToValueBlock, err = w.makeAddPointDecisionV3(key, len(value))
+		setHasSameKeyPrefix, writeToValueBlock, isObsolete, err =
+			w.makeAddPointDecisionV3(key, len(value))
 		addPrefixToValueStoredWithKey = base.TrailerKind(key.Trailer) == InternalKeyKindSet
 	} else {
 		err = w.makeAddPointDecisionV2(key)
@@ -852,6 +942,8 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error {
 	if err != nil {
 		return err
 	}
+	isObsolete = w.tableFormat >= TableFormatPebblev4 && (isObsolete || forceObsolete)
+	w.lastPointKeyInfo.isObsolete = isObsolete
 	var valueStoredWithKey []byte
 	var prefix valuePrefix
 	var valueStoredWithKeyLen int
@@ -907,16 +999,19 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error {
 			return err
 		}
 	}
+	if w.tableFormat >= TableFormatPebblev4 {
+		w.obsoleteCollector.AddPoint(isObsolete)
+	}
 
 	w.maybeAddToFilter(key.UserKey)
 	w.dataBlockBuf.dataBlock.addWithOptionalValuePrefix(
-		key, valueStoredWithKey, maxSharedKeyLen, addPrefixToValueStoredWithKey, prefix,
+		key, isObsolete, valueStoredWithKey, maxSharedKeyLen, addPrefixToValueStoredWithKey, prefix,
 		setHasSameKeyPrefix)
 
 	w.meta.updateSeqNum(key.SeqNum())
 
 	if !w.meta.HasPointKeys {
-		k := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey)
+		k := w.dataBlockBuf.dataBlock.getCurKey()
 		// NB: We need to ensure that SmallestPoint.UserKey is set, so we create
 		// an InternalKey which is semantically identical to the key, but won't
 		// have a nil UserKey. We do this, because key.UserKey could be nil, and
@@ -932,7 +1027,22 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error {
 	switch key.Kind() {
 	case InternalKeyKindDelete, InternalKeyKindSingleDelete:
 		w.props.NumDeletions++
-		w.props.RawPointTombstoneKeySize += uint64(key.Size())
+		w.props.RawPointTombstoneKeySize += uint64(len(key.UserKey))
+	case InternalKeyKindDeleteSized:
+		var size uint64
+		if len(value) > 0 {
+			var n int
+			size, n = binary.Uvarint(value)
+			if n <= 0 {
+				w.err = errors.Newf("%s key's value (%x) does not parse as uvarint",
+					errors.Safe(key.Kind().String()), value)
+				return w.err
+			}
+		}
+		w.props.NumDeletions++
+		w.props.NumSizedDeletions++
+		w.props.RawPointTombstoneKeySize += uint64(len(key.UserKey))
+		w.props.RawPointTombstoneValueSize += size
 	case InternalKeyKindMerge:
 		w.props.NumMergeOperands++
 	}
@@ -953,7 +1063,7 @@ func (w *Writer) addTombstone(key InternalKey, value []byte) error {
 	if !w.disableKeyOrderChecks && !w.rangeDelV1Format && w.rangeDelBlock.nEntries > 0 {
 		// Check that tombstones are being added in fragmented order. If the two
 		// tombstones overlap, their start and end keys must be identical.
-		prevKey := base.DecodeInternalKey(w.rangeDelBlock.curKey)
+		prevKey := w.rangeDelBlock.getCurKey()
 		switch c := w.compare(prevKey.UserKey, key.UserKey); {
 		case c > 0:
 			w.err = errors.Errorf("pebble: keys must be added in order: %s, %s",
@@ -1149,7 +1259,7 @@ func (w *Writer) encodeRangeKeySpan(span keyspan.Span) {
 
 func (w *Writer) addRangeKey(key InternalKey, value []byte) error {
 	if !w.disableKeyOrderChecks && w.rangeKeyBlock.nEntries > 0 {
-		prevStartKey := base.DecodeInternalKey(w.rangeKeyBlock.curKey)
+		prevStartKey := w.rangeKeyBlock.getCurKey()
 		prevEndKey, _, ok := rangekey.DecodeEndKey(prevStartKey.Kind(), w.rangeKeyBlock.curValue)
 		if !ok {
 			// We panic here as we should have previously decoded and validated this
@@ -1295,7 +1405,7 @@ func (w *Writer) flush(key InternalKey) error {
 	// dataBlockBuf is added back to a sync.Pool. In this particular case, the
 	// byte slice which supports "sep" will eventually be copied when "sep" is
 	// added to the index block.
-	prevKey := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey)
+	prevKey := w.dataBlockBuf.dataBlock.getCurKey()
 	sep := w.indexEntrySep(prevKey, key, w.dataBlockBuf)
 	// We determine that we should flush an index block from the Writer client
 	// goroutine, but we actually finish the index block from the writeQueue.
@@ -1585,7 +1695,7 @@ func (w *Writer) finishIndexBlock(indexBuf *indexBlockBuf, props []byte) error {
 		nEntries: indexBuf.block.nEntries, properties: props,
 	}
 	w.indexSepAlloc, part.sep = cloneKeyWithBuf(
-		base.DecodeInternalKey(indexBuf.block.curKey), w.indexSepAlloc,
+		indexBuf.block.getCurKey(), w.indexSepAlloc,
 	)
 	bk := indexBuf.finish()
 	if len(w.indexBlockAlloc) < len(bk) {
@@ -1729,12 +1839,20 @@ func (w *Writer) assertFormatCompatibility() error {
 		)
 	}
 
+	// PebbleDBv3: value blocks.
 	if (w.props.NumValueBlocks > 0 || w.props.NumValuesInValueBlocks > 0 ||
 		w.props.ValueBlocksSize > 0) && w.tableFormat < TableFormatPebblev3 {
 		return errors.Newf(
 			"table format version %s is less than the minimum required version %s for value blocks",
 			w.tableFormat, TableFormatPebblev3)
 	}
+
+	// PebbleDBv4: DELSIZED tombstones.
+	if w.props.NumSizedDeletions > 0 && w.tableFormat < TableFormatPebblev4 {
+		return errors.Newf(
+			"table format version %s is less than the minimum required version %s for sized deletion tombstones",
+			w.tableFormat, TableFormatPebblev4)
+	}
 	return nil
 }
 
@@ -1781,7 +1899,7 @@ func (w *Writer) Close() (err error) {
 	//    however, if a dataBlock is flushed, then we add a key to the new w.dataBlockBuf in the
 	//    addPoint function after the flush occurs.
 	if w.dataBlockBuf.dataBlock.nEntries >= 1 {
-		w.meta.SetLargestPointKey(base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey).Clone())
+		w.meta.SetLargestPointKey(w.dataBlockBuf.dataBlock.getCurKey().Clone())
 	}
 
 	// Finish the last data block, or force an empty data block if there
@@ -1795,7 +1913,7 @@ func (w *Writer) Close() (err error) {
 		if err != nil {
 			return err
 		}
-		prevKey := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey)
+		prevKey := w.dataBlockBuf.dataBlock.getCurKey()
 		if err := w.addIndexEntrySync(prevKey, InternalKey{}, bhp, w.dataBlockBuf.tmp[:]); err != nil {
 			return err
 		}
@@ -1874,7 +1992,7 @@ func (w *Writer) Close() (err error) {
 
 	var rangeKeyBH BlockHandle
 	if w.props.NumRangeKeys() > 0 {
-		key := base.DecodeInternalKey(w.rangeKeyBlock.curKey)
+		key := w.rangeKeyBlock.getCurKey()
 		kind := key.Kind()
 		endKey, _, ok := rangekey.DecodeEndKey(kind, w.rangeKeyBlock.curValue)
 		if !ok {
@@ -2063,7 +2181,7 @@ func (o PreviousPointKeyOpt) UnsafeKey() base.InternalKey {
 	if o.w.dataBlockBuf.dataBlock.nEntries >= 1 {
 		// o.w.dataBlockBuf.dataBlock.curKey is guaranteed to point to the last point key
 		// which was added to the Writer.
-		return base.DecodeInternalKey(o.w.dataBlockBuf.dataBlock.curKey)
+		return o.w.dataBlockBuf.dataBlock.getCurKey()
 	}
 	return base.InternalKey{}
 }
@@ -2092,6 +2210,8 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write
 		separator:               o.Comparer.Separator,
 		successor:               o.Comparer.Successor,
 		tableFormat:             o.TableFormat,
+		isStrictObsolete:        o.IsStrictObsolete,
+		writingToLowestLevel:    o.WritingToLowestLevel,
 		cache:                   o.Cache,
 		restartInterval:         o.BlockRestartInterval,
 		checksumType:            o.Checksum,
@@ -2158,14 +2278,14 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write
 		}
 	}
 
-	w.props.ColumnFamilyID = math.MaxInt32
 	w.props.ComparerName = o.Comparer.Name
 	w.props.CompressionName = o.Compression.String()
 	w.props.MergerName = o.MergerName
 	w.props.PropertyCollectorNames = "[]"
 	w.props.ExternalFormatVersion = rocksDBExternalFormatVersion
 
-	if len(o.TablePropertyCollectors) > 0 || len(o.BlockPropertyCollectors) > 0 {
+	if len(o.TablePropertyCollectors) > 0 || len(o.BlockPropertyCollectors) > 0 ||
+		w.tableFormat >= TableFormatPebblev4 {
 		var buf bytes.Buffer
 		buf.WriteString("[")
 		if len(o.TablePropertyCollectors) > 0 {
@@ -2178,16 +2298,22 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write
 				buf.WriteString(w.propCollectors[i].Name())
 			}
 		}
+		numBlockPropertyCollectors := len(o.BlockPropertyCollectors)
+		if w.tableFormat >= TableFormatPebblev4 {
+			numBlockPropertyCollectors++
+		}
+		// shortID is a uint8, so we cannot exceed that number of block
+		// property collectors.
+		if numBlockPropertyCollectors > math.MaxUint8 {
+			w.err = errors.New("pebble: too many block property collectors")
+			return w
+		}
+		if numBlockPropertyCollectors > 0 {
+			w.blockPropCollectors = make([]BlockPropertyCollector, numBlockPropertyCollectors)
+		}
 		if len(o.BlockPropertyCollectors) > 0 {
-			// shortID is a uint8, so we cannot exceed that number of block
-			// property collectors.
-			if len(o.BlockPropertyCollectors) > math.MaxUint8 {
-				w.err = errors.New("pebble: too many block property collectors")
-				return w
-			}
 			// The shortID assigned to a collector is the same as its index in
 			// this slice.
-			w.blockPropCollectors = make([]BlockPropertyCollector, len(o.BlockPropertyCollectors))
 			for i := range o.BlockPropertyCollectors {
 				w.blockPropCollectors[i] = o.BlockPropertyCollectors[i]()
 				if i > 0 || len(o.TablePropertyCollectors) > 0 {
@@ -2196,6 +2322,13 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write
 				buf.WriteString(w.blockPropCollectors[i].Name())
 			}
 		}
+		if w.tableFormat >= TableFormatPebblev4 {
+			if numBlockPropertyCollectors > 1 || len(o.TablePropertyCollectors) > 0 {
+				buf.WriteString(",")
+			}
+			w.blockPropCollectors[numBlockPropertyCollectors-1] = &w.obsoleteCollector
+			buf.WriteString(w.obsoleteCollector.Name())
+		}
 		buf.WriteString("]")
 		w.props.PropertyCollectorNames = buf.String()
 	}
@@ -2229,3 +2362,85 @@ func init() {
 	}
 	private.SSTableInternalProperties = internalGetProperties
 }
+
+type obsoleteKeyBlockPropertyCollector struct {
+	blockIsNonObsolete bool
+	indexIsNonObsolete bool
+	tableIsNonObsolete bool
+}
+
+func encodeNonObsolete(isNonObsolete bool, buf []byte) []byte {
+	if isNonObsolete {
+		return buf
+	}
+	return append(buf, 't')
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) Name() string {
+	return "obsolete-key"
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) Add(key InternalKey, value []byte) error {
+	// Ignore.
+	return nil
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) AddPoint(isObsolete bool) {
+	o.blockIsNonObsolete = o.blockIsNonObsolete || !isObsolete
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) FinishDataBlock(buf []byte) ([]byte, error) {
+	o.tableIsNonObsolete = o.tableIsNonObsolete || o.blockIsNonObsolete
+	return encodeNonObsolete(o.blockIsNonObsolete, buf), nil
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) AddPrevDataBlockToIndexBlock() {
+	o.indexIsNonObsolete = o.indexIsNonObsolete || o.blockIsNonObsolete
+	o.blockIsNonObsolete = false
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) FinishIndexBlock(buf []byte) ([]byte, error) {
+	indexIsNonObsolete := o.indexIsNonObsolete
+	o.indexIsNonObsolete = false
+	return encodeNonObsolete(indexIsNonObsolete, buf), nil
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) FinishTable(buf []byte) ([]byte, error) {
+	return encodeNonObsolete(o.tableIsNonObsolete, buf), nil
+}
+
+func (o *obsoleteKeyBlockPropertyCollector) UpdateKeySuffixes(
+	oldProp []byte, oldSuffix, newSuffix []byte,
+) error {
+	_, err := propToIsObsolete(oldProp)
+	if err != nil {
+		return err
+	}
+	// Suffix rewriting currently loses the obsolete bit.
+	o.blockIsNonObsolete = true
+	return nil
+}
+
+// NB: obsoleteKeyBlockPropertyFilter is stateless. This aspect of the filter
+// is used in table_cache.go for in-place modification of a filters slice.
+type obsoleteKeyBlockPropertyFilter struct{}
+
+func (o obsoleteKeyBlockPropertyFilter) Name() string {
+	return "obsolete-key"
+}
+
+// Intersects returns true if the set represented by prop intersects with
+// the set in the filter.
+func (o obsoleteKeyBlockPropertyFilter) Intersects(prop []byte) (bool, error) {
+	return propToIsObsolete(prop)
+}
+
+func propToIsObsolete(prop []byte) (bool, error) {
+	if len(prop) == 0 {
+		return true, nil
+	}
+	if len(prop) > 1 || prop[0] != 't' {
+		return false, errors.Errorf("unexpected property %x", prop)
+	}
+	return false, nil
+}

vendor/github.com/cockroachdb/pebble/table_cache.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"runtime/debug"
 	"runtime/pprof"
 	"sync"
@@ -21,6 +22,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/internal/private"
+	"github.com/cockroachdb/pebble/internal/rangekey"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
 	"github.com/cockroachdb/pebble/sstable"
@@ -138,7 +140,7 @@ func (c *tableCacheContainer) newIters(
 }
 
 func (c *tableCacheContainer) newRangeKeyIter(
-	file *manifest.FileMetadata, opts *keyspan.SpanIterOptions,
+	file *manifest.FileMetadata, opts keyspan.SpanIterOptions,
 ) (keyspan.FragmentIterator, error) {
 	return c.tableCache.getShard(file.FileBacking.DiskFileNum).newRangeKeyIter(file, opts, &c.dbOpts)
 }
@@ -168,6 +170,32 @@ func (c *tableCacheContainer) metrics() (CacheMetrics, FilterMetrics) {
 	return m, f
 }
 
+func (c *tableCacheContainer) estimateSize(
+	meta *fileMetadata, lower, upper []byte,
+) (size uint64, err error) {
+	if meta.Virtual {
+		err = c.withVirtualReader(
+			meta.VirtualMeta(),
+			func(r sstable.VirtualReader) (err error) {
+				size, err = r.EstimateDiskUsage(lower, upper)
+				return err
+			},
+		)
+	} else {
+		err = c.withReader(
+			meta.PhysicalMeta(),
+			func(r *sstable.Reader) (err error) {
+				size, err = r.EstimateDiskUsage(lower, upper)
+				return err
+			},
+		)
+	}
+	if err != nil {
+		return 0, err
+	}
+	return size, nil
+}
+
 func (c *tableCacheContainer) withReader(meta physicalMeta, fn func(*sstable.Reader) error) error {
 	s := c.tableCache.getShard(meta.FileBacking.DiskFileNum)
 	v := s.findNode(meta.FileMetadata, &c.dbOpts)
@@ -373,6 +401,26 @@ func (c *tableCacheShard) newIters(
 		return nil, nil, v.err
 	}
 
+	hideObsoletePoints := false
+	if opts != nil {
+		// This code is appending (at most one filter) in-place to
+		// opts.PointKeyFilters even though the slice is shared for iterators in
+		// the same iterator tree. This is acceptable since all the following
+		// properties are true:
+		// - The iterator tree is single threaded, so the shared backing for the
+		//   slice is being mutated in a single threaded manner.
+		// - Each shallow copy of the slice has its own notion of length.
+		// - The appended element is always the obsoleteKeyBlockPropertyFilter
+		//   struct, which is stateless, so overwriting that struct when creating
+		//   one sstable iterator is harmless to other sstable iterators that are
+		//   relying on that struct.
+		//
+		// An alternative would be to have different slices for different sstable
+		// iterators, but that requires more work to avoid allocations.
+		hideObsoletePoints, opts.PointKeyFilters =
+			v.reader.TryAddBlockPropertyFilterForHideObsoletePoints(
+				opts.snapshotForHideObsoletePoints, file.LargestSeqNum, opts.PointKeyFilters)
+	}
 	ok := true
 	var filterer *sstable.BlockPropertiesFilterer
 	var err error
@@ -387,14 +435,7 @@ func (c *tableCacheShard) newIters(
 
 	type iterCreator interface {
 		NewRawRangeDelIter() (keyspan.FragmentIterator, error)
-		NewIterWithBlockPropertyFiltersAndContext(
-			ctx context.Context,
-			lower, upper []byte,
-			filterer *sstable.BlockPropertiesFilterer,
-			useFilterBlock bool,
-			stats *base.InternalIteratorStats,
-			rp sstable.ReaderProvider,
-		) (sstable.Iterator, error)
+		NewIterWithBlockPropertyFiltersAndContextEtc(ctx context.Context, lower, upper []byte, filterer *sstable.BlockPropertiesFilterer, hideObsoletePoints, useFilterBlock bool, stats *base.InternalIteratorStats, rp sstable.ReaderProvider) (sstable.Iterator, error)
 		NewCompactionIter(
 			bytesIterated *uint64,
 			rp sstable.ReaderProvider,
@@ -410,11 +451,38 @@ func (c *tableCacheShard) newIters(
 		ic = &virtualReader
 	}
 
+	provider := dbOpts.objProvider
+	// Check if this file is a foreign file.
+	objMeta, err := provider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	// NB: range-del iterator does not maintain a reference to the table, nor
 	// does it need to read from it after creation.
 	var rangeDelIter keyspan.FragmentIterator
-	rangeDelIter, err = ic.NewRawRangeDelIter()
-
+	if provider.IsForeign(objMeta) {
+		if opts == nil {
+			panic("unexpected nil opts when reading foreign file")
+		}
+		if !file.Virtual {
+			// Foreign sstables must be virtual by definition.
+			panic(fmt.Sprintf("sstable is foreign but not virtual: %s", file.FileNum))
+		}
+		switch manifest.LevelToInt(opts.level) {
+		case 5:
+			rangeDelIter, err = ic.NewRawRangeDelIter()
+		case 6:
+		// Let rangeDelIter remain nil. We don't need to return rangedels from
+		// this file as they will not apply to any other files. For the purpose
+		// of collapsing rangedels within this file, we create another rangeDelIter
+		// below for use with the interleaving iter.
+		default:
+			panic(fmt.Sprintf("unexpected level for foreign sstable: %d", manifest.LevelToInt(opts.level)))
+		}
+	} else {
+		rangeDelIter, err = ic.NewRawRangeDelIter()
+	}
 	if err != nil {
 		c.unrefValue(v)
 		return nil, nil, err
@@ -454,11 +522,9 @@ func (c *tableCacheShard) newIters(
 	if internalOpts.bytesIterated != nil {
 		iter, err = ic.NewCompactionIter(internalOpts.bytesIterated, rp)
 	} else {
-		iter, err = ic.NewIterWithBlockPropertyFiltersAndContext(
-			ctx,
-			opts.GetLowerBound(), opts.GetUpperBound(),
-			filterer, useFilter, internalOpts.stats, rp,
-		)
+		iter, err = ic.NewIterWithBlockPropertyFiltersAndContextEtc(
+			ctx, opts.GetLowerBound(), opts.GetUpperBound(), filterer, hideObsoletePoints, useFilter,
+			internalOpts.stats, rp)
 	}
 	if err != nil {
 		if rangeDelIter != nil {
@@ -470,6 +536,35 @@ func (c *tableCacheShard) newIters(
 	// NB: v.closeHook takes responsibility for calling unrefValue(v) here. Take
 	// care to avoid introducing an allocation here by adding a closure.
 	iter.SetCloseHook(v.closeHook)
+	if provider.IsForeign(objMeta) {
+		// NB: IsForeign() guarantees IsShared, so opts must not be nil as we've
+		// already panicked on the nil case above.
+		pointKeySeqNum := base.SeqNumForLevel(manifest.LevelToInt(opts.level))
+		pcIter := pointCollapsingIterator{
+			comparer:          dbOpts.opts.Comparer,
+			merge:             dbOpts.opts.Merge,
+			seqNum:            math.MaxUint64,
+			elideRangeDeletes: true,
+			fixedSeqNum:       pointKeySeqNum,
+		}
+		// Open a second rangedel iter. This is solely for the interleaving iter to
+		// be able to efficiently delete covered range deletes. We don't need to fix
+		// the sequence number in this iter, as these range deletes will not be
+		// exposed to anything other than the interleaving iter and
+		// pointCollapsingIter.
+		rangeDelIter, err := v.reader.NewRawRangeDelIter()
+		if err != nil {
+			c.unrefValue(v)
+			return nil, nil, err
+		}
+		if rangeDelIter == nil {
+			rangeDelIter = emptyKeyspanIter
+		}
+		pcIter.iter.Init(dbOpts.opts.Comparer, iter, rangeDelIter, nil /* mask */, opts.LowerBound, opts.UpperBound)
+		pcSSTIter := pcSSTIterPool.Get().(*pointCollapsingSSTIterator)
+		*pcSSTIter = pointCollapsingSSTIterator{pointCollapsingIterator: pcIter, childIter: iter}
+		iter = pcSSTIter
+	}
 
 	c.iterCount.Add(1)
 	dbOpts.iterCount.Add(1)
@@ -482,7 +577,7 @@ func (c *tableCacheShard) newIters(
 }
 
 func (c *tableCacheShard) newRangeKeyIter(
-	file *manifest.FileMetadata, opts *keyspan.SpanIterOptions, dbOpts *tableCacheOpts,
+	file *manifest.FileMetadata, opts keyspan.SpanIterOptions, dbOpts *tableCacheOpts,
 ) (keyspan.FragmentIterator, error) {
 	// Calling findNode gives us the responsibility of decrementing v's
 	// refCount. If opening the underlying table resulted in error, then we
@@ -501,7 +596,7 @@ func (c *tableCacheShard) newRangeKeyIter(
 	// file's range key blocks may surface deleted range keys below. This is
 	// done here, rather than deferring to the block-property collector in order
 	// to maintain parity with point keys and the treatment of RANGEDELs.
-	if opts != nil && v.reader.Properties.NumRangeKeyDels == 0 {
+	if v.reader.Properties.NumRangeKeyDels == 0 {
 		ok, _, err = c.checkAndIntersectFilters(v, nil, opts.RangeKeyFilters, nil)
 	}
 	if err != nil {
@@ -539,6 +634,29 @@ func (c *tableCacheShard) newRangeKeyIter(
 		return emptyKeyspanIter, nil
 	}
 
+	objMeta, err := dbOpts.objProvider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
+	if err != nil {
+		return nil, err
+	}
+	if dbOpts.objProvider.IsForeign(objMeta) {
+		if opts.Level == 0 {
+			panic("unexpected zero level when reading foreign file")
+		}
+		transform := &rangekey.ForeignSSTTransformer{
+			Comparer: dbOpts.opts.Comparer,
+			Level:    manifest.LevelToInt(opts.Level),
+		}
+		if iter == nil {
+			iter = emptyKeyspanIter
+		}
+		transformIter := &keyspan.TransformerIter{
+			FragmentIterator: iter,
+			Transformer:      transform,
+			Compare:          dbOpts.opts.Comparer.Compare,
+		}
+		return transformIter, nil
+	}
+
 	return iter, nil
 }
 
@@ -986,9 +1104,16 @@ type loadInfo struct {
 func (v *tableCacheValue) load(loadInfo loadInfo, c *tableCacheShard, dbOpts *tableCacheOpts) {
 	// Try opening the file first.
 	var f objstorage.Readable
-	f, v.err = dbOpts.objProvider.OpenForReading(
+	var err error
+	f, err = dbOpts.objProvider.OpenForReading(
 		context.TODO(), fileTypeTable, loadInfo.backingFileNum, objstorage.OpenOptions{MustExist: true},
 	)
+	if err != nil {
+		v.err = errors.Wrap(
+			err,
+			fmt.Sprintf("pebble: could not open backing file %s", errors.Safe(loadInfo.backingFileNum.FileNum())),
+		)
+	}
 	if v.err == nil {
 		cacheOpts := private.SSTableCacheOpts(dbOpts.cacheID, loadInfo.backingFileNum).(sstable.ReaderOption)
 		v.reader, v.err = sstable.NewReader(f, dbOpts.opts, cacheOpts, dbOpts.filterMetrics)

vendor/github.com/cockroachdb/pebble/table_stats.go

@@ -126,7 +126,7 @@ func (d *DB) collectTableStats() bool {
 	maybeCompact := false
 	for _, c := range collected {
 		c.fileMetadata.Stats = c.TableStats
-		maybeCompact = maybeCompact || c.TableStats.RangeDeletionsBytesEstimate > 0
+		maybeCompact = maybeCompact || fileCompensation(c.fileMetadata) > 0
 		c.fileMetadata.StatsMarkValid()
 	}
 	d.mu.tableStats.cond.Broadcast()
@@ -186,6 +186,11 @@ func (d *DB) loadNewFileStats(
 			continue
 		}
 
+		if nf.Meta.Virtual {
+			// cannot load virtual table stats
+			continue
+		}
+
 		stats, newHints, err := d.loadTableStats(
 			rs.current, nf.Level,
 			nf.Meta.PhysicalMeta(),
@@ -217,6 +222,12 @@ func (d *DB) scanReadStateTableStats(
 	for l, levelMetadata := range rs.current.Levels {
 		iter := levelMetadata.Iter()
 		for f := iter.First(); f != nil; f = iter.Next() {
+			if f.Virtual {
+				// TODO(bananabrick): Support stats collection for virtual
+				// sstables.
+				continue
+			}
+
 			// NB: We're not holding d.mu which protects f.Stats, but only the
 			// active stats collection job updates f.Stats for active files,
 			// and we ensure only one goroutine runs it at a time through
@@ -225,6 +236,11 @@ func (d *DB) scanReadStateTableStats(
 			if f.StatsValid() {
 				continue
 			}
+			// TODO(bilal): Remove this guard when table stats collection is
+			// implemented for virtual sstables.
+			if f.Virtual {
+				continue
+			}
 
 			// Limit how much work we do per read state. The older the read
 			// state is, the higher the likelihood files are no longer being
@@ -304,12 +320,12 @@ func (d *DB) loadTablePointKeyStats(
 	// We could write the ranges of 'clusters' of point tombstones to
 	// a sstable property and call averageValueSizeBeneath for each of
 	// these narrower ranges to improve the estimate.
-	avgValSize, err := d.averagePhysicalValueSizeBeneath(v, level, meta)
+	avgValLogicalSize, compressionRatio, err := d.estimateSizesBeneath(v, level, meta)
 	if err != nil {
 		return err
 	}
 	stats.PointDeletionsBytesEstimate =
-		pointDeletionsBytesEstimate(meta.Size, &r.Properties, avgValSize)
+		pointDeletionsBytesEstimate(meta.Size, &r.Properties, avgValLogicalSize, compressionRatio)
 	return nil
 }
 
@@ -420,51 +436,60 @@ func (d *DB) loadTableRangeDelStats(
 	return compactionHints, err
 }
 
-func (d *DB) averagePhysicalValueSizeBeneath(
+func (d *DB) estimateSizesBeneath(
 	v *version, level int, meta physicalMeta,
-) (avgValueSize uint64, err error) {
+) (avgValueLogicalSize, compressionRatio float64, err error) {
 	// Find all files in lower levels that overlap with meta,
 	// summing their value sizes and entry counts.
+	file := meta.FileMetadata
 	var fileSum, keySum, valSum, entryCount uint64
+
+	addPhysicalTableStats := func(r *sstable.Reader) (err error) {
+		fileSum += file.Size
+		entryCount += r.Properties.NumEntries
+		keySum += r.Properties.RawKeySize
+		valSum += r.Properties.RawValueSize
+		return nil
+	}
+	addVirtualTableStats := func(v sstable.VirtualReader) (err error) {
+		fileSum += file.Size
+		entryCount += file.Stats.NumEntries
+		keySum += v.Properties.RawKeySize
+		valSum += v.Properties.RawValueSize
+		return nil
+	}
+
+	// Include the file itself. This is important because in some instances, the
+	// computed compression ratio is applied to the tombstones contained within
+	// `meta` itself. If there are no files beneath `meta` in the LSM, we would
+	// calculate a compression ratio of 0 which is not accurate for the file's
+	// own tombstones.
+	if err = d.tableCache.withReader(meta, addPhysicalTableStats); err != nil {
+		return 0, 0, err
+	}
+
 	for l := level + 1; l < numLevels; l++ {
 		overlaps := v.Overlaps(l, d.cmp, meta.Smallest.UserKey,
 			meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel())
 		iter := overlaps.Iter()
-		for file := iter.First(); file != nil; file = iter.Next() {
+		for file = iter.First(); file != nil; file = iter.Next() {
 			var err error
 			if file.Virtual {
-				err = d.tableCache.withVirtualReader(
-					file.VirtualMeta(),
-					func(v sstable.VirtualReader) (err error) {
-						fileSum += file.Size
-						entryCount += file.Stats.NumEntries
-						keySum += v.Properties.RawKeySize
-						valSum += v.Properties.RawValueSize
-						return nil
-					})
+				err = d.tableCache.withVirtualReader(file.VirtualMeta(), addVirtualTableStats)
 			} else {
-				err = d.tableCache.withReader(
-					file.PhysicalMeta(),
-					func(r *sstable.Reader) (err error) {
-						fileSum += file.Size
-						entryCount += r.Properties.NumEntries
-						keySum += r.Properties.RawKeySize
-						valSum += r.Properties.RawValueSize
-						return nil
-					})
+				err = d.tableCache.withReader(file.PhysicalMeta(), addPhysicalTableStats)
 			}
-
 			if err != nil {
-				return 0, err
+				return 0, 0, err
 			}
 		}
 	}
 	if entryCount == 0 {
-		return 0, nil
+		return 0, 0, nil
 	}
-	// RawKeySize and RawValueSize are uncompressed totals. Scale the value sum
-	// according to the data size to account for compression, index blocks and
-	// metadata overhead. Eg:
+	// RawKeySize and RawValueSize are uncompressed totals. We'll need to scale
+	// the value sum according to the data size to account for compression,
+	// index blocks and metadata overhead. Eg:
 	//
 	//    Compression rate        ×  Average uncompressed value size
 	//
@@ -473,10 +498,14 @@ func (d *DB) averagePhysicalValueSizeBeneath(
 	//         FileSize              RawValueSize
 	//   -----------------------  ×  ------------
 	//   RawKeySize+RawValueSize     NumEntries
+	//
+	// We return the average logical value size plus the compression ratio,
+	// leaving the scaling to the caller. This allows the caller to perform
+	// additional compression ratio scaling if necessary.
 	uncompressedSum := float64(keySum + valSum)
-	compressionRatio := float64(fileSum) / uncompressedSum
-	avgCompressedValueSize := (float64(valSum) / float64(entryCount)) * compressionRatio
-	return uint64(avgCompressedValueSize), nil
+	compressionRatio = float64(fileSum) / uncompressedSum
+	avgValueLogicalSize = (float64(valSum) / float64(entryCount))
+	return avgValueLogicalSize, compressionRatio, nil
 }
 
 func (d *DB) estimateReclaimedSizeBeneath(
@@ -593,12 +622,12 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
 		return false
 	}
 
-	// If a table is more than 10% point deletions, don't calculate the
-	// PointDeletionsBytesEstimate statistic using our limited knowledge. The
-	// table stats collector can populate the stats and calculate an average
-	// of value size of all the tables beneath the table in the LSM, which
-	// will be more accurate.
-	if props.NumDeletions > props.NumEntries/10 {
+	// If a table is more than 10% point deletions without user-provided size
+	// estimates, don't calculate the PointDeletionsBytesEstimate statistic
+	// using our limited knowledge. The table stats collector can populate the
+	// stats and calculate an average of value size of all the tables beneath
+	// the table in the LSM, which will be more accurate.
+	if unsizedDels := (props.NumDeletions - props.NumSizedDeletions); unsizedDels > props.NumEntries/10 {
 		return false
 	}
 
@@ -608,8 +637,8 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
 		// doesn't require any additional IO and since the number of point
 		// deletions in the file is low, the error introduced by this crude
 		// estimate is expected to be small.
-		avgValSize := estimateValuePhysicalSize(meta.Size, props)
-		pointEstimate = pointDeletionsBytesEstimate(meta.Size, props, avgValSize)
+		avgValSize, compressionRatio := estimatePhysicalSizes(meta.Size, props)
+		pointEstimate = pointDeletionsBytesEstimate(meta.Size, props, avgValSize, compressionRatio)
 	}
 
 	meta.Stats.NumEntries = props.NumEntries
@@ -623,7 +652,7 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
 }
 
 func pointDeletionsBytesEstimate(
-	fileSize uint64, props *sstable.Properties, avgValPhysicalSize uint64,
+	fileSize uint64, props *sstable.Properties, avgValLogicalSize, compressionRatio float64,
 ) (estimate uint64) {
 	if props.NumEntries == 0 {
 		return 0
@@ -639,43 +668,79 @@ func pointDeletionsBytesEstimate(
 	// If there are covered key(s), we also get to drop key and value bytes for
 	// each covered key.
 	//
-	// We estimate assuming that each point tombstone on average covers 1 key.
+	// Some point tombstones (DELSIZEDs) carry a user-provided estimate of the
+	// uncompressed size of entries that will be elided by fully compacting the
+	// tombstone. For these tombstones, there's no guesswork—we use the
+	// RawPointTombstoneValueSizeHint property which is the sum of all these
+	// tombstones' encoded values.
+	//
+	// For un-sized point tombstones (DELs), we estimate assuming that each
+	// point tombstone on average covers 1 key and using average vaue sizes.
 	// This is almost certainly an overestimate, but that's probably okay
 	// because point tombstones can slow range iterations even when they don't
-	// cover a key. It may be beneficial in the future to more accurately
-	// estimate which tombstones cover keys and which do not.
+	// cover a key.
+	//
+	// TODO(jackson): This logic doesn't directly incorporate fixed per-key
+	// overhead (8-byte trailer, plus at least 1 byte encoding the length of the
+	// key and 1 byte encoding the length of the value). This overhead is
+	// indirectly incorporated through the compression ratios, but that results
+	// in the overhead being smeared per key-byte and value-byte, rather than
+	// per-entry. This per-key fixed overhead can be nontrivial, especially for
+	// dense swaths of point tombstones. Give some thought as to whether we
+	// should directly include fixed per-key overhead in the calculations.
 
-	// Calculate the contribution of the key sizes: 1x for the point tombstone
-	// itself and 1x for the key that's expected to exist lower in the LSM.
-	var keysLogicalSize uint64
+	// Below, we calculate the tombstone contributions and the shadowed keys'
+	// contributions separately.
+	var tombstonesLogicalSize float64
+	var shadowedLogicalSize float64
+
+	// 1. Calculate the contribution of the tombstone keys themselves.
 	if props.RawPointTombstoneKeySize > 0 {
-		// This table has a RawPointTombstoneKeySize property, so we know the
-		// exact size of the logical, uncompressed bytes of the point tombstone
-		// keys.
-		keysLogicalSize = 2 * props.RawPointTombstoneKeySize
+		tombstonesLogicalSize += float64(props.RawPointTombstoneKeySize)
 	} else {
 		// This sstable predates the existence of the RawPointTombstoneKeySize
 		// property. We can use the average key size within the file itself and
 		// the count of point deletions to estimate the size.
-		keysLogicalSize = 2 * numPointDels * props.RawKeySize / props.NumEntries
+		tombstonesLogicalSize += float64(numPointDels * props.RawKeySize / props.NumEntries)
 	}
-	// Scale the logical key size by the logical:physical ratio of the file to
+
+	// 2. Calculate the contribution of the keys shadowed by tombstones.
+	//
+	// 2a. First account for keys shadowed by DELSIZED tombstones. THE DELSIZED
+	// tombstones encode the size of both the key and value of the shadowed KV
+	// entries. These sizes are aggregated into a sstable property.
+	shadowedLogicalSize += float64(props.RawPointTombstoneValueSize)
+
+	// 2b. Calculate the contribution of the KV entries shadowed by ordinary DEL
+	// keys.
+	numUnsizedDels := numPointDels - props.NumSizedDeletions
+	{
+		// The shadowed keys have the same exact user keys as the tombstones
+		// themselves, so we can use the `tombstonesLogicalSize` we computed
+		// earlier as an estimate. There's a complication that
+		// `tombstonesLogicalSize` may include DELSIZED keys we already
+		// accounted for.
+		shadowedLogicalSize += float64(tombstonesLogicalSize) / float64(numPointDels) * float64(numUnsizedDels)
+
+		// Calculate the contribution of the deleted values. The caller has
+		// already computed an average logical size (possibly computed across
+		// many sstables).
+		shadowedLogicalSize += float64(numUnsizedDels) * avgValLogicalSize
+	}
+
+	// Scale both tombstone and shadowed totals by logical:physical ratios to
 	// account for compression, metadata overhead, etc.
 	//
 	//      Physical             FileSize
 	//     -----------  = -----------------------
 	//      Logical       RawKeySize+RawValueSize
 	//
-	estimate += (keysLogicalSize * fileSize) / (props.RawKeySize + props.RawValueSize)
-
-	// Calculate the contribution of the deleted values. The caller has already
-	// computed an average physical size (possibly comptued across many
-	// sstables), so there's no need to scale it.
-	estimate += numPointDels * avgValPhysicalSize
-	return estimate
+	return uint64((tombstonesLogicalSize + shadowedLogicalSize) * compressionRatio)
 }
 
-func estimateValuePhysicalSize(fileSize uint64, props *sstable.Properties) (avgValSize uint64) {
+func estimatePhysicalSizes(
+	fileSize uint64, props *sstable.Properties,
+) (avgValLogicalSize, compressionRatio float64) {
 	// RawKeySize and RawValueSize are uncompressed totals. Scale according to
 	// the data size to account for compression, index blocks and metadata
 	// overhead. Eg:
@@ -689,9 +754,9 @@ func estimateValuePhysicalSize(fileSize uint64, props *sstable.Properties) (avgV
 	//   RawKeySize+RawValueSize     NumEntries
 	//
 	uncompressedSum := props.RawKeySize + props.RawValueSize
-	compressionRatio := float64(fileSize) / float64(uncompressedSum)
-	avgCompressedValueSize := (float64(props.RawValueSize) / float64(props.NumEntries)) * compressionRatio
-	return uint64(avgCompressedValueSize)
+	compressionRatio = float64(fileSize) / float64(uncompressedSum)
+	avgValLogicalSize = (float64(props.RawValueSize) / float64(props.NumEntries))
+	return avgValLogicalSize, compressionRatio
 }
 
 // newCombinedDeletionKeyspanIter returns a keyspan.FragmentIterator that

vendor/github.com/cockroachdb/pebble/version_set.go

@@ -293,8 +293,8 @@ func (vs *versionSet) load(
 	}
 	vs.markFileNumUsed(vs.minUnflushedLogNum)
 
-	// Populate the fileBackingMap since we have finished version
-	// edit accumulation.
+	// Populate the fileBackingMap and the FileBacking for virtual sstables since
+	// we have finished version edit accumulation.
 	for _, s := range bve.AddedFileBacking {
 		vs.fileBackingMap[s.DiskFileNum] = s
 	}
@@ -490,6 +490,9 @@ func (vs *versionSet) logAndApply(
 		defer vs.mu.Lock()
 
 		var err error
+		if vs.opts.FormatMajorVersion < ExperimentalFormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 {
+			return errors.AssertionFailedf("MANIFEST cannot contain virtual sstable records due to format major version")
+		}
 		newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE(
 			ve, currentVersion, vs.cmp, vs.opts.Comparer.FormatKey,
 			vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate,
@@ -516,7 +519,7 @@ func (vs *versionSet) logAndApply(
 			return errors.Wrap(err, "MANIFEST next record write failed")
 		}
 
-		// NB: Any error from this point on is considered fatal as we don't now if
+		// NB: Any error from this point on is considered fatal as we don't know if
 		// the MANIFEST write occurred or not. Trying to determine that is
 		// fraught. Instead we rely on the standard recovery mechanism run when a
 		// database is open. In particular, that mechanism generates a new MANIFEST
@@ -569,6 +572,14 @@ func (vs *versionSet) logAndApply(
 	for fileNum, size := range zombies {
 		vs.zombieTables[fileNum] = size
 	}
+	// Update the Deleted bools. We can't use the zombieTables struct for this
+	// as it works on FileBackings, not FileMetadatas.
+	for _, f := range ve.DeletedFiles {
+		f.Deleted = true
+	}
+	for i := range ve.NewFiles {
+		ve.NewFiles[i].Meta.Deleted = false
+	}
 
 	// Install the new version.
 	vs.append(newVersion)
@@ -688,7 +699,6 @@ func (vs *versionSet) createManifest(
 				Level: level,
 				Meta:  meta,
 			})
-			// TODO(bananabrick): Test snapshot changes.
 			if _, ok := dedup[meta.FileBacking.DiskFileNum]; meta.Virtual && !ok {
 				dedup[meta.FileBacking.DiskFileNum] = struct{}{}
 				snapshot.CreatedBackingTables = append(

vendor/github.com/cockroachdb/pebble/vfs/disk_full.go

@@ -228,6 +228,17 @@ func (fs *enospcFS) Open(name string, opts ...OpenOption) (File, error) {
 	return f, err
 }
 
+func (fs *enospcFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
+	f, err := fs.inner.OpenReadWrite(name, opts...)
+	if f != nil {
+		f = &enospcFile{
+			fs:    fs,
+			inner: f,
+		}
+	}
+	return f, err
+}
+
 func (fs *enospcFS) OpenDir(name string) (File, error) {
 	f, err := fs.inner.OpenDir(name)
 	if f != nil {
@@ -375,6 +386,20 @@ func (f *enospcFile) Write(p []byte) (n int, err error) {
 	return n, err
 }
 
+func (f *enospcFile) WriteAt(p []byte, ofs int64) (n int, err error) {
+	gen := f.fs.waitUntilReady()
+
+	n, err = f.inner.WriteAt(p, ofs)
+
+	if err != nil && isENOSPC(err) {
+		f.fs.handleENOSPC(gen)
+		var n2 int
+		n2, err = f.inner.WriteAt(p[n:], ofs+int64(n))
+		n += n2
+	}
+	return n, err
+}
+
 func (f *enospcFile) Prefetch(offset, length int64) error {
 	return f.inner.Prefetch(offset, length)
 }

vendor/github.com/cockroachdb/pebble/vfs/disk_health.go

@@ -219,6 +219,14 @@ func (d *diskHealthCheckingFile) Write(p []byte) (n int, err error) {
 	return n, err
 }
 
+// Write implements the io.WriterAt interface.
+func (d *diskHealthCheckingFile) WriteAt(p []byte, ofs int64) (n int, err error) {
+	d.timeDiskOp(OpTypeWrite, int64(len(p)), func() {
+		n, err = d.file.WriteAt(p, ofs)
+	})
+	return n, err
+}
+
 // Close implements the io.Closer interface.
 func (d *diskHealthCheckingFile) Close() error {
 	d.stopTicker()
@@ -680,6 +688,11 @@ func (d *diskHealthCheckingFS) Open(name string, opts ...OpenOption) (File, erro
 	return d.fs.Open(name, opts...)
 }
 
+// OpenReadWrite implements the FS interface.
+func (d *diskHealthCheckingFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
+	return d.fs.Open(name, opts...)
+}
+
 // OpenDir implements the FS interface.
 func (d *diskHealthCheckingFS) OpenDir(name string) (File, error) {
 	f, err := d.fs.OpenDir(name)

vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go

@@ -46,6 +46,15 @@ func (fs *loggingFS) Open(name string, opts ...OpenOption) (File, error) {
 	return newLoggingFile(f, name, fs.logFn), nil
 }
 
+func (fs *loggingFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
+	fs.logFn("open-read-write: %s", name)
+	f, err := fs.FS.OpenReadWrite(name, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return newLoggingFile(f, name, fs.logFn), nil
+}
+
 func (fs *loggingFS) Link(oldname, newname string) error {
 	fs.logFn("link: %s -> %s", oldname, newname)
 	return fs.FS.Link(oldname, newname)
@@ -137,6 +146,11 @@ func (f *loggingFile) ReadAt(p []byte, offset int64) (int, error) {
 	return f.File.ReadAt(p, offset)
 }
 
+func (f *loggingFile) WriteAt(p []byte, offset int64) (int, error) {
+	f.logFn("write-at(%d, %d): %s", offset, len(p), f.name)
+	return f.File.WriteAt(p, offset)
+}
+
 func (f *loggingFile) Prefetch(offset int64, length int64) error {
 	f.logFn("prefetch(%d, %d): %s", offset, length, f.name)
 	return f.File.Prefetch(offset, length)

vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go

@@ -208,6 +208,7 @@ func (y *MemFS) Create(fullname string) (File, error) {
 			ret = &memFile{
 				n:     n,
 				fs:    y,
+				read:  true,
 				write: true,
 			}
 		}
@@ -262,7 +263,7 @@ func (y *MemFS) Link(oldname, newname string) error {
 	})
 }
 
-func (y *MemFS) open(fullname string) (File, error) {
+func (y *MemFS) open(fullname string, openForWrite bool) (File, error) {
 	var ret *memFile
 	err := y.walk(fullname, func(dir *memNode, frag string, final bool) error {
 		if final {
@@ -275,9 +276,10 @@ func (y *MemFS) open(fullname string) (File, error) {
 			}
 			if n := dir.children[frag]; n != nil {
 				ret = &memFile{
-					n:    n,
-					fs:   y,
-					read: true,
+					n:     n,
+					fs:    y,
+					read:  true,
+					write: openForWrite,
 				}
 			}
 		}
@@ -299,12 +301,22 @@ func (y *MemFS) open(fullname string) (File, error) {
 
 // Open implements FS.Open.
 func (y *MemFS) Open(fullname string, opts ...OpenOption) (File, error) {
-	return y.open(fullname)
+	return y.open(fullname, false /* openForWrite */)
+}
+
+// OpenReadWrite implements FS.OpenReadWrite.
+func (y *MemFS) OpenReadWrite(fullname string, opts ...OpenOption) (File, error) {
+	f, err := y.open(fullname, true /* openForWrite */)
+	pathErr, ok := err.(*os.PathError)
+	if ok && pathErr.Err == oserror.ErrNotExist {
+		return y.Create(fullname)
+	}
+	return f, err
 }
 
 // OpenDir implements FS.OpenDir.
 func (y *MemFS) OpenDir(fullname string) (File, error) {
-	return y.open(fullname)
+	return y.open(fullname, false /* openForWrite */)
 }
 
 // Remove implements FS.Remove.
@@ -702,6 +714,29 @@ func (f *memFile) Write(p []byte) (int, error) {
 	return len(p), nil
 }
 
+func (f *memFile) WriteAt(p []byte, ofs int64) (int, error) {
+	if !f.write {
+		return 0, errors.New("pebble/vfs: file was not created for writing")
+	}
+	if f.n.isDir {
+		return 0, errors.New("pebble/vfs: cannot write a directory")
+	}
+	f.n.mu.Lock()
+	defer f.n.mu.Unlock()
+	f.n.mu.modTime = time.Now()
+
+	for len(f.n.mu.data) < int(ofs)+len(p) {
+		f.n.mu.data = append(f.n.mu.data, 0)
+	}
+
+	n := copy(f.n.mu.data[int(ofs):int(ofs)+len(p)], p)
+	if n != len(p) {
+		panic("stuff")
+	}
+
+	return len(p), nil
+}
+
 func (f *memFile) Prefetch(offset int64, length int64) error { return nil }
 func (f *memFile) Preallocate(offset, length int64) error    { return nil }
 

vendor/github.com/cockroachdb/pebble/vfs/vfs.go

@@ -29,6 +29,8 @@ type File interface {
 	// method *is* allowed to modify the slice passed in, whether temporarily
 	// or permanently. Callers of Write() need to take this into account.
 	io.Writer
+	// WriteAt() is only supported for files that were opened with FS.OpenReadWrite.
+	io.WriterAt
 
 	// Preallocate optionally preallocates storage for `length` at `offset`
 	// within the file. Implementations may choose to do nothing.
@@ -99,6 +101,10 @@ type FS interface {
 	// Open opens the named file for reading. openOptions provides
 	Open(name string, opts ...OpenOption) (File, error)
 
+	// OpenReadWrite opens the named file for reading and writing. If the file
+	// does not exist, it is created.
+	OpenReadWrite(name string, opts ...OpenOption) (File, error)
+
 	// OpenDir opens the named directory for syncing.
 	OpenDir(name string) (File, error)
 
@@ -232,6 +238,18 @@ func (defaultFS) Open(name string, opts ...OpenOption) (File, error) {
 	return file, nil
 }
 
+func (defaultFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
+	osFile, err := os.OpenFile(name, os.O_RDWR|syscall.O_CLOEXEC|os.O_CREATE, 0666)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	file := wrapOSFile(osFile)
+	for _, opt := range opts {
+		opt.Apply(file)
+	}
+	return file, nil
+}
+
 func (defaultFS) Remove(name string) error {
 	return errors.WithStack(os.Remove(name))
 }

vendor/github.com/cockroachdb/redact/internal/buffer/buffer.go

@@ -15,6 +15,7 @@
 package buffer
 
 import (
+	"bytes"
 	origFmt "fmt"
 	"unicode/utf8"
 	"unsafe"
@@ -197,11 +198,16 @@ func (b *Buffer) endRedactable() {
 	if len(b.buf) == 0 {
 		return
 	}
-	p, ok := b.tryGrowByReslice(len(m.EndS))
-	if !ok {
-		p = b.grow(len(m.EndS))
+	if bytes.HasSuffix(b.buf, m.StartBytes) {
+		// Special case: remove a trailing open marker and call it a day.
+		b.buf = b.buf[:len(b.buf)-m.StartLen]
+	} else {
+		p, ok := b.tryGrowByReslice(m.EndLen)
+		if !ok {
+			p = b.grow(m.EndLen)
+		}
+		copy(b.buf[p:], m.EndS)
 	}
-	copy(b.buf[p:], m.EndS)
 	b.markerOpen = false
 }
 
@@ -214,18 +220,23 @@ func (b *Buffer) startWrite() {
 
 // endRedactable adds the closing redaction marker.
 func (b *Buffer) startRedactable() {
-	p, ok := b.tryGrowByReslice(len(m.StartS))
-	if !ok {
-		p = b.grow(len(m.StartS))
+	if bytes.HasSuffix(b.buf, m.EndBytes) {
+		// Special case: remove a trailing closing marker and call it a day.
+		b.buf = b.buf[:len(b.buf)-m.EndLen]
+	} else {
+		p, ok := b.tryGrowByReslice(len(m.StartS))
+		if !ok {
+			p = b.grow(len(m.StartS))
+		}
+		copy(b.buf[p:], m.StartS)
 	}
-	copy(b.buf[p:], m.StartS)
 	b.markerOpen = true
 }
 
 // escapeToEnd escapes occurrences of redaction markers in
 // b.buf[b.validUntil:] and advances b.validUntil until the end.
 func (b *Buffer) escapeToEnd(breakNewLines bool) {
-	b.buf = escape.InternalEscapeBytes(b.buf, b.validUntil, breakNewLines, breakNewLines)
+	b.buf = escape.InternalEscapeBytes(b.buf, b.validUntil, breakNewLines, false /* trim */)
 	b.validUntil = len(b.buf)
 }
 
@@ -313,6 +324,13 @@ func (b *Buffer) Grow(n int) {
 	b.buf = b.buf[:m]
 }
 
+// clone is used in tests.
+func (b *Buffer) clone() *Buffer {
+	c := *b
+	c.buf = append([]byte(nil), b.buf...)
+	return &c
+}
+
 // makeSlice allocates a slice of size n. If the allocation fails, it panics
 // with ErrTooLarge.
 func makeSlice(n int) []byte {

vendor/github.com/cockroachdb/redact/internal/escape/escape.go

@@ -28,6 +28,9 @@ import (
 // If breakNewLines is set, a closing redaction marker
 // is placed before sequences of one or more newline characters,
 // and an open redaction marker is placed afterwards.
+//
+// If strip is set, final newlines and spaces are trimmed from the
+// output.
 func InternalEscapeBytes(b []byte, startLoc int, breakNewLines, strip bool) (res []byte) {
 	// Note: we use len(...RedactableS) and not len(...RedactableBytes)
 	// because the ...S variant is a compile-time constant so this
@@ -69,7 +72,12 @@ func InternalEscapeBytes(b []byte, startLoc int, breakNewLines, strip bool) (res
 				copied = true
 			}
 			res = append(res, b[k:i]...)
-			res = append(res, end...)
+			// Either add an end marker, or elide a start marker immediately prior.
+			if bytes.HasSuffix(res, start) {
+				res = res[:len(res)-ls]
+			} else {
+				res = append(res, end...)
+			}
 			// Advance to the last newline character. We want to forward
 			// them all in a single call to doWrite, for performance.
 			lastNewLine := i

vendor/github.com/cockroachdb/redact/internal/markers/constants.go

@@ -20,8 +20,10 @@ import "regexp"
 const (
 	Start       = '‹'
 	StartS      = string(Start)
+	StartLen    = len(StartS)
 	End         = '›'
 	EndS        = string(End)
+	EndLen      = len(EndS)
 	EscapeMark  = '?'
 	EscapeMarkS = string(EscapeMark)
 	RedactedS   = StartS + "×" + EndS