Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -24,6 +24,14 @@ const (
 	//InternalKeyKindColumnFamilyDeletion     InternalKeyKind = 4
 	//InternalKeyKindColumnFamilyValue        InternalKeyKind = 5
 	//InternalKeyKindColumnFamilyMerge        InternalKeyKind = 6
+
+	// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
+	// solely for compactions (to reduce write amp and space amp). Readers other
+	// than compactions should treat SINGLEDEL as equivalent to a DEL.
+	// Historically, it was simpler for readers other than compactions to treat
+	// SINGLEDEL as equivalent to DEL, but as of the introduction of
+	// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
+	// correctness.
 	InternalKeyKindSingleDelete InternalKeyKind = 7
 	//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
 	//InternalKeyKindBeginPrepareXID          InternalKeyKind = 9
@@ -66,8 +74,15 @@ const (
 	// batch, or in an sstable.
 	InternalKeyKindIngestSST InternalKeyKind = 22
 
-	// This maximum value isn't part of the file format. It's unlikely,
-	// but future extensions may increase this value.
+	// InternalKeyKindDeleteSized keys behave identically to
+	// InternalKeyKindDelete keys, except that they hold an associated uint64
+	// value indicating the (len(key)+len(value)) of the shadowed entry the
+	// tombstone is expected to delete. This value is used to inform compaction
+	// heuristics, but is not required to be accurate for correctness.
+	InternalKeyKindDeleteSized InternalKeyKind = 23
+
+	// This maximum value isn't part of the file format. Future extensions may
+	// increase this value.
 	//
 	// When constructing an internal key to pass to DB.Seek{GE,LE},
 	// internalKeyComparer sorts decreasing by kind (after sorting increasing by
@@ -75,14 +90,19 @@ const (
 	// which sorts 'less than or equal to' any other valid internalKeyKind, when
 	// searching for any kind of internal key formed by a certain user key and
 	// seqNum.
-	InternalKeyKindMax InternalKeyKind = 22
+	InternalKeyKindMax InternalKeyKind = 23
+
+	// Internal to the sstable format. Not exposed by any sstable iterator.
+	// Declared here to prevent definition of valid key kinds that set this bit.
+	InternalKeyKindSSTableInternalObsoleteBit  InternalKeyKind = 64
+	InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
 
 	// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
 	// zero sequence number.
-	InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid)
+	InternalKeyZeroSeqnumMaxTrailer = uint64(255)
 
 	// A marker for an invalid key.
-	InternalKeyKindInvalid InternalKeyKind = 255
+	InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
 
 	// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
 	// which prevents those entries from being excluded from iteration.
@@ -105,6 +125,9 @@ const (
 	InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
 )
 
+// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
+const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
+
 var internalKeyKindNames = []string{
 	InternalKeyKindDelete:         "DEL",
 	InternalKeyKindSet:            "SET",
@@ -118,6 +141,7 @@ var internalKeyKindNames = []string{
 	InternalKeyKindRangeKeyUnset:  "RANGEKEYUNSET",
 	InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
 	InternalKeyKindIngestSST:      "INGESTSST",
+	InternalKeyKindDeleteSized:    "DELSIZED",
 	InternalKeyKindInvalid:        "INVALID",
 }
 
@@ -204,6 +228,7 @@ var kindsMap = map[string]InternalKeyKind{
 	"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
 	"RANGEKEYDEL":   InternalKeyKindRangeKeyDelete,
 	"INGESTSST":     InternalKeyKindIngestSST,
+	"DELSIZED":      InternalKeyKindDeleteSized,
 }
 
 // ParseInternalKey parses the string representation of an internal key. The

vendor/github.com/cockroachdb/pebble/internal/base/metrics.go

@@ -53,6 +53,15 @@ func (tm *ThroughputMetric) Rate() int64 {
 		float64(time.Second))
 }
 
+// Utilization returns a percent [0, 1.0] indicating the percent of time
+// work was performed.
+func (tm *ThroughputMetric) Utilization() float64 {
+	if tm.WorkDuration == 0 {
+		return 0
+	}
+	return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
+}
+
 // GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
 // accumulating samples of that gauge.
 type GaugeSampleMetric struct {

vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go

@@ -4,6 +4,8 @@
 
 package base
 
+import "fmt"
+
 // This file defines sequence numbers that are reserved for foreign keys i.e.
 // internal keys coming from other Pebble instances and existing in shared
 // storage, as those will "slot below" any internal keys added by our own Pebble
@@ -14,41 +16,41 @@ const (
 	// guarantee there are no keys underneath an internal key.
 	SeqNumZero = uint64(0)
 
-	// SeqNumL6Point is the sequence number reserved for foreign point keys in L6.
-	// This sequence number must be lower than SeqNumL6RangeKey for range key
-	// masking to work correctly.
-	SeqNumL6Point = uint64(1)
+	// SeqNumL6 is the sequence number reserved for foreign keys in L6. This seqnum
+	// will be used to expose any range key sets as well as point keys in L6. Range
+	// deletes do not need to be exposed in L6.
+	SeqNumL6 = uint64(1)
 
-	// SeqNumL6RangeKey is the sequence number reserved for foreign range keys in
-	// L6. Only RangeKeySets are expected at this level.
-	SeqNumL6RangeKey = uint64(2)
+	// SeqNumL5 is the sequence number reserved for foreign keys in L5. This seqnum
+	// needs to be greater than SeqNumL6, so that range deletes in L5 can be
+	// exposed at this level and will correctly delete covering points in L6. Also
+	// range key unsets/dels will be exposed at this seqnum and will need to shadow
+	// overlapping range keys in L6.
+	//
+	// Note that we can use the same sequence number for all exposed keys in L5 as
+	// range dels do not delete points at the same seqnum, and range key
+	// unsets/deletes do not coalesce with range key sets at the same seqnum. Range
+	// key masking does not care about the sequence number of overlapping points
+	// (rather, it applies to points based on suffixes), so we can use this seqnum
+	// for all L5 keys exposed.
+	SeqNumL5 = uint64(2)
 
-	// SeqNumL5RangeDel is the sequence number reserved for foreign range deletes
-	// in L5. These keys could delete L6 points.
-	SeqNumL5RangeDel = uint64(3)
-
-	// SeqNumL5Point is the sequence number reserved for foreign point keys in L5.
-	// Any sst-local range deletions would have already been applied to these keys,
-	// so they can safely get a sequence number higher than SeqNumL5RangeDel.
-	// However they must have a sequence number lower than SeqNumL5RangeKey* for
-	// range key masking to work correctly.
-	SeqNumL5Point = uint64(4)
-
-	// SeqNumL5RangeKeyUnsetDel is the sequence number reserved for foreign
-	// range key unsets/deletes in L5. These operations could apply to L6
-	// RangeKeySets, so this sequence number must be > SeqNumL6RangeKey.
-	SeqNumL5RangeKeyUnsetDel = uint64(5)
-
-	// SeqNumL5RangeKeySet is the sequence number reserved for foreign range key
-	// Sets in L5. These operations could apply to L6 RangeKeySets, so this
-	// sequence number must be > SeqNumL6RangeKey. Any SST-local rangekey
-	// unsets/dels have already been applied to them, so their sequence number must
-	// be > SeqNumL5RangeKeyUnsetDel.
-	SeqNumL5RangeKeySet = uint64(6)
-
-	// Sequence numbers 7-9 are reserved for future use.
+	// Sequence numbers 3-9 are reserved for future use.
 
 	// SeqNumStart is the first sequence number assigned to a key written by
 	// ourselves.
 	SeqNumStart = uint64(10)
 )
+
+// SeqNumForLevel returns the appropriate reserved sequence number for keys in
+// foreign sstables at the specified level.
+func SeqNumForLevel(level int) uint64 {
+	switch level {
+	case 5:
+		return SeqNumL5
+	case 6:
+		return SeqNumL6
+	default:
+		panic(fmt.Sprintf("unexpected foreign sstable at level %d", level))
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -62,7 +62,7 @@ type FragmentIterator interface {
 
 // TableNewSpanIter creates a new iterator for range key spans for the given
 // file.
-type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
+type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
 
 // SpanIterOptions is a subset of IterOptions that are necessary to instantiate
 // per-sstable span iterators.
@@ -70,6 +70,9 @@ type SpanIterOptions struct {
 	// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
 	// when iterating over range keys.
 	RangeKeyFilters []base.BlockPropertyFilter
+	// Level specifies the level where this sstable is being read. Must be
+	// specified for foreign (i.e. shared not-created-by-this-instance) sstables.
+	Level manifest.Level
 }
 
 // Iter is an iterator over a set of fragmented spans.

vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go

@@ -98,7 +98,7 @@ func (l *LevelIter) Init(
 ) {
 	l.err = nil
 	l.level = level
-	l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
+	l.tableOpts = opts
 	l.cmp = cmp
 	l.iterFile = nil
 	l.newIter = newIter
@@ -179,7 +179,7 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur
 		return noFileLoaded
 	}
 	if indicator != fileAlreadyLoaded {
-		l.iter, l.err = l.newIter(file, &l.tableOpts)
+		l.iter, l.err = l.newIter(file, l.tableOpts)
 		indicator = newFileLoaded
 	}
 	if l.err != nil {

vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go

@@ -25,49 +25,6 @@ import (
 // MergingIter implementation, but will require a bit of plumbing to thread the
 // Equal function.
 
-// Transformer defines a transformation to be applied to a Span.
-type Transformer interface {
-	// Transform takes a Span as input and writes the transformed Span to the
-	// provided output *Span pointer. The output Span's Keys slice may be reused
-	// by Transform to reduce allocations.
-	Transform(cmp base.Compare, in Span, out *Span) error
-}
-
-// The TransformerFunc type is an adapter to allow the use of ordinary functions
-// as Transformers. If f is a function with the appropriate signature,
-// TransformerFunc(f) is a Transformer that calls f.
-type TransformerFunc func(base.Compare, Span, *Span) error
-
-// Transform calls f(cmp, in, out).
-func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
-	return tf(cmp, in, out)
-}
-
-var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
-	dst.Start, dst.End = s.Start, s.End
-	dst.Keys = append(dst.Keys[:0], s.Keys...)
-	return nil
-})
-
-// VisibleTransform filters keys that are invisible at the provided snapshot
-// sequence number.
-func VisibleTransform(snapshot uint64) Transformer {
-	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
-		dst.Start, dst.End = s.Start, s.End
-		dst.Keys = dst.Keys[:0]
-		for _, k := range s.Keys {
-			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
-			// because a batch's visible span keys are filtered when they're
-			// fragmented. There's no requirement to enforce visibility at
-			// iteration time.
-			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
-				dst.Keys = append(dst.Keys, k)
-			}
-		}
-		return nil
-	})
-}
-
 // MergingIter merges spans across levels of the LSM, exposing an iterator over
 // spans that yields sets of spans fragmented at unique user key boundaries.
 //

vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go

@@ -368,6 +368,7 @@ func (s Span) String() string {
 
 // Pretty returns a formatter for the span.
 func (s Span) Pretty(f base.FormatKey) fmt.Formatter {
+	// TODO(jackson): Take a base.FormatValue to format Key.Value too.
 	return prettySpan{s, f}
 }
 

vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go

@@ -0,0 +1,149 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Transformer defines a transformation to be applied to a Span.
+type Transformer interface {
+	// Transform takes a Span as input and writes the transformed Span to the
+	// provided output *Span pointer. The output Span's Keys slice may be reused
+	// by Transform to reduce allocations.
+	Transform(cmp base.Compare, in Span, out *Span) error
+}
+
+// The TransformerFunc type is an adapter to allow the use of ordinary functions
+// as Transformers. If f is a function with the appropriate signature,
+// TransformerFunc(f) is a Transformer that calls f.
+type TransformerFunc func(base.Compare, Span, *Span) error
+
+// Transform calls f(cmp, in, out).
+func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
+	return tf(cmp, in, out)
+}
+
+var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+	dst.Start, dst.End = s.Start, s.End
+	dst.Keys = append(dst.Keys[:0], s.Keys...)
+	return nil
+})
+
+// VisibleTransform filters keys that are invisible at the provided snapshot
+// sequence number.
+func VisibleTransform(snapshot uint64) Transformer {
+	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+		dst.Start, dst.End = s.Start, s.End
+		dst.Keys = dst.Keys[:0]
+		for _, k := range s.Keys {
+			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
+			// because a batch's visible span keys are filtered when they're
+			// fragmented. There's no requirement to enforce visibility at
+			// iteration time.
+			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
+				dst.Keys = append(dst.Keys, k)
+			}
+		}
+		return nil
+	})
+}
+
+// TransformerIter is a FragmentIterator that applies a Transformer on all
+// returned keys. Used for when a caller needs to apply a transformer on an
+// iterator but does not otherwise need the mergingiter's merging ability.
+type TransformerIter struct {
+	FragmentIterator
+
+	// Transformer is applied on every Span returned by this iterator.
+	Transformer Transformer
+	// Comparer in use for this keyspace.
+	Compare base.Compare
+
+	span Span
+	err  error
+}
+
+func (t *TransformerIter) applyTransform(span *Span) *Span {
+	t.span = Span{
+		Start: t.span.Start[:0],
+		End:   t.span.End[:0],
+		Keys:  t.span.Keys[:0],
+	}
+	if err := t.Transformer.Transform(t.Compare, *span, &t.span); err != nil {
+		t.err = err
+		return nil
+	}
+	return &t.span
+}
+
+// SeekGE implements the FragmentIterator interface.
+func (t *TransformerIter) SeekGE(key []byte) *Span {
+	span := t.FragmentIterator.SeekGE(key)
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// SeekLT implements the FragmentIterator interface.
+func (t *TransformerIter) SeekLT(key []byte) *Span {
+	span := t.FragmentIterator.SeekLT(key)
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// First implements the FragmentIterator interface.
+func (t *TransformerIter) First() *Span {
+	span := t.FragmentIterator.First()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Last implements the FragmentIterator interface.
+func (t *TransformerIter) Last() *Span {
+	span := t.FragmentIterator.Last()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Next implements the FragmentIterator interface.
+func (t *TransformerIter) Next() *Span {
+	span := t.FragmentIterator.Next()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Prev implements the FragmentIterator interface.
+func (t *TransformerIter) Prev() *Span {
+	span := t.FragmentIterator.Prev()
+	if span == nil {
+		return nil
+	}
+	return t.applyTransform(span)
+}
+
+// Error implements the FragmentIterator interface.
+func (t *TransformerIter) Error() error {
+	if t.err != nil {
+		return t.err
+	}
+	return t.FragmentIterator.Error()
+}
+
+// Close implements the FragmentIterator interface.
+func (t *TransformerIter) Close() error {
+	err := t.FragmentIterator.Close()
+	if err != nil {
+		return err
+	}
+	return t.err
+}

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -227,6 +227,12 @@ type FileMetadata struct {
 	// we'd have to write virtual sstable stats to the version edit.
 	Stats TableStats
 
+	// Deleted is set to true if a VersionEdit gets installed that has deleted
+	// this file. Protected by the manifest lock (see versionSet.logLock()).
+	Deleted bool
+
+	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
+	// pick L0 compactions. Only accurate for the most recent Version.
 	SubLevel         int
 	L0Index          int
 	minIntervalIndex int
@@ -235,9 +241,6 @@ type FileMetadata struct {
 	// NB: the alignment of this struct is 8 bytes. We pack all the bools to
 	// ensure an optimal packing.
 
-	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
-	// pick L0 compactions. Only accurate for the most recent Version.
-	//
 	// IsIntraL0Compacting is set to True if this file is part of an intra-L0
 	// compaction. When it's true, IsCompacting must also return true. If
 	// Compacting is true and IsIntraL0Compacting is false for an L0 file, the
@@ -370,6 +373,17 @@ func (m *FileMetadata) InitPhysicalBacking() {
 	}
 }
 
+// InitProviderBacking creates a new FileBacking for a file backed by
+// an objstorage.Provider.
+func (m *FileMetadata) InitProviderBacking(fileNum base.DiskFileNum) {
+	if !m.Virtual {
+		panic("pebble: provider-backed sstables must be virtual")
+	}
+	if m.FileBacking == nil {
+		m.FileBacking = &FileBacking{DiskFileNum: fileNum}
+	}
+}
+
 // ValidateVirtual should be called once the FileMetadata for a virtual sstable
 // is created to verify that the fields of the virtual sstable are sound.
 func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -51,7 +51,9 @@ const (
 	tagMaxColumnFamily  = 203
 
 	// Pebble tags.
-	tagNewFile5 = 104 // Range keys.
+	tagNewFile5            = 104 // Range keys.
+	tagCreatedBackingTable = 105
+	tagRemovedBackingTable = 106
 
 	// The custom tags sub-format used by tagNewFile4 and above.
 	customTagTerminate         = 1
@@ -59,6 +61,7 @@ const (
 	customTagCreationTime      = 6
 	customTagPathID            = 65
 	customTagNonSafeIgnoreMask = 1 << 6
+	customTagVirtual           = 66
 )
 
 // DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -73,6 +76,9 @@ type DeletedFileEntry struct {
 type NewFileEntry struct {
 	Level int
 	Meta  *FileMetadata
+	// BackingFileNum is only set during manifest replay, and only for virtual
+	// sstables.
+	BackingFileNum base.DiskFileNum
 }
 
 // VersionEdit holds the state for an edit to a Version along with other
@@ -143,7 +149,9 @@ type VersionEdit struct {
 
 // Decode decodes an edit from the specified reader.
 //
-// TODO(bananabrick): Support decoding of virtual sstable state.
+// Note that the Decode step will not set the FileBacking for virtual sstables
+// and the responsibility is left to the caller. However, the Decode step will
+// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles.
 func (v *VersionEdit) Decode(r io.Reader) error {
 	br, ok := r.(byteReader)
 	if !ok {
@@ -196,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			// NB: RocksDB does not use compaction pointers anymore.
 
+		case tagRemovedBackingTable:
+			n, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			v.RemovedBackingTables = append(
+				v.RemovedBackingTables, base.FileNum(n).DiskFileNum(),
+			)
+		case tagCreatedBackingTable:
+			dfn, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			size, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			fileBacking := &FileBacking{
+				DiskFileNum: base.FileNum(dfn).DiskFileNum(),
+				Size:        size,
+			}
+			v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking)
 		case tagDeletedFile:
 			level, err := d.readLevel()
 			if err != nil {
@@ -301,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			var markedForCompaction bool
 			var creationTime uint64
+			virtualState := struct {
+				virtual        bool
+				backingFileNum uint64
+			}{}
 			if tag == tagNewFile4 || tag == tagNewFile5 {
 				for {
 					customTag, err := d.readUvarint()
@@ -309,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 					}
 					if customTag == customTagTerminate {
 						break
+					} else if customTag == customTagVirtual {
+						virtualState.virtual = true
+						n, err := d.readUvarint()
+						if err != nil {
+							return err
+						}
+						virtualState.backingFileNum = n
+						continue
 					}
+
 					field, err := d.readBytes()
 					if err != nil {
 						return err
@@ -345,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				SmallestSeqNum:      smallestSeqNum,
 				LargestSeqNum:       largestSeqNum,
 				MarkedForCompaction: markedForCompaction,
+				Virtual:             virtualState.virtual,
 			}
 			if tag != tagNewFile5 { // no range keys present
 				m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
@@ -376,11 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				}
 			}
 			m.boundsSet = true
-			m.InitPhysicalBacking()
-			v.NewFiles = append(v.NewFiles, NewFileEntry{
+			if !virtualState.virtual {
+				m.InitPhysicalBacking()
+			}
+
+			nfe := NewFileEntry{
 				Level: level,
 				Meta:  m,
-			})
+			}
+			if virtualState.virtual {
+				nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum()
+			}
+			v.NewFiles = append(v.NewFiles, nfe)
 
 		case tagPrevLogNumber:
 			n, err := d.readUvarint()
@@ -442,8 +493,6 @@ func (v *VersionEdit) String() string {
 }
 
 // Encode encodes an edit to the specified writer.
-//
-// TODO(bananabrick): Support encoding of virtual sstable state.
 func (v *VersionEdit) Encode(w io.Writer) error {
 	e := versionEditEncoder{new(bytes.Buffer)}
 
@@ -463,6 +512,15 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(tagNextFileNumber)
 		e.writeUvarint(uint64(v.NextFileNum))
 	}
+	for _, dfn := range v.RemovedBackingTables {
+		e.writeUvarint(tagRemovedBackingTable)
+		e.writeUvarint(uint64(dfn.FileNum()))
+	}
+	for _, fileBacking := range v.CreatedBackingTables {
+		e.writeUvarint(tagCreatedBackingTable)
+		e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum()))
+		e.writeUvarint(fileBacking.Size)
+	}
 	// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
 	// even though its value is zero. We detect this by encoding LastSeqNum when
 	// ComparerName is set.
@@ -476,7 +534,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(uint64(x.FileNum))
 	}
 	for _, x := range v.NewFiles {
-		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
+		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
 		var tag uint64
 		switch {
 		case x.Meta.HasRangeKeys:
@@ -529,6 +587,10 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 				e.writeUvarint(customTagNeedsCompaction)
 				e.writeBytes([]byte{1})
 			}
+			if x.Meta.Virtual {
+				e.writeUvarint(customTagVirtual)
+				e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum()))
+			}
 			e.writeUvarint(customTagTerminate)
 		}
 	}
@@ -536,6 +598,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 	return err
 }
 
+// versionEditDecoder should be used to decode version edits.
 type versionEditDecoder struct {
 	byteReader
 }
@@ -634,7 +697,9 @@ type BulkVersionEdit struct {
 	Added   [NumLevels]map[base.FileNum]*FileMetadata
 	Deleted [NumLevels]map[base.FileNum]*FileMetadata
 
-	AddedFileBacking   []*FileBacking
+	// AddedFileBacking is a map to support lookup so that we can populate the
+	// FileBacking of virtual sstables during manifest replay.
+	AddedFileBacking   map[base.DiskFileNum]*FileBacking
 	RemovedFileBacking []base.DiskFileNum
 
 	// AddedByFileNum maps file number to file metadata for all added files
@@ -697,6 +762,16 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		}
 	}
 
+	// Generate state for Added backing files. Note that these must be generated
+	// before we loop through the NewFiles, because we need to populate the
+	// FileBackings which might be used by the NewFiles loop.
+	if b.AddedFileBacking == nil {
+		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
+	}
+	for _, fb := range ve.CreatedBackingTables {
+		b.AddedFileBacking[fb.DiskFileNum] = fb
+	}
+
 	for _, nf := range ve.NewFiles {
 		// A new file should not have been deleted in this or a preceding
 		// VersionEdit at the same level (though files can move across levels).
@@ -705,6 +780,17 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 				return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
 			}
 		}
+		if nf.Meta.Virtual && nf.Meta.FileBacking == nil {
+			// FileBacking for a virtual sstable must only be nil if we're performing
+			// manifest replay.
+			nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum]
+			if nf.Meta.FileBacking == nil {
+				return errors.Errorf("FileBacking for virtual sstable must not be nil")
+			}
+		} else if nf.Meta.FileBacking == nil {
+			return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum)
+		}
+
 		if b.Added[nf.Level] == nil {
 			b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
 		}
@@ -717,9 +803,6 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		}
 	}
 
-	// Generate state for the backing files.
-	b.AddedFileBacking = append(b.AddedFileBacking, ve.CreatedBackingTables...)
-
 	// Since a file can be removed from backing files in exactly one version
 	// edit it is safe to just append without any de-duplication.
 	b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -6,6 +6,7 @@ package rangekey
 
 import (
 	"bytes"
+	"fmt"
 	"math"
 	"sort"
 
@@ -367,3 +368,78 @@ func coalesce(
 	}
 	return nil
 }
+
+// ForeignSSTTransformer implements a keyspan.Transformer for range keys in
+// foreign sstables (i.e. shared sstables not created by us). It is largely
+// similar to the Transform function implemented in UserIteratorConfig in that
+// it calls coalesce to remove range keys shadowed by other range keys, but also
+// retains the range key that does the shadowing. In addition, it elides
+// RangeKey unsets/dels in L6 as they are inapplicable when reading from a
+// different Pebble instance.
+type ForeignSSTTransformer struct {
+	Comparer *base.Comparer
+	Level    int
+	sortBuf  keyspan.KeysBySuffix
+}
+
+// Transform implements the Transformer interface.
+func (f *ForeignSSTTransformer) Transform(
+	cmp base.Compare, s keyspan.Span, dst *keyspan.Span,
+) error {
+	// Apply shadowing of keys.
+	dst.Start = s.Start
+	dst.End = s.End
+	f.sortBuf = keyspan.KeysBySuffix{
+		Cmp:  cmp,
+		Keys: f.sortBuf.Keys[:0],
+	}
+	if err := coalesce(f.Comparer.Equal, &f.sortBuf, math.MaxUint64, s.Keys); err != nil {
+		return err
+	}
+	keys := f.sortBuf.Keys
+	dst.Keys = dst.Keys[:0]
+	for i := range keys {
+		seqNum := keys[i].SeqNum()
+		switch keys[i].Kind() {
+		case base.InternalKeyKindRangeKeySet:
+			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
+				panic("pebble: keys unexpectedly not in ascending suffix order")
+			}
+			switch f.Level {
+			case 5:
+				fallthrough
+			case 6:
+				if seqNum != base.SeqNumForLevel(f.Level) {
+					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
+				}
+			}
+		case base.InternalKeyKindRangeKeyUnset:
+			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
+				panic("pebble: keys unexpectedly not in ascending suffix order")
+			}
+			fallthrough
+		case base.InternalKeyKindRangeKeyDelete:
+			switch f.Level {
+			case 5:
+				// Emit this key.
+				if seqNum != base.SeqNumForLevel(f.Level) {
+					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
+				}
+			case 6:
+				// Skip this key, as foreign sstable in L6 do not need to emit range key
+				// unsets/dels as they do not apply to any other sstables.
+				continue
+			}
+		default:
+			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
+		}
+		dst.Keys = append(dst.Keys, keyspan.Key{
+			Trailer: base.MakeTrailer(seqNum, keys[i].Kind()),
+			Suffix:  keys[i].Suffix,
+			Value:   keys[i].Value,
+		})
+	}
+	// coalesce results in dst.Keys being sorted by Suffix.
+	dst.KeysOrder = keyspan.BySuffixAsc
+	return nil
+}