[WIP] lint a bit, update deps

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -11,11 +11,11 @@ import (
 	"fmt"
 	"io"
 	"sort"
-	"sync/atomic"
 	"time"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // TODO(peter): describe the MANIFEST file format, independently of the C++
@@ -51,7 +51,9 @@ const (
 	tagMaxColumnFamily  = 203
 
 	// Pebble tags.
-	tagNewFile5 = 104 // Range keys.
+	tagNewFile5            = 104 // Range keys.
+	tagCreatedBackingTable = 105
+	tagRemovedBackingTable = 106
 
 	// The custom tags sub-format used by tagNewFile4 and above.
 	customTagTerminate         = 1
@@ -59,6 +61,7 @@ const (
 	customTagCreationTime      = 6
 	customTagPathID            = 65
 	customTagNonSafeIgnoreMask = 1 << 6
+	customTagVirtual           = 66
 )
 
 // DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -73,6 +76,9 @@ type DeletedFileEntry struct {
 type NewFileEntry struct {
 	Level int
 	Meta  *FileMetadata
+	// BackingFileNum is only set during manifest replay, and only for virtual
+	// sstables.
+	BackingFileNum base.DiskFileNum
 }
 
 // VersionEdit holds the state for an edit to a Version along with other
@@ -110,9 +116,42 @@ type VersionEdit struct {
 	// found that there was no overlapping file at the higher level).
 	DeletedFiles map[DeletedFileEntry]*FileMetadata
 	NewFiles     []NewFileEntry
+	// CreatedBackingTables can be used to preserve the FileBacking associated
+	// with a physical sstable. This is useful when virtual sstables in the
+	// latest version are reconstructed during manifest replay, and we also need
+	// to reconstruct the FileBacking which is required by these virtual
+	// sstables.
+	//
+	// INVARIANT: The FileBacking associated with a physical sstable must only
+	// be added as a backing file in the same version edit where the physical
+	// sstable is first virtualized. This means that the physical sstable must
+	// be present in DeletedFiles and that there must be at least one virtual
+	// sstable with the same FileBacking as the physical sstable in NewFiles. A
+	// file must be present in CreatedBackingTables in exactly one version edit.
+	// The physical sstable associated with the FileBacking must also not be
+	// present in NewFiles.
+	CreatedBackingTables []*FileBacking
+	// RemovedBackingTables is used to remove the FileBacking associated with a
+	// virtual sstable. Note that a backing sstable can be removed as soon as
+	// there are no virtual sstables in the latest version which are using the
+	// backing sstable, but the backing sstable doesn't necessarily have to be
+	// removed atomically with the version edit which removes the last virtual
+	// sstable associated with the backing sstable. The removal can happen in a
+	// future version edit.
+	//
+	// INVARIANT: A file must only be added to RemovedBackingTables if it was
+	// added to CreateBackingTables in a prior version edit. The same version
+	// edit also cannot have the same file present in both CreateBackingTables
+	// and RemovedBackingTables. A file must be present in RemovedBackingTables
+	// in exactly one version edit.
+	RemovedBackingTables []base.DiskFileNum
 }
 
 // Decode decodes an edit from the specified reader.
+//
+// Note that the Decode step will not set the FileBacking for virtual sstables
+// and the responsibility is left to the caller. However, the Decode step will
+// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles.
 func (v *VersionEdit) Decode(r io.Reader) error {
 	br, ok := r.(byteReader)
 	if !ok {
@@ -165,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			// NB: RocksDB does not use compaction pointers anymore.
 
+		case tagRemovedBackingTable:
+			n, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			v.RemovedBackingTables = append(
+				v.RemovedBackingTables, base.FileNum(n).DiskFileNum(),
+			)
+		case tagCreatedBackingTable:
+			dfn, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			size, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			fileBacking := &FileBacking{
+				DiskFileNum: base.FileNum(dfn).DiskFileNum(),
+				Size:        size,
+			}
+			v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking)
 		case tagDeletedFile:
 			level, err := d.readLevel()
 			if err != nil {
@@ -270,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			var markedForCompaction bool
 			var creationTime uint64
+			virtualState := struct {
+				virtual        bool
+				backingFileNum uint64
+			}{}
 			if tag == tagNewFile4 || tag == tagNewFile5 {
 				for {
 					customTag, err := d.readUvarint()
@@ -278,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 					}
 					if customTag == customTagTerminate {
 						break
+					} else if customTag == customTagVirtual {
+						virtualState.virtual = true
+						n, err := d.readUvarint()
+						if err != nil {
+							return err
+						}
+						virtualState.backingFileNum = n
+						continue
 					}
+
 					field, err := d.readBytes()
 					if err != nil {
 						return err
@@ -314,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				SmallestSeqNum:      smallestSeqNum,
 				LargestSeqNum:       largestSeqNum,
 				MarkedForCompaction: markedForCompaction,
+				Virtual:             virtualState.virtual,
 			}
 			if tag != tagNewFile5 { // no range keys present
 				m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
@@ -345,10 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				}
 			}
 			m.boundsSet = true
-			v.NewFiles = append(v.NewFiles, NewFileEntry{
+			if !virtualState.virtual {
+				m.InitPhysicalBacking()
+			}
+
+			nfe := NewFileEntry{
 				Level: level,
 				Meta:  m,
-			})
+			}
+			if virtualState.virtual {
+				nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum()
+			}
+			v.NewFiles = append(v.NewFiles, nfe)
 
 		case tagPrevLogNumber:
 			n, err := d.readUvarint()
@@ -367,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 	return nil
 }
 
-// String implements fmt.Stringer for a VersionEdit.
-func (v *VersionEdit) String() string {
+func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
 	var buf bytes.Buffer
 	if v.ComparerName != "" {
 		fmt.Fprintf(&buf, "  comparer:     %s", v.ComparerName)
@@ -399,7 +481,12 @@ func (v *VersionEdit) String() string {
 		fmt.Fprintf(&buf, "  deleted:       L%d %s\n", df.Level, df.FileNum)
 	}
 	for _, nf := range v.NewFiles {
-		fmt.Fprintf(&buf, "  added:         L%d %s", nf.Level, nf.Meta.String())
+		fmt.Fprintf(&buf, "  added:         L%d", nf.Level)
+		if verbose {
+			fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
+		} else {
+			fmt.Fprintf(&buf, " %s", nf.Meta.String())
+		}
 		if nf.Meta.CreationTime != 0 {
 			fmt.Fprintf(&buf, " (%s)",
 				time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -409,6 +496,16 @@ func (v *VersionEdit) String() string {
 	return buf.String()
 }
 
+// DebugString is a more verbose version of String(). Use this in tests.
+func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
+	return v.string(true /* verbose */, fmtKey)
+}
+
+// String implements fmt.Stringer for a VersionEdit.
+func (v *VersionEdit) String() string {
+	return v.string(false /* verbose */, base.DefaultFormatter)
+}
+
 // Encode encodes an edit to the specified writer.
 func (v *VersionEdit) Encode(w io.Writer) error {
 	e := versionEditEncoder{new(bytes.Buffer)}
@@ -429,6 +526,15 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(tagNextFileNumber)
 		e.writeUvarint(uint64(v.NextFileNum))
 	}
+	for _, dfn := range v.RemovedBackingTables {
+		e.writeUvarint(tagRemovedBackingTable)
+		e.writeUvarint(uint64(dfn.FileNum()))
+	}
+	for _, fileBacking := range v.CreatedBackingTables {
+		e.writeUvarint(tagCreatedBackingTable)
+		e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum()))
+		e.writeUvarint(fileBacking.Size)
+	}
 	// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
 	// even though its value is zero. We detect this by encoding LastSeqNum when
 	// ComparerName is set.
@@ -442,7 +548,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(uint64(x.FileNum))
 	}
 	for _, x := range v.NewFiles {
-		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
+		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
 		var tag uint64
 		switch {
 		case x.Meta.HasRangeKeys:
@@ -495,6 +601,10 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 				e.writeUvarint(customTagNeedsCompaction)
 				e.writeBytes([]byte{1})
 			}
+			if x.Meta.Virtual {
+				e.writeUvarint(customTagVirtual)
+				e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum()))
+			}
 			e.writeUvarint(customTagTerminate)
 		}
 	}
@@ -502,6 +612,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 	return err
 }
 
+// versionEditDecoder should be used to decode version edits.
 type versionEditDecoder struct {
 	byteReader
 }
@@ -581,10 +692,30 @@ func (e versionEditEncoder) writeUvarint(u uint64) {
 
 // BulkVersionEdit summarizes the files added and deleted from a set of version
 // edits.
+//
+// INVARIANTS:
+// No file can be added to a level more than once. This is true globally, and
+// also true for all of the calls to Accumulate for a single bulk version edit.
+//
+// No file can be removed from a level more than once. This is true globally,
+// and also true for all of the calls to Accumulate for a single bulk version
+// edit.
+//
+// A file must not be added and removed from a given level in the same version
+// edit.
+//
+// A file that is being removed from a level must have been added to that level
+// before (in a prior version edit). Note that a given file can be deleted from
+// a level and added to another level in a single version edit
 type BulkVersionEdit struct {
-	Added   [NumLevels][]*FileMetadata
+	Added   [NumLevels]map[base.FileNum]*FileMetadata
 	Deleted [NumLevels]map[base.FileNum]*FileMetadata
 
+	// AddedFileBacking is a map to support lookup so that we can populate the
+	// FileBacking of virtual sstables during manifest replay.
+	AddedFileBacking   map[base.DiskFileNum]*FileBacking
+	RemovedFileBacking []base.DiskFileNum
+
 	// AddedByFileNum maps file number to file metadata for all added files
 	// from accumulated version edits. AddedByFileNum is only populated if set
 	// to non-nil by a caller. It must be set to non-nil when replaying
@@ -603,6 +734,19 @@ type BulkVersionEdit struct {
 
 // Accumulate adds the file addition and deletions in the specified version
 // edit to the bulk edit's internal state.
+//
+// INVARIANTS:
+// If a file is added to a given level in a call to Accumulate and then removed
+// from that level in a subsequent call, the file will not be present in the
+// resulting BulkVersionEdit.Deleted for that level.
+//
+// After accumulation of version edits, the bulk version edit may have
+// information about a file which has been deleted from a level, but it may
+// not have information about the same file added to the same level. The add
+// could've occurred as part of a previous bulk version edit. In this case,
+// the deleted file must be present in BulkVersionEdit.Deleted, at the end
+// of the accumulation, because we need to decrease the refcount of the
+// deleted file in Apply.
 func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 	for df, m := range ve.DeletedFiles {
 		dmap := b.Deleted[df.Level]
@@ -624,7 +768,28 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		if m.MarkedForCompaction {
 			b.MarkedForCompactionCountDiff--
 		}
-		dmap[df.FileNum] = m
+		if _, ok := b.Added[df.Level][df.FileNum]; !ok {
+			dmap[df.FileNum] = m
+		} else {
+			// Present in b.Added for the same level.
+			delete(b.Added[df.Level], df.FileNum)
+		}
+	}
+
+	// Generate state for Added backing files. Note that these must be generated
+	// before we loop through the NewFiles, because we need to populate the
+	// FileBackings which might be used by the NewFiles loop.
+	if b.AddedFileBacking == nil {
+		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
+	}
+	for _, fb := range ve.CreatedBackingTables {
+		if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
+			// There is already a FileBacking associated with fb.DiskFileNum.
+			// This should never happen. There must always be only one FileBacking
+			// associated with a backing sstable.
+			panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
+		}
+		b.AddedFileBacking[fb.DiskFileNum] = fb
 	}
 
 	for _, nf := range ve.NewFiles {
@@ -635,7 +800,21 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 				return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
 			}
 		}
-		b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta)
+		if nf.Meta.Virtual && nf.Meta.FileBacking == nil {
+			// FileBacking for a virtual sstable must only be nil if we're performing
+			// manifest replay.
+			nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum]
+			if nf.Meta.FileBacking == nil {
+				return errors.Errorf("FileBacking for virtual sstable must not be nil")
+			}
+		} else if nf.Meta.FileBacking == nil {
+			return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum)
+		}
+
+		if b.Added[nf.Level] == nil {
+			b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
+		}
+		b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta
 		if b.AddedByFileNum != nil {
 			b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta
 		}
@@ -643,36 +822,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 			b.MarkedForCompactionCountDiff++
 		}
 	}
+
+	// Since a file can be removed from backing files in exactly one version
+	// edit it is safe to just append without any de-duplication.
+	b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)
+
 	return nil
 }
 
+// AccumulateIncompleteAndApplySingleVE should be called if a single version edit
+// is to be applied to the provided curr Version and if the caller needs to
+// update the versionSet.zombieTables map. This function exists separately from
+// BulkVersionEdit.Apply because it is easier to reason about properties
+// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we
+// know that exactly one version edit is being accumulated.
+//
+// Note that the version edit passed into this function may be incomplete
+// because compactions don't have the ref counting information necessary to
+// populate VersionEdit.RemovedBackingTables. This function will complete such a
+// version edit by populating RemovedBackingTables.
+//
+// Invariant: Any file being deleted through ve must belong to the curr Version.
+// We can't have a delete for some arbitrary file which does not exist in curr.
+func AccumulateIncompleteAndApplySingleVE(
+	ve *VersionEdit,
+	curr *Version,
+	cmp Compare,
+	formatKey base.FormatKey,
+	flushSplitBytes int64,
+	readCompactionRate int64,
+	backingStateMap map[base.DiskFileNum]*FileBacking,
+	addBackingFunc func(*FileBacking),
+	removeBackingFunc func(base.DiskFileNum),
+	orderingInvariants OrderingInvariants,
+) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
+	if len(ve.RemovedBackingTables) != 0 {
+		panic("pebble: invalid incomplete version edit")
+	}
+	var b BulkVersionEdit
+	err := b.Accumulate(ve)
+	if err != nil {
+		return nil, nil, err
+	}
+	zombies = make(map[base.DiskFileNum]uint64)
+	v, err := b.Apply(
+		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, s := range b.AddedFileBacking {
+		addBackingFunc(s)
+	}
+
+	for fileNum := range zombies {
+		if _, ok := backingStateMap[fileNum]; ok {
+			// This table was backing some virtual sstable in the latest version,
+			// but is now a zombie. We add RemovedBackingTables entries for
+			// these, before the version edit is written to disk.
+			ve.RemovedBackingTables = append(
+				ve.RemovedBackingTables, fileNum,
+			)
+			removeBackingFunc(fileNum)
+		}
+	}
+	return v, zombies, nil
+}
+
 // Apply applies the delta b to the current version to produce a new
 // version. The new version is consistent with respect to the comparer cmp.
 //
 // curr may be nil, which is equivalent to a pointer to a zero version.
 //
-// On success, a map of zombie files containing the file numbers and sizes of
-// deleted files is returned. These files are considered zombies because they
-// are no longer referenced by the returned Version, but cannot be deleted from
-// disk as they are still in use by the incoming Version.
+// On success, if a non-nil zombies map is provided to Apply, the map is updated
+// with file numbers and files sizes of deleted files. These files are
+// considered zombies because they are no longer referenced by the returned
+// Version, but cannot be deleted from disk as they are still in use by the
+// incoming Version.
 func (b *BulkVersionEdit) Apply(
 	curr *Version,
 	cmp Compare,
 	formatKey base.FormatKey,
 	flushSplitBytes int64,
 	readCompactionRate int64,
-) (_ *Version, zombies map[base.FileNum]uint64, _ error) {
-	addZombie := func(fileNum base.FileNum, size uint64) {
-		if zombies == nil {
-			zombies = make(map[base.FileNum]uint64)
-		}
-		zombies[fileNum] = size
-	}
-	// The remove zombie function is used to handle tables that are moved from
-	// one level to another during a version edit (i.e. a "move" compaction).
-	removeZombie := func(fileNum base.FileNum) {
+	zombies map[base.DiskFileNum]uint64,
+	orderingInvariants OrderingInvariants,
+) (*Version, error) {
+	addZombie := func(state *FileBacking) {
 		if zombies != nil {
-			delete(zombies, fileNum)
+			zombies[state.DiskFileNum] = state.Size
+		}
+	}
+	removeZombie := func(state *FileBacking) {
+		if zombies != nil {
+			delete(zombies, state.DiskFileNum)
 		}
 	}
 
@@ -684,7 +928,7 @@ func (b *BulkVersionEdit) Apply(
 	}
 	v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff
 	if v.Stats.MarkedForCompaction < 0 {
-		return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
+		return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
 	}
 
 	for level := range v.Levels {
@@ -705,7 +949,7 @@ func (b *BulkVersionEdit) Apply(
 				// Initialize L0Sublevels.
 				if curr == nil || curr.L0Sublevels == nil {
 					if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
-						return nil, nil, errors.Wrap(err, "pebble: internal error")
+						return nil, errors.Wrap(err, "pebble: internal error")
 					}
 				} else {
 					v.L0Sublevels = curr.L0Sublevels
@@ -718,47 +962,70 @@ func (b *BulkVersionEdit) Apply(
 		// Some edits on this level.
 		lm := &v.Levels[level]
 		lmRange := &v.RangeKeyLevels[level]
-		addedFiles := b.Added[level]
-		deletedMap := b.Deleted[level]
-		if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
-			return nil, nil, base.CorruptionErrorf(
+
+		addedFilesMap := b.Added[level]
+		deletedFilesMap := b.Deleted[level]
+		if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 {
+			return nil, base.CorruptionErrorf(
 				"pebble: internal error: No current or added files but have deleted files: %d",
-				errors.Safe(len(deletedMap)))
+				errors.Safe(len(deletedFilesMap)))
 		}
 
-		// NB: addedFiles may be empty and it also is not necessarily
-		// internally consistent: it does not reflect deletions in deletedMap.
+		// NB: addedFilesMap may be empty. If a file is present in addedFilesMap
+		// for a level, it won't be present in deletedFilesMap for the same
+		// level.
 
-		for _, f := range deletedMap {
-			addZombie(f.FileNum, f.Size)
-			if obsolete := v.Levels[level].tree.Delete(f); obsolete {
+		for _, f := range deletedFilesMap {
+			if obsolete := v.Levels[level].remove(f); obsolete {
 				// Deleting a file from the B-Tree may decrement its
 				// reference count. However, because we cloned the
 				// previous level's B-Tree, this should never result in a
 				// file's reference count dropping to zero.
 				err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
-				return nil, nil, err
+				return nil, err
 			}
 			if f.HasRangeKeys {
-				if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
+				if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
 					// Deleting a file from the B-Tree may decrement its
 					// reference count. However, because we cloned the
 					// previous level's B-Tree, this should never result in a
 					// file's reference count dropping to zero.
 					err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum)
-					return nil, nil, err
+					return nil, err
 				}
 			}
+
+			// Note that a backing sst will only become a zombie if the
+			// references to it in the latest version is 0. We will remove the
+			// backing sst from the zombie list in the next loop if one of the
+			// addedFiles in any of the levels is referencing the backing sst.
+			// This is possible if a physical sstable is virtualized, or if it
+			// is moved.
+			latestRefCount := f.LatestRefs()
+			if latestRefCount <= 0 {
+				// If a file is present in deletedFilesMap for a level, then it
+				// must have already been added to the level previously, which
+				// means that its latest ref count cannot be 0.
+				err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum)
+				return nil, err
+			} else if f.LatestUnref() == 0 {
+				addZombie(f.FileBacking)
+			}
 		}
 
+		addedFiles := make([]*FileMetadata, 0, len(addedFilesMap))
+		for _, f := range addedFilesMap {
+			addedFiles = append(addedFiles, f)
+		}
+		// Sort addedFiles by file number. This isn't necessary, but tests which
+		// replay invalid manifests check the error output, and the error output
+		// depends on the order in which files are added to the btree.
+		sort.Slice(addedFiles, func(i, j int) bool {
+			return addedFiles[i].FileNum < addedFiles[j].FileNum
+		})
+
 		var sm, la *FileMetadata
 		for _, f := range addedFiles {
-			if _, ok := deletedMap[f.FileNum]; ok {
-				// Already called addZombie on this file in the preceding
-				// loop, so we don't need to do it here.
-				continue
-			}
-
 			// NB: allowedSeeks is used for read triggered compactions. It is set using
 			// Options.Experimental.ReadCompactionRate which defaults to 32KB.
 			var allowedSeeks int64
@@ -768,20 +1035,23 @@ func (b *BulkVersionEdit) Apply(
 			if allowedSeeks < 100 {
 				allowedSeeks = 100
 			}
-			atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks)
+			f.AllowedSeeks.Store(allowedSeeks)
 			f.InitAllowedSeeks = allowedSeeks
 
-			err := lm.tree.Insert(f)
+			err := lm.insert(f)
+			// We're adding this file to the new version, so increment the
+			// latest refs count.
+			f.LatestRef()
 			if err != nil {
-				return nil, nil, errors.Wrap(err, "pebble")
+				return nil, errors.Wrap(err, "pebble")
 			}
 			if f.HasRangeKeys {
-				err = lmRange.tree.Insert(f)
+				err = lmRange.insert(f)
 				if err != nil {
-					return nil, nil, errors.Wrap(err, "pebble")
+					return nil, errors.Wrap(err, "pebble")
 				}
 			}
-			removeZombie(f.FileNum)
+			removeZombie(f.FileBacking)
 			// Track the keys with the smallest and largest keys, so that we can
 			// check consistency of the modified span.
 			if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 {
@@ -793,27 +1063,36 @@ func (b *BulkVersionEdit) Apply(
 		}
 
 		if level == 0 {
-			if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 {
+			if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 {
 				// Flushes and ingestions that do not delete any L0 files do not require
 				// a regeneration of L0Sublevels from scratch. We can instead generate
 				// it incrementally.
 				var err error
 				// AddL0Files requires addedFiles to be sorted in seqnum order.
-				addedFiles = append([]*FileMetadata(nil), addedFiles...)
 				SortBySeqNum(addedFiles)
 				v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0])
 				if errors.Is(err, errInvalidL0SublevelsOpt) {
 					err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes)
+				} else if invariants.Enabled && err == nil {
+					copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes)
+					if err != nil {
+						panic(fmt.Sprintf("error when regenerating sublevels: %s", err))
+					}
+					s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels)
+					s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels)
+					if s1 != s2 {
+						panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2))
+					}
 				}
 				if err != nil {
-					return nil, nil, errors.Wrap(err, "pebble: internal error")
+					return nil, errors.Wrap(err, "pebble: internal error")
 				}
 				v.L0SublevelFiles = v.L0Sublevels.Levels
 			} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
-			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			continue
 		}
@@ -833,10 +1112,10 @@ func (b *BulkVersionEdit) Apply(
 					end.Prev()
 				}
 			})
-			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 		}
 	}
-	return v, zombies, nil
+	return v, nil
 }