aleesa-bot/aleesa-irc-go
0
Fork 0
Code Activity

Upgrade versions of vendored libs

Browse Source
This commit is contained in:
eleksir
2024-08-03 21:26:21 +03:00
parent 9bac4fb257
commit b6f8cf9155
812 changed files with 56387 additions and 31548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored
View File

@@ -108,7 +108,7 @@ type annotation struct {
}
type leafNode struct {
ref int32
ref atomic.Int32
count int16
leaf bool
// subtreeCount holds the count of files in the entire subtree formed by
@@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node {
func newLeafNode() *node {
n := leafToNode(new(leafNode))
n.leaf = true
n.ref = 1
n.ref.Store(1)
return n
}
func newNode() *node {
n := new(node)
n.ref = 1
n.ref.Store(1)
return n
}
@@ -159,7 +159,7 @@ func newNode() *node {
// When a node is cloned, the provided pointer will be redirected to the new
// mutable node.
func mut(n **node) *node {
if atomic.LoadInt32(&(*n).ref) == 1 {
if (*n).ref.Load() == 1 {
// Exclusive ownership. Can mutate in place.
// Whenever a node will be mutated, reset its annotations to be marked
@@ -186,7 +186,7 @@ func mut(n **node) *node {
// incRef acquires a reference to the node.
func (n *node) incRef() {
atomic.AddInt32(&n.ref, 1)
n.ref.Add(1)
}
// decRef releases a reference to the node. If requested, the method will unref
@@ -195,8 +195,8 @@ func (n *node) incRef() {
// new nodes pass contentsToo=false to preserve existing reference counts during
// operations that should yield a net-zero change to descendant refcounts.
// When a node is released, its contained files are dereferenced.
func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if atomic.AddInt32(&n.ref, -1) > 0 {
func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
if n.ref.Add(-1) > 0 {
// Other references remain. Can't free.
return
}
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if obsolete == nil {
panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
}
*obsolete = append(*obsolete, f)
// Reference counting is performed on the FileBacking. In the case
// of a virtual sstable, this reference counting is performed on
// a FileBacking which is shared by every single virtual sstable
// with the same backing sstable. If the reference count hits 0,
// then we know that the FileBacking won't be required by any
// sstable in Pebble, and that the backing sstable can be deleted.
*obsolete = append(*obsolete, f.FileBacking)
}
}
if !n.leaf {
@@ -761,8 +767,8 @@ type btree struct {
// Release dereferences and clears the root node of the btree, removing all
// items from the btree. In doing so, it decrements contained file counts.
// It returns a slice of newly obsolete files, if any.
func (t *btree) Release() (obsolete []*FileMetadata) {
// It returns a slice of newly obsolete backing files, if any.
func (t *btree) Release() (obsolete []*FileBacking) {
if t.root != nil {
t.root.decRef(true /* contentsToo */, &obsolete)
t.root = nil

9
vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go generated vendored
View File

@@ -712,7 +712,7 @@ func (s *L0Sublevels) calculateFlushSplitKeys(flushSplitMaxBytes int64) {
// InitCompactingFileInfo initializes internal flags relating to compacting
// files. Must be called after sublevel initialization.
//
// Requires DB.mu to be held.
// Requires DB.mu *and* the manifest lock to be held.
func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
for i := range s.orderedIntervals {
s.orderedIntervals[i].compactingFileCount = 0
@@ -735,6 +735,13 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
if !f.IsCompacting() {
continue
}
if invariants.Enabled {
if s.cmp(s.orderedIntervals[f.minIntervalIndex].startKey.key, f.Smallest.UserKey) != 0 || s.cmp(s.orderedIntervals[f.maxIntervalIndex+1].startKey.key, f.Largest.UserKey) != 0 {
panic(fmt.Sprintf("file %s has inconsistent L0 Sublevel interval bounds: %s-%s, %s-%s", f.FileNum,
s.orderedIntervals[f.minIntervalIndex].startKey.key, s.orderedIntervals[f.maxIntervalIndex+1].startKey.key,
f.Smallest.UserKey, f.Largest.UserKey))
}
}
for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
interval := &s.orderedIntervals[i]
interval.compactingFileCount++

77
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored
View File

@@ -15,20 +15,28 @@ import (
// LevelMetadata contains metadata for all of the files within
// a level of the LSM.
type LevelMetadata struct {
level int
tree btree
level int
totalSize uint64
// NumVirtual is the number of virtual sstables in the level.
NumVirtual uint64
// VirtualSize is the size of the virtual sstables in the level.
VirtualSize uint64
tree btree
}
// clone makes a copy of the level metadata, implicitly increasing the ref
// count of every file contained within lm.
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
tree: lm.tree.Clone(),
level: lm.level,
totalSize: lm.totalSize,
NumVirtual: lm.NumVirtual,
VirtualSize: lm.VirtualSize,
tree: lm.tree.Clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
return lm.tree.Release()
}
@@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
var lm LevelMetadata
lm.level = level
lm.tree, _ = makeBTree(bcmp, files)
for _, f := range files {
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
}
return lm
}
@@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
return t, newLevelSlice(t.Iter())
}
func (lm *LevelMetadata) insert(f *FileMetadata) error {
if err := lm.tree.Insert(f); err != nil {
return err
}
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
return nil
}
func (lm *LevelMetadata) remove(f *FileMetadata) bool {
lm.totalSize -= f.Size
if f.Virtual {
lm.NumVirtual--
lm.VirtualSize -= f.Size
}
return lm.tree.Delete(f)
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.Count() == 0
@@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int {
return lm.tree.Count()
}
// Size returns the cumulative size of all the files within the level.
func (lm *LevelMetadata) Size() uint64 {
return lm.totalSize
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.Iter()}
@@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
return sum
}
// NumVirtual returns the number of virtual sstables in the level. Its runtime is
// linear in the length of the slice.
func (ls *LevelSlice) NumVirtual() uint64 {
var n uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
n++
}
}
return n
}
// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
// level.
func (ls *LevelSlice) VirtualSizeSum() uint64 {
var sum uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
sum += f.Size
}
}
return sum
}
// Reslice constructs a new slice backed by the same underlying level, with
// new start and end positions. Reslice invokes the provided function, passing
// two LevelIterators: one positioned to i's inclusive start and one

180
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored
View File

@@ -49,6 +49,9 @@ type TableStats struct {
// The number of point and range deletion entries in the table.
NumDeletions uint64
// NumRangeKeySets is the total number of range key sets in the table.
//
// NB: If there's a chance that the sstable contains any range key sets,
// then NumRangeKeySets must be > 0.
NumRangeKeySets uint64
// Estimate of the total disk space that may be dropped by this table's
// point deletions by compacting them.
@@ -169,22 +172,13 @@ type FileMetadata struct {
// FileNum is the file number.
//
// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
//
// TODO(bananabrick): Consider creating separate types for
// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
// an indentifier for the FileMetadata in Pebble, and also as a handle to
// perform reads and writes. We should ensure through types that
// FileMetadata.FileNum isn't used to perform reads, and that
// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
FileNum base.FileNum
// Size is the size of the file, in bytes. Size is an approximate value for
// virtual sstables.
//
// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
//
// TODO(bananabrick): Size is currently used in metrics, and for many key
// Pebble level heuristics. Make sure that the heuristics will still work
// appropriately with an approximate value of size.
// INVARIANTS:
// - When !FileMetadata.Virtual, Size == FileBacking.Size.
// - Size should be non-zero. Size 0 virtual sstables must not be created.
Size uint64
// File creation time in seconds since the epoch (1970-01-01 00:00:00
// UTC). For ingested sstables, this corresponds to the time the file was
@@ -227,6 +221,8 @@ type FileMetadata struct {
// we'd have to write virtual sstable stats to the version edit.
Stats TableStats
// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
// pick L0 compactions. Only accurate for the most recent Version.
SubLevel int
L0Index int
minIntervalIndex int
@@ -235,9 +231,6 @@ type FileMetadata struct {
// NB: the alignment of this struct is 8 bytes. We pack all the bools to
// ensure an optimal packing.
// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
// pick L0 compactions. Only accurate for the most recent Version.
//
// IsIntraL0Compacting is set to True if this file is part of an intra-L0
// compaction. When it's true, IsCompacting must also return true. If
// Compacting is true and IsIntraL0Compacting is false for an L0 file, the
@@ -289,6 +282,34 @@ type PhysicalFileMeta struct {
// VirtualFileMeta is used by functions which want a guarantee that their input
// belongs to a virtual sst and not a physical sst.
//
// A VirtualFileMeta inherits all the same fields as a FileMetadata. These
// fields have additional invariants imposed on them, and/or slightly varying
// meanings:
// - Smallest and Largest (and their counterparts
// {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a
// key at that exact bound. We make the effort to determine the next smallest
// or largest key in an sstable after virtualizing it, to maintain this
// tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it
// could mean that a rangedel or range key ends at that user key, or has been
// truncated to that user key.
// - One invariant is that if a rangedel or range key is truncated on its
// upper bound, the virtual sstable *must* have a rangedel or range key
// sentinel key as its upper bound. This is because truncation yields
// an exclusive upper bound for the rangedel/rangekey, and if there are
// any points at that exclusive upper bound within the same virtual
// sstable, those could get uncovered by this truncation. We enforce this
// invariant in calls to keyspan.Truncate.
// - Size is an estimate of the size of the virtualized portion of this sstable.
// The underlying file's size is stored in FileBacking.Size, though it could
// also be estimated or could correspond to just the referenced portion of
// a file (eg. if the file originated on another node).
// - Size must be > 0.
// - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
// This means that all keys in the virtual sstable must have seqnums within
// [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
// a key with a seqnum at either of the bounds. Calculating tight seqnum
// bounds would be too expensive and deliver little value.
//
// NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
type VirtualFileMeta struct {
*FileMetadata
@@ -370,6 +391,17 @@ func (m *FileMetadata) InitPhysicalBacking() {
}
}
// InitProviderBacking creates a new FileBacking for a file backed by
// an objstorage.Provider.
func (m *FileMetadata) InitProviderBacking(fileNum base.DiskFileNum) {
if !m.Virtual {
panic("pebble: provider-backed sstables must be virtual")
}
if m.FileBacking == nil {
m.FileBacking = &FileBacking{DiskFileNum: fileNum}
}
}
// ValidateVirtual should be called once the FileMetadata for a virtual sstable
// is created to verify that the fields of the virtual sstable are sound.
func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {
@@ -575,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive
return true
}
// ContainedWithinSpan returns true if the file key range completely overlaps with the
// given range ("end" is assumed to exclusive).
func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool {
lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end)
return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel()))
}
// ContainsKeyType returns whether or not the file contains keys of the provided
// type.
func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
@@ -672,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
if !verbose {
return b.String()
}
fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum)
if m.HasPointKeys {
fmt.Fprintf(&b, " points:[%s-%s]",
m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format))
@@ -687,7 +727,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
// representation.
func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
// Split lines of the form:
// 000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
// 000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...]
fields := strings.FieldsFunc(s, func(c rune) bool {
switch c {
case ':', '[', '-', ']':
@@ -702,6 +742,19 @@ func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
m := &FileMetadata{}
for len(fields) > 0 {
prefix := fields[0]
if prefix == "seqnums" {
smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum
fields = fields[3:]
continue
}
smallest := base.ParsePrettyInternalKey(fields[1])
largest := base.ParsePrettyInternalKey(fields[2])
switch prefix {
@@ -969,6 +1022,9 @@ func NewVersion(
} else {
v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp)
}
for _, f := range files[l] {
v.Levels[l].totalSize += f.Size
}
}
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
panic(err)
@@ -1001,7 +1057,7 @@ func NewVersion(
// key in a higher level table that has both the same user key and a higher
// sequence number.
type Version struct {
refs int32
refs atomic.Int32
// The level 0 sstables are organized in a series of sublevels. Similar to
// the seqnum invariant in normal levels, there is no internal key in a
@@ -1131,12 +1187,12 @@ func ParseVersionDebug(
// Refs returns the number of references to the version.
func (v *Version) Refs() int32 {
return atomic.LoadInt32(&v.refs)
return v.refs.Load()
}
// Ref increments the version refcount.
func (v *Version) Ref() {
atomic.AddInt32(&v.refs, 1)
v.refs.Add(1)
}
// Unref decrements the version refcount. If the last reference to the version
@@ -1144,16 +1200,11 @@ func (v *Version) Ref() {
// Deleted callback is invoked. Requires that the VersionList mutex is NOT
// locked.
func (v *Version) Unref() {
if atomic.AddInt32(&v.refs, -1) == 0 {
if v.refs.Add(-1) == 0 {
l := v.list
l.mu.Lock()
l.Remove(v)
obsolete := v.unrefFiles()
fileBacking := make([]*FileBacking, len(obsolete))
for i, f := range obsolete {
fileBacking[i] = f.FileBacking
}
v.Deleted(fileBacking)
v.Deleted(v.unrefFiles())
l.mu.Unlock()
}
}
@@ -1163,19 +1214,14 @@ func (v *Version) Unref() {
// the Deleted callback is invoked. Requires that the VersionList mutex is
// already locked.
func (v *Version) UnrefLocked() {
if atomic.AddInt32(&v.refs, -1) == 0 {
if v.refs.Add(-1) == 0 {
v.list.Remove(v)
obsolete := v.unrefFiles()
fileBacking := make([]*FileBacking, len(obsolete))
for i, f := range obsolete {
fileBacking[i] = f.FileBacking
}
v.Deleted(fileBacking)
v.Deleted(v.unrefFiles())
}
}
func (v *Version) unrefFiles() []*FileMetadata {
var obsolete []*FileMetadata
func (v *Version) unrefFiles() []*FileBacking {
var obsolete []*FileBacking
for _, lm := range v.Levels {
obsolete = append(obsolete, lm.release()...)
}
@@ -1307,16 +1353,20 @@ func (v *Version) Overlaps(
// CheckOrdering checks that the files are consistent with respect to
// increasing file numbers (for level 0 files) and increasing and non-
// overlapping internal key ranges (for level non-0 files).
func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
func (v *Version) CheckOrdering(
cmp Compare, format base.FormatKey, order OrderingInvariants,
) error {
for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
sublevelIter := v.L0SublevelFiles[sublevel].Iter()
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
// Sublevels have NEVER allowed split user keys, so we can pass
// ProhibitSplitUserKeys.
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
for level, lm := range v.Levels {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
@@ -1385,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
v.list = nil // avoid memory leaks
}
// OrderingInvariants dictates the file ordering invariants active.
type OrderingInvariants int8
const (
// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
// contain the same user key.
ProhibitSplitUserKeys OrderingInvariants = iota
// AllowSplitUserKeys indicates that adjacent files within a level may
// contain the same user key. This is only allowed by historical format
// major versions.
//
// TODO(jackson): Remove.
AllowSplitUserKeys
)
// CheckOrdering checks that the files are consistent with respect to
// seqnums (for level 0 files -- see detailed comment below) and increasing and non-
// overlapping internal key ranges (for non-level 0 files).
func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
//
// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
// inclusive of the same user key. Pebble no longer creates version edits
// installing such files, and Pebble databases with sufficiently high format
// major version should no longer have any such files within their LSM.
// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
// earlier format major versions.
func CheckOrdering(
cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
) error {
// The invariants to check for L0 sublevels are the same as the ones to
// check for all other levels. However, if L0 is not organized into
// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1466,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
// What's considered "overlapping" is dependent on the format
// major version. If ordering=ProhibitSplitUserKeys, then both
// files cannot contain keys with the same user keys. If the
// bounds have the same user key, the previous file's boundary
// must have a Trailer indicating that it's exclusive.
switch ordering {
case AllowSplitUserKeys:
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
case ProhibitSplitUserKeys:
if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
default:
panic("unreachable")
}
}
}

156
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored
View File

@@ -51,7 +51,9 @@ const (
tagMaxColumnFamily = 203
// Pebble tags.
tagNewFile5 = 104 // Range keys.
tagNewFile5 = 104 // Range keys.
tagCreatedBackingTable = 105
tagRemovedBackingTable = 106
// The custom tags sub-format used by tagNewFile4 and above.
customTagTerminate = 1
@@ -59,6 +61,7 @@ const (
customTagCreationTime = 6
customTagPathID = 65
customTagNonSafeIgnoreMask = 1 << 6
customTagVirtual = 66
)
// DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -73,6 +76,9 @@ type DeletedFileEntry struct {
type NewFileEntry struct {
Level int
Meta *FileMetadata
// BackingFileNum is only set during manifest replay, and only for virtual
// sstables.
BackingFileNum base.DiskFileNum
}
// VersionEdit holds the state for an edit to a Version along with other
@@ -143,7 +149,9 @@ type VersionEdit struct {
// Decode decodes an edit from the specified reader.
//
// TODO(bananabrick): Support decoding of virtual sstable state.
// Note that the Decode step will not set the FileBacking for virtual sstables
// and the responsibility is left to the caller. However, the Decode step will
// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles.
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(byteReader)
if !ok {
@@ -196,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
// NB: RocksDB does not use compaction pointers anymore.
case tagRemovedBackingTable:
n, err := d.readUvarint()
if err != nil {
return err
}
v.RemovedBackingTables = append(
v.RemovedBackingTables, base.FileNum(n).DiskFileNum(),
)
case tagCreatedBackingTable:
dfn, err := d.readUvarint()
if err != nil {
return err
}
size, err := d.readUvarint()
if err != nil {
return err
}
fileBacking := &FileBacking{
DiskFileNum: base.FileNum(dfn).DiskFileNum(),
Size: size,
}
v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking)
case tagDeletedFile:
level, err := d.readLevel()
if err != nil {
@@ -301,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
var markedForCompaction bool
var creationTime uint64
virtualState := struct {
virtual bool
backingFileNum uint64
}{}
if tag == tagNewFile4 || tag == tagNewFile5 {
for {
customTag, err := d.readUvarint()
@@ -309,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
if customTag == customTagTerminate {
break
} else if customTag == customTagVirtual {
virtualState.virtual = true
n, err := d.readUvarint()
if err != nil {
return err
}
virtualState.backingFileNum = n
continue
}
field, err := d.readBytes()
if err != nil {
return err
@@ -345,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
SmallestSeqNum: smallestSeqNum,
LargestSeqNum: largestSeqNum,
MarkedForCompaction: markedForCompaction,
Virtual: virtualState.virtual,
}
if tag != tagNewFile5 { // no range keys present
m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
@@ -376,11 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
}
m.boundsSet = true
m.InitPhysicalBacking()
v.NewFiles = append(v.NewFiles, NewFileEntry{
if !virtualState.virtual {
m.InitPhysicalBacking()
}
nfe := NewFileEntry{
Level: level,
Meta: m,
})
}
if virtualState.virtual {
nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum()
}
v.NewFiles = append(v.NewFiles, nfe)
case tagPrevLogNumber:
n, err := d.readUvarint()
@@ -399,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
return nil
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
var buf bytes.Buffer
if v.ComparerName != "" {
fmt.Fprintf(&buf, " comparer: %s", v.ComparerName)
@@ -431,7 +481,12 @@ func (v *VersionEdit) String() string {
fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum)
}
for _, nf := range v.NewFiles {
fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String())
fmt.Fprintf(&buf, " added: L%d", nf.Level)
if verbose {
fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
} else {
fmt.Fprintf(&buf, " %s", nf.Meta.String())
}
if nf.Meta.CreationTime != 0 {
fmt.Fprintf(&buf, " (%s)",
time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -441,9 +496,17 @@ func (v *VersionEdit) String() string {
return buf.String()
}
// DebugString is a more verbose version of String(). Use this in tests.
func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
return v.string(true /* verbose */, fmtKey)
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
return v.string(false /* verbose */, base.DefaultFormatter)
}
// Encode encodes an edit to the specified writer.
//
// TODO(bananabrick): Support encoding of virtual sstable state.
func (v *VersionEdit) Encode(w io.Writer) error {
e := versionEditEncoder{new(bytes.Buffer)}
@@ -463,6 +526,15 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(tagNextFileNumber)
e.writeUvarint(uint64(v.NextFileNum))
}
for _, dfn := range v.RemovedBackingTables {
e.writeUvarint(tagRemovedBackingTable)
e.writeUvarint(uint64(dfn.FileNum()))
}
for _, fileBacking := range v.CreatedBackingTables {
e.writeUvarint(tagCreatedBackingTable)
e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum()))
e.writeUvarint(fileBacking.Size)
}
// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
// even though its value is zero. We detect this by encoding LastSeqNum when
// ComparerName is set.
@@ -476,7 +548,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(uint64(x.FileNum))
}
for _, x := range v.NewFiles {
customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
var tag uint64
switch {
case x.Meta.HasRangeKeys:
@@ -529,6 +601,10 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(customTagNeedsCompaction)
e.writeBytes([]byte{1})
}
if x.Meta.Virtual {
e.writeUvarint(customTagVirtual)
e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum()))
}
e.writeUvarint(customTagTerminate)
}
}
@@ -536,6 +612,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
return err
}
// versionEditDecoder should be used to decode version edits.
type versionEditDecoder struct {
byteReader
}
@@ -634,7 +711,9 @@ type BulkVersionEdit struct {
Added [NumLevels]map[base.FileNum]*FileMetadata
Deleted [NumLevels]map[base.FileNum]*FileMetadata
AddedFileBacking []*FileBacking
// AddedFileBacking is a map to support lookup so that we can populate the
// FileBacking of virtual sstables during manifest replay.
AddedFileBacking map[base.DiskFileNum]*FileBacking
RemovedFileBacking []base.DiskFileNum
// AddedByFileNum maps file number to file metadata for all added files
@@ -697,6 +776,22 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
}
}
// Generate state for Added backing files. Note that these must be generated
// before we loop through the NewFiles, because we need to populate the
// FileBackings which might be used by the NewFiles loop.
if b.AddedFileBacking == nil {
b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
}
for _, fb := range ve.CreatedBackingTables {
if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
// There is already a FileBacking associated with fb.DiskFileNum.
// This should never happen. There must always be only one FileBacking
// associated with a backing sstable.
panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
}
b.AddedFileBacking[fb.DiskFileNum] = fb
}
for _, nf := range ve.NewFiles {
// A new file should not have been deleted in this or a preceding
// VersionEdit at the same level (though files can move across levels).
@@ -705,6 +800,17 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
}
}
if nf.Meta.Virtual && nf.Meta.FileBacking == nil {
// FileBacking for a virtual sstable must only be nil if we're performing
// manifest replay.
nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum]
if nf.Meta.FileBacking == nil {
return errors.Errorf("FileBacking for virtual sstable must not be nil")
}
} else if nf.Meta.FileBacking == nil {
return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum)
}
if b.Added[nf.Level] == nil {
b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
}
@@ -717,9 +823,6 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
}
}
// Generate state for the backing files.
b.AddedFileBacking = append(b.AddedFileBacking, ve.CreatedBackingTables...)
// Since a file can be removed from backing files in exactly one version
// edit it is safe to just append without any de-duplication.
b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)
@@ -749,6 +852,9 @@ func AccumulateIncompleteAndApplySingleVE(
flushSplitBytes int64,
readCompactionRate int64,
backingStateMap map[base.DiskFileNum]*FileBacking,
addBackingFunc func(*FileBacking),
removeBackingFunc func(base.DiskFileNum),
orderingInvariants OrderingInvariants,
) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
if len(ve.RemovedBackingTables) != 0 {
panic("pebble: invalid incomplete version edit")
@@ -760,14 +866,14 @@ func AccumulateIncompleteAndApplySingleVE(
}
zombies = make(map[base.DiskFileNum]uint64)
v, err := b.Apply(
curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
)
if err != nil {
return nil, nil, err
}
for _, s := range b.AddedFileBacking {
backingStateMap[s.DiskFileNum] = s
addBackingFunc(s)
}
for fileNum := range zombies {
@@ -778,10 +884,9 @@ func AccumulateIncompleteAndApplySingleVE(
ve.RemovedBackingTables = append(
ve.RemovedBackingTables, fileNum,
)
delete(backingStateMap, fileNum)
removeBackingFunc(fileNum)
}
}
return v, zombies, nil
}
@@ -802,6 +907,7 @@ func (b *BulkVersionEdit) Apply(
flushSplitBytes int64,
readCompactionRate int64,
zombies map[base.DiskFileNum]uint64,
orderingInvariants OrderingInvariants,
) (*Version, error) {
addZombie := func(state *FileBacking) {
if zombies != nil {
@@ -870,7 +976,7 @@ func (b *BulkVersionEdit) Apply(
// level.
for _, f := range deletedFilesMap {
if obsolete := v.Levels[level].tree.Delete(f); obsolete {
if obsolete := v.Levels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
@@ -879,7 +985,7 @@ func (b *BulkVersionEdit) Apply(
return nil, err
}
if f.HasRangeKeys {
if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
@@ -932,7 +1038,7 @@ func (b *BulkVersionEdit) Apply(
f.AllowedSeeks.Store(allowedSeeks)
f.InitAllowedSeeks = allowedSeeks
err := lm.tree.Insert(f)
err := lm.insert(f)
// We're adding this file to the new version, so increment the
// latest refs count.
f.LatestRef()
@@ -940,7 +1046,7 @@ func (b *BulkVersionEdit) Apply(
return nil, errors.Wrap(err, "pebble")
}
if f.HasRangeKeys {
err = lmRange.tree.Insert(f)
err = lmRange.insert(f)
if err != nil {
return nil, errors.Wrap(err, "pebble")
}
@@ -985,7 +1091,7 @@ func (b *BulkVersionEdit) Apply(
} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
continue
@@ -1006,7 +1112,7 @@ func (b *BulkVersionEdit) Apply(
end.Prev()
}
})
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
}