aleesa-bot/aleesa-telegram-go
0
Fork 0
Code Activity

[WIP] lint a bit, update deps

Browse Source
This commit is contained in:
eleksir
2024-02-27 00:34:44 +03:00
parent 1892960896
commit 1f97424f31
201 changed files with 21474 additions and 9282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go generated vendored
View File

@@ -18,22 +18,21 @@
package arenaskl
import (
"math"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/constants"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Arena is lock-free.
type Arena struct {
n uint64
n atomic.Uint64
buf []byte
}
const (
align4 = 3
)
const nodeAlignment = 4
var (
// ErrArenaFull indicates that the arena is full and cannot perform any more
@@ -44,20 +43,28 @@ var (
// NewArena allocates a new arena using the specified buffer as the backing
// store.
func NewArena(buf []byte) *Arena {
// Don't store data at position 0 in order to reserve offset=0 as a kind
// of nil pointer.
return &Arena{
n: 1,
if len(buf) > constants.MaxUint32OrInt {
if invariants.Enabled {
panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
}
buf = buf[:constants.MaxUint32OrInt]
}
a := &Arena{
buf: buf,
}
// We don't store data at position 0 in order to reserve offset=0 as a kind of
// nil pointer.
a.n.Store(1)
return a
}
// Size returns the number of bytes allocated by the arena.
func (a *Arena) Size() uint32 {
s := atomic.LoadUint64(&a.n)
if s > math.MaxUint32 {
// Saturate at MaxUint32.
return math.MaxUint32
s := a.n.Load()
if s > constants.MaxUint32OrInt {
// The last failed allocation can push the size higher than len(a.buf).
// Saturate at the maximum representable offset.
return constants.MaxUint32OrInt
}
return uint32(s)
}
@@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 {
return uint32(len(a.buf))
}
func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
// alloc allocates a buffer of the given size and with the given alignment
// (which must be a power of 2).
//
// If overflow is not 0, it also ensures that many bytes after the buffer are
// inside the arena (this is used for structures that are larger than the
// requested size but don't use those extra bytes).
func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
if invariants.Enabled && (alignment&(alignment-1)) != 0 {
panic(errors.AssertionFailedf("invalid alignment %d", alignment))
}
// Verify that the arena isn't already full.
origSize := atomic.LoadUint64(&a.n)
origSize := a.n.Load()
if int(origSize) > len(a.buf) {
return 0, 0, ErrArenaFull
}
// Pad the allocation with enough bytes to ensure the requested alignment.
padded := uint32(size) + align
padded := uint64(size) + uint64(alignment) - 1
newSize := atomic.AddUint64(&a.n, uint64(padded))
if int(newSize)+int(overflow) > len(a.buf) {
newSize := a.n.Add(padded)
if newSize+uint64(overflow) > uint64(len(a.buf)) {
return 0, 0, ErrArenaFull
}
// Return the aligned offset.
offset := (uint32(newSize) - padded + align) & ^align
return offset, padded, nil
offset := (uint32(newSize) - size) & ^(alignment - 1)
return offset, uint32(padded), nil
}
func (a *Arena) getBytes(offset uint32, size uint32) []byte {

14
vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go generated vendored
View File

@@ -18,7 +18,6 @@
package arenaskl
import (
"encoding/binary"
"sync"
"github.com/cockroachdb/pebble/internal/base"
@@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
}
func (it *Iterator) decodeKey() {
b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
// This is a manual inline of base.DecodeInternalKey, because the Go compiler
// seems to refuse to automatically inline it currently.
l := len(b) - 8
if l >= 0 {
it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
it.key.UserKey = b[:l:l]
} else {
it.key.Trailer = uint64(base.InternalKeyKindInvalid)
it.key.UserKey = nil
}
it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
it.key.Trailer = it.nd.keyTrailer
}
func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {

36
vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go generated vendored
View File

@@ -29,25 +29,27 @@ import (
// is used here. If a key/value overflows a uint32, it should not be added to
// the skiplist.
func MaxNodeSize(keySize, valueSize uint32) uint64 {
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
const maxPadding = nodeAlignment - 1
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
}
type links struct {
nextOffset uint32
prevOffset uint32
nextOffset atomic.Uint32
prevOffset atomic.Uint32
}
func (l *links) init(prevOffset, nextOffset uint32) {
l.nextOffset = nextOffset
l.prevOffset = prevOffset
l.nextOffset.Store(nextOffset)
l.prevOffset.Store(prevOffset)
}
type node struct {
// Immutable fields, so no need to lock to access key.
keyOffset uint32
keySize uint32
valueSize uint32
allocSize uint32
keyOffset uint32
keySize uint32
keyTrailer uint64
valueSize uint32
allocSize uint32
// Most nodes do not need to use the full height of the tower, since the
// probability of each successive level decreases exponentially. Because
@@ -65,7 +67,7 @@ func newNode(
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
keySize := key.Size()
keySize := len(key.UserKey)
if int64(keySize) > math.MaxUint32 {
panic("key is too large")
}
@@ -81,8 +83,8 @@ func newNode(
if err != nil {
return
}
key.Encode(nd.getKeyBytes(arena))
nd.keyTrailer = key.Trailer
copy(nd.getKeyBytes(arena), key.UserKey)
copy(nd.getValue(arena), value)
return
}
@@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod
unusedSize := uint32((maxHeight - int(height)) * linksSize)
nodeSize := uint32(maxNodeSize) - unusedSize
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
if err != nil {
return
}
@@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte {
}
func (n *node) nextOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].nextOffset)
return n.tower[h].nextOffset.Load()
}
func (n *node) prevOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].prevOffset)
return n.tower[h].prevOffset.Load()
}
func (n *node) casNextOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
return n.tower[h].nextOffset.CompareAndSwap(old, val)
}
func (n *node) casPrevOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
return n.tower[h].prevOffset.CompareAndSwap(old, val)
}

53
vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go generated vendored
View File

@@ -44,7 +44,6 @@ Key differences:
package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
import (
"encoding/binary"
"math"
"runtime"
"sync/atomic"
@@ -68,7 +67,7 @@ const (
// keys.
var ErrRecordExists = errors.New("record with this key already exists")
// Skiplist is a fast, cocnurrent skiplist implementation that supports forward
// Skiplist is a fast, concurrent skiplist implementation that supports forward
// and backward iteration. See batchskl.Skiplist for a non-concurrent
// skiplist. Keys and values are immutable once added to the skiplist and
// deletion is not supported. Instead, higher-level code is expected to add new
@@ -80,7 +79,7 @@ type Skiplist struct {
cmp base.Compare
head *node
tail *node
height uint32 // Current height. 1 <= height <= maxHeight. CAS.
height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
// If set to true by tests, then extra delays are added to make it easier to
// detect unusual race conditions.
@@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
headOffset := arena.getPointerOffset(unsafe.Pointer(head))
tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
for i := 0; i < maxHeight; i++ {
head.tower[i].nextOffset = tailOffset
tail.tower[i].prevOffset = headOffset
head.tower[i].nextOffset.Store(tailOffset)
tail.tower[i].prevOffset.Store(headOffset)
}
*s = Skiplist{
arena: arena,
cmp: cmp,
head: head,
tail: tail,
height: 1,
arena: arena,
cmp: cmp,
head: head,
tail: tail,
}
s.height.Store(1)
}
// Height returns the height of the highest tower within any of the nodes that
// have ever been allocated as part of this skiplist.
func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
func (s *Skiplist) Height() uint32 { return s.height.Load() }
// Arena returns the arena backing this skiplist.
func (s *Skiplist) Arena() *Arena { return s.arena }
@@ -327,7 +326,7 @@ func (s *Skiplist) newNode(
// Try to increase s.height via CAS.
listHeight := s.Height()
for height > listHeight {
if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
if s.height.CompareAndSwap(listHeight, height) {
// Successfully increased skiplist.height.
break
}
@@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel(
offset, size := next.keyOffset, next.keySize
nextKey := s.arena.buf[offset : offset+size]
n := int32(size) - 8
cmp := s.cmp(key.UserKey, nextKey[:n])
cmp := s.cmp(key.UserKey, nextKey)
if cmp < 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
if cmp == 0 {
// User-key equality.
var nextTrailer uint64
if n >= 0 {
nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
} else {
nextTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == nextTrailer {
if key.Trailer == next.keyTrailer {
// Internal key equality.
found = true
break
}
if key.Trailer > nextTrailer {
if key.Trailer > next.keyTrailer {
// We are done for this level, since prev.key < key < next.key.
break
}
@@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel(
func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
n := int32(nd.keySize) - 8
cmp := s.cmp(ndKey[:n], key.UserKey)
cmp := s.cmp(ndKey, key.UserKey)
if cmp < 0 {
return true
}
@@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
return false
}
// User-key equality.
var ndTrailer uint64
if n >= 0 {
ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
} else {
ndTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == ndTrailer {
if key.Trailer == nd.keyTrailer {
// Internal key equality.
return false
}
return key.Trailer < ndTrailer
return key.Trailer < nd.keyTrailer
}
func (s *Skiplist) getNext(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
offset := nd.tower[h].nextOffset.Load()
return (*node)(s.arena.getPointer(offset))
}
func (s *Skiplist) getPrev(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
offset := nd.tower[h].prevOffset.Load()
return (*node)(s.arena.getPointer(offset))
}

89
vendor/github.com/cockroachdb/pebble/internal/base/filenames.go generated vendored
View File

@@ -20,6 +20,41 @@ type FileNum uint64
// String returns a string representation of the file number.
func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
// SafeFormat implements redact.SafeFormatter.
func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("%06d", redact.SafeUint(fn))
}
// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
// called if the caller can ensure that the FileNum belongs to a physical file
// on disk. These could be manifests, log files, physical sstables on disk, the
// options file, but not virtual sstables.
func (fn FileNum) DiskFileNum() DiskFileNum {
return DiskFileNum{fn}
}
// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
// Note that a FileNum is an internal DB identifier and it could belong to files
// which don't exist on disk. An example would be virtual sstable FileNums.
// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
// FileNum to DiskFileNum may not be valid and care should be taken to prove
// that the FileNum actually exists on disk.
type DiskFileNum struct {
fn FileNum
}
func (dfn DiskFileNum) String() string { return dfn.fn.String() }
// SafeFormat implements redact.SafeFormatter.
func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
dfn.fn.SafeFormat(w, verb)
}
// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
func (dfn DiskFileNum) FileNum() FileNum {
return dfn.fn
}
// FileType enumerates the types of files found in a DB.
type FileType int
@@ -36,92 +71,92 @@ const (
)
// MakeFilename builds a filename from components.
func MakeFilename(fileType FileType, fileNum FileNum) string {
func MakeFilename(fileType FileType, dfn DiskFileNum) string {
switch fileType {
case FileTypeLog:
return fmt.Sprintf("%s.log", fileNum)
return fmt.Sprintf("%s.log", dfn)
case FileTypeLock:
return "LOCK"
case FileTypeTable:
return fmt.Sprintf("%s.sst", fileNum)
return fmt.Sprintf("%s.sst", dfn)
case FileTypeManifest:
return fmt.Sprintf("MANIFEST-%s", fileNum)
return fmt.Sprintf("MANIFEST-%s", dfn)
case FileTypeCurrent:
return "CURRENT"
case FileTypeOptions:
return fmt.Sprintf("OPTIONS-%s", fileNum)
return fmt.Sprintf("OPTIONS-%s", dfn)
case FileTypeOldTemp:
return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum)
return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
case FileTypeTemp:
return fmt.Sprintf("temporary.%s.dbtmp", fileNum)
return fmt.Sprintf("temporary.%s.dbtmp", dfn)
}
panic("unreachable")
}
// MakeFilepath builds a filepath from components.
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, fileNum))
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
}
// ParseFilename parses the components from a filename.
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) {
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
filename = fs.PathBase(filename)
switch {
case filename == "CURRENT":
return FileTypeCurrent, 0, true
return FileTypeCurrent, DiskFileNum{0}, true
case filename == "LOCK":
return FileTypeLock, 0, true
return FileTypeLock, DiskFileNum{0}, true
case strings.HasPrefix(filename, "MANIFEST-"):
fileNum, ok = parseFileNum(filename[len("MANIFEST-"):])
dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
if !ok {
break
}
return FileTypeManifest, fileNum, true
return FileTypeManifest, dfn, true
case strings.HasPrefix(filename, "OPTIONS-"):
fileNum, ok = parseFileNum(filename[len("OPTIONS-"):])
dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
if !ok {
break
}
return FileTypeOptions, fileNum, ok
return FileTypeOptions, dfn, ok
case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeOldTemp, fileNum, ok
return FileTypeOldTemp, dfn, ok
case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeTemp, fileNum, ok
return FileTypeTemp, dfn, ok
default:
i := strings.IndexByte(filename, '.')
if i < 0 {
break
}
fileNum, ok = parseFileNum(filename[:i])
dfn, ok = parseDiskFileNum(filename[:i])
if !ok {
break
}
switch filename[i+1:] {
case "sst":
return FileTypeTable, fileNum, true
return FileTypeTable, dfn, true
case "log":
return FileTypeLog, fileNum, true
return FileTypeLog, dfn, true
}
}
return 0, fileNum, false
return 0, dfn, false
}
func parseFileNum(s string) (fileNum FileNum, ok bool) {
func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
u, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return fileNum, false
return dfn, false
}
return FileNum(u), true
return DiskFileNum{FileNum(u)}, true
}
// A Fataler fatals a process with a message when called.

71
vendor/github.com/cockroachdb/pebble/internal/base/internal.go generated vendored
View File

@@ -9,6 +9,17 @@ import (
"fmt"
"strconv"
"strings"
"github.com/cockroachdb/redact"
)
const (
// SeqNumZero is the zero sequence number, set by compactions if they can
// guarantee there are no keys underneath an internal key.
SeqNumZero = uint64(0)
// SeqNumStart is the first sequence number assigned to a key. Sequence
// numbers 1-9 are reserved for potential future use.
SeqNumStart = uint64(10)
)
// InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
@@ -24,6 +35,14 @@ const (
//InternalKeyKindColumnFamilyDeletion InternalKeyKind = 4
//InternalKeyKindColumnFamilyValue InternalKeyKind = 5
//InternalKeyKindColumnFamilyMerge InternalKeyKind = 6
// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
// solely for compactions (to reduce write amp and space amp). Readers other
// than compactions should treat SINGLEDEL as equivalent to a DEL.
// Historically, it was simpler for readers other than compactions to treat
// SINGLEDEL as equivalent to DEL, but as of the introduction of
// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
// correctness.
InternalKeyKindSingleDelete InternalKeyKind = 7
//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
//InternalKeyKindBeginPrepareXID InternalKeyKind = 9
@@ -60,14 +79,21 @@ const (
InternalKeyKindRangeKeyUnset InternalKeyKind = 20
InternalKeyKindRangeKeySet InternalKeyKind = 21
// InternalKeyIngestSST is used to distinguish a batch that corresponds to
// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
// the WAL entry for ingested sstables that are added to the flushable
// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
// batch, or in an sstable.
InternalKeyIngestSST InternalKeyKind = 22
InternalKeyKindIngestSST InternalKeyKind = 22
// This maximum value isn't part of the file format. It's unlikely,
// but future extensions may increase this value.
// InternalKeyKindDeleteSized keys behave identically to
// InternalKeyKindDelete keys, except that they hold an associated uint64
// value indicating the (len(key)+len(value)) of the shadowed entry the
// tombstone is expected to delete. This value is used to inform compaction
// heuristics, but is not required to be accurate for correctness.
InternalKeyKindDeleteSized InternalKeyKind = 23
// This maximum value isn't part of the file format. Future extensions may
// increase this value.
//
// When constructing an internal key to pass to DB.Seek{GE,LE},
// internalKeyComparer sorts decreasing by kind (after sorting increasing by
@@ -75,14 +101,19 @@ const (
// which sorts 'less than or equal to' any other valid internalKeyKind, when
// searching for any kind of internal key formed by a certain user key and
// seqNum.
InternalKeyKindMax InternalKeyKind = 22
InternalKeyKindMax InternalKeyKind = 23
// Internal to the sstable format. Not exposed by any sstable iterator.
// Declared here to prevent definition of valid key kinds that set this bit.
InternalKeyKindSSTableInternalObsoleteBit InternalKeyKind = 64
InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
// zero sequence number.
InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid)
InternalKeyZeroSeqnumMaxTrailer = uint64(255)
// A marker for an invalid key.
InternalKeyKindInvalid InternalKeyKind = 255
InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
// which prevents those entries from being excluded from iteration.
@@ -105,6 +136,9 @@ const (
InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
)
// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
var internalKeyKindNames = []string{
InternalKeyKindDelete: "DEL",
InternalKeyKindSet: "SET",
@@ -117,7 +151,8 @@ var internalKeyKindNames = []string{
InternalKeyKindRangeKeySet: "RANGEKEYSET",
InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET",
InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
InternalKeyIngestSST: "INGESTSST",
InternalKeyKindIngestSST: "INGESTSST",
InternalKeyKindDeleteSized: "DELSIZED",
InternalKeyKindInvalid: "INVALID",
}
@@ -128,6 +163,11 @@ func (k InternalKeyKind) String() string {
return fmt.Sprintf("UNKNOWN:%d", k)
}
// SafeFormat implements redact.SafeFormatter.
func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
w.Print(redact.SafeString(k.String()))
}
// InternalKey is a key used for the in-memory and on-disk partial DBs that
// make up a pebble DB.
//
@@ -194,6 +234,7 @@ var kindsMap = map[string]InternalKeyKind{
"DEL": InternalKeyKindDelete,
"SINGLEDEL": InternalKeyKindSingleDelete,
"RANGEDEL": InternalKeyKindRangeDelete,
"LOGDATA": InternalKeyKindLogData,
"SET": InternalKeyKindSet,
"MERGE": InternalKeyKindMerge,
"INVALID": InternalKeyKindInvalid,
@@ -202,6 +243,8 @@ var kindsMap = map[string]InternalKeyKind{
"RANGEKEYSET": InternalKeyKindRangeKeySet,
"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
"RANGEKEYDEL": InternalKeyKindRangeKeyDelete,
"INGESTSST": InternalKeyKindIngestSST,
"DELSIZED": InternalKeyKindDeleteSized,
}
// ParseInternalKey parses the string representation of an internal key. The
@@ -338,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
return k.Trailer >> 8
}
// SeqNumFromTrailer returns the sequence number component of a trailer.
func SeqNumFromTrailer(t uint64) uint64 {
return t >> 8
}
// Visible returns true if the key is visible at the specified snapshot
// sequence number.
func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
@@ -395,6 +443,13 @@ func (k InternalKey) Clone() InternalKey {
}
}
// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
// reusing any space already used for the current UserKey.
func (k *InternalKey) CopyFrom(k2 InternalKey) {
k.UserKey = append(k.UserKey[:0], k2.UserKey...)
k.Trailer = k2.Trailer
}
// String returns a string representation of the key.
func (k InternalKey) String() string {
return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())

9
vendor/github.com/cockroachdb/pebble/internal/base/metrics.go generated vendored
View File

@@ -53,6 +53,15 @@ func (tm *ThroughputMetric) Rate() int64 {
float64(time.Second))
}
// Utilization returns a percent [0, 1.0] indicating the percent of time
// work was performed.
func (tm *ThroughputMetric) Utilization() float64 {
if tm.WorkDuration == 0 {
return 0
}
return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
}
// GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
// accumulating samples of that gauge.
type GaugeSampleMetric struct {

15
vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go generated vendored
View File

@@ -63,14 +63,15 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/constants"
"golang.org/x/exp/rand"
)
const (
maxHeight = 20
maxNodeSize = int(unsafe.Sizeof(node{}))
linksSize = int(unsafe.Sizeof(links{}))
maxNodesSize = math.MaxUint32
maxNodeSize = uint64(unsafe.Sizeof(node{}))
linksSize = uint64(unsafe.Sizeof(links{}))
maxNodesSize = constants.MaxUint32OrInt
)
var (
@@ -281,7 +282,7 @@ func (s *Skiplist) newNode(
panic("height cannot be less than one or greater than the max height")
}
unusedSize := (maxHeight - int(height)) * linksSize
unusedSize := uint64(maxHeight-int(height)) * linksSize
nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
if err != nil {
return 0, err
@@ -296,13 +297,13 @@ func (s *Skiplist) newNode(
}
func (s *Skiplist) alloc(size uint32) (uint32, error) {
offset := len(s.nodes)
offset := uint64(len(s.nodes))
// We only have a need for memory up to offset + size, but we never want
// to allocate a node whose tail points into unallocated memory.
minAllocSize := offset + maxNodeSize
if cap(s.nodes) < minAllocSize {
allocSize := cap(s.nodes) * 2
if uint64(cap(s.nodes)) < minAllocSize {
allocSize := uint64(cap(s.nodes)) * 2
if allocSize < minAllocSize {
allocSize = minAllocSize
}

208
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go generated vendored
View File

@@ -33,7 +33,7 @@ import (
type fileKey struct {
// id is the namespace for fileNums.
id uint64
fileNum base.FileNum
fileNum base.DiskFileNum
}
type key struct {
@@ -71,14 +71,12 @@ func (h Handle) Get() []byte {
// Release releases the reference to the cache entry.
func (h Handle) Release() {
if h.value != nil {
h.value.release()
}
h.value.release()
}
type shard struct {
hits int64
misses int64
hits atomic.Int64
misses atomic.Int64
mu sync.RWMutex
@@ -113,25 +111,25 @@ type shard struct {
countTest int64
}
func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
c.mu.RLock()
var value *Value
if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
value = e.acquireValue()
if value != nil {
atomic.StoreInt32(&e.referenced, 1)
e.referenced.Store(true)
}
}
c.mu.RUnlock()
if value == nil {
atomic.AddInt64(&c.misses, 1)
c.misses.Add(1)
return Handle{}
}
atomic.AddInt64(&c.hits, 1)
c.hits.Add(1)
return Handle{value: value}
}
func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
if n := value.refs(); n != 1 {
panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
}
@@ -160,7 +158,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
case e.peekValue() != nil:
// cache entry was a hot or cold page
e.setValue(value)
atomic.StoreInt32(&e.referenced, 1)
e.referenced.Store(true)
delta := int64(len(value.buf)) - e.size
e.size = int64(len(value.buf))
if e.ptype == etHot {
@@ -176,7 +174,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
// cache entry was a test page
c.sizeTest -= e.size
c.countTest--
c.metaDel(e)
c.metaDel(e).release()
c.metaCheck(e)
e.size = int64(len(value.buf))
@@ -185,7 +183,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
c.coldTarget = c.targetSize()
}
atomic.StoreInt32(&e.referenced, 0)
e.referenced.Store(false)
e.setValue(value)
e.ptype = etHot
if c.metaAdd(k, e) {
@@ -222,7 +220,7 @@ func (c *shard) checkConsistency() {
}
// Delete deletes the cached value for the specified file and offset.
func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
// The common case is there is nothing to delete, so do a quick check with
// shared lock.
k := key{fileKey{id, fileNum}, offset}
@@ -233,37 +231,75 @@ func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
return
}
c.mu.Lock()
defer c.mu.Unlock()
var deletedValue *Value
func() {
c.mu.Lock()
defer c.mu.Unlock()
e := c.blocks.Get(k)
if e == nil {
return
}
c.metaEvict(e)
c.checkConsistency()
e := c.blocks.Get(k)
if e == nil {
return
}
deletedValue = c.metaEvict(e)
c.checkConsistency()
}()
// Now that the mutex has been dropped, release the reference which will
// potentially free the memory associated with the previous cached value.
deletedValue.release()
}
// EvictFile evicts all of the cache values for the specified file.
func (c *shard) EvictFile(id uint64, fileNum base.FileNum) {
c.mu.Lock()
defer c.mu.Unlock()
func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
fkey := key{fileKey{id, fileNum}, 0}
for c.evictFileRun(fkey) {
// Sched switch to give another goroutine an opportunity to acquire the
// shard mutex.
runtime.Gosched()
}
}
func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
// If most of the file's blocks are held in the block cache, evicting all
// the blocks may take a while. We don't want to block the entire cache
// shard, forcing concurrent readers to wait until we're finished. We drop
// the mutex every [blocksPerMutexAcquisition] blocks to give other
// goroutines an opportunity to make progress.
const blocksPerMutexAcquisition = 5
c.mu.Lock()
// Releasing a value may result in free-ing it back to the memory allocator.
// This can have a nontrivial cost that we'd prefer to not pay while holding
// the shard mutex, so we collect the evicted values in a local slice and
// only release them in a defer after dropping the cache mutex.
var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
obsoleteValues := obsoleteValuesAlloc[:0]
defer func() {
c.mu.Unlock()
for _, v := range obsoleteValues {
v.release()
}
}()
blocks := c.files.Get(fkey)
if blocks == nil {
return
}
for b, n := blocks, (*entry)(nil); ; b = n {
n = b.fileLink.next
c.metaEvict(b)
if b == n {
break
}
// No blocks for this file.
return false
}
c.checkConsistency()
// b is the current head of the doubly linked list, and n is the entry after b.
for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
n = b.fileLink.next
obsoleteValues = append(obsoleteValues, c.metaEvict(b))
if b == n {
// b == n represents the case where b was the last entry remaining
// in the doubly linked list, which is why it pointed at itself. So
// no more entries left.
c.checkConsistency()
return false
}
}
// Exhausted blocksPerMutexAcquisition.
return true
}
func (c *shard) Free() {
@@ -274,7 +310,7 @@ func (c *shard) Free() {
// metaCheck call when the "invariants" build tag is specified.
for c.handHot != nil {
e := c.handHot
c.metaDel(c.handHot)
c.metaDel(c.handHot).release()
e.free()
}
@@ -359,12 +395,14 @@ func (c *shard) metaAdd(key key, e *entry) bool {
// Remove the entry from the cache. This removes the entry from the blocks map,
// the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
// entry.
func (c *shard) metaDel(e *entry) {
// entry. Returns the deleted value that must be released, if any.
func (c *shard) metaDel(e *entry) (deletedValue *Value) {
if value := e.peekValue(); value != nil {
value.ref.trace("metaDel")
}
e.setValue(nil)
// Remove the pointer to the value.
deletedValue = e.val
e.val = nil
c.blocks.Delete(e.key)
if entriesGoAllocated {
@@ -396,6 +434,7 @@ func (c *shard) metaDel(e *entry) {
} else {
c.files.Put(fkey, next)
}
return deletedValue
}
// Check that the specified entry is not referenced by the cache.
@@ -455,7 +494,7 @@ func (c *shard) metaCheck(e *entry) {
}
}
func (c *shard) metaEvict(e *entry) {
func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
switch e.ptype {
case etHot:
c.sizeHot -= e.size
@@ -467,9 +506,10 @@ func (c *shard) metaEvict(e *entry) {
c.sizeTest -= e.size
c.countTest--
}
c.metaDel(e)
evictedValue = c.metaDel(e)
c.metaCheck(e)
e.free()
return evictedValue
}
func (c *shard) evict() {
@@ -491,8 +531,8 @@ func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
e := c.handCold
if e.ptype == etCold {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
if e.referenced.Load() {
e.referenced.Store(false)
e.ptype = etHot
c.sizeCold -= e.size
c.countCold--
@@ -528,8 +568,8 @@ func (c *shard) runHandHot() {
e := c.handHot
if e.ptype == etHot {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
if e.referenced.Load() {
e.referenced.Store(false)
} else {
e.ptype = etCold
c.sizeHot -= e.size
@@ -564,7 +604,7 @@ func (c *shard) runHandTest() {
if c.coldTarget < 0 {
c.coldTarget = 0
}
c.metaDel(e)
c.metaDel(e).release()
c.metaCheck(e)
e.free()
}
@@ -587,7 +627,7 @@ type Metrics struct {
// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
// used for page replacement
// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
// order to provide better concurrency, 2 x NumCPUs shards are created, with
// order to provide better concurrency, 4 x NumCPUs shards are created, with
// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
// is run independently on each shard.
//
@@ -621,9 +661,9 @@ type Metrics struct {
// used in combination by specifying `-tags invariants,tracing`. Note that
// "tracing" produces a significant slowdown, while "invariants" does not.
type Cache struct {
refs int64
refs atomic.Int64
maxSize int64
idAlloc uint64
idAlloc atomic.Uint64
shards []shard
// Traces recorded by Cache.trace. Used for debugging.
@@ -643,17 +683,44 @@ type Cache struct {
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func New(size int64) *Cache {
return newShards(size, 2*runtime.GOMAXPROCS(0))
// How many cache shards should we create?
//
// Note that the probability two processors will try to access the same
// shard at the same time increases superlinearly with the number of
// processors (Eg, consider the brithday problem where each CPU is a person,
// and each shard is a possible birthday).
//
// We could consider growing the number of shards superlinearly, but
// increasing the shard count may reduce the effectiveness of the caching
// algorithm if frequently-accessed blocks are insufficiently distributed
// across shards. If a shard's size is smaller than a single frequently
// scanned sstable, then the shard will be unable to hold the entire
// frequently-scanned table in memory despite other shards still holding
// infrequently accessed blocks.
//
// Experimentally, we've observed contention contributing to tail latencies
// at 2 shards per processor. For now we use 4 shards per processor,
// recognizing this may not be final word.
m := 4 * runtime.GOMAXPROCS(0)
// In tests we can use large CPU machines with small cache sizes and have
// many caches in existence at a time. If sharding into m shards would
// produce too small shards, constrain the number of shards to 4.
const minimumShardSize = 4 << 20 // 4 MiB
if m > 4 && int(size)/m < minimumShardSize {
m = 4
}
return newShards(size, m)
}
func newShards(size int64, shards int) *Cache {
c := &Cache{
refs: 1,
maxSize: size,
idAlloc: 1,
shards: make([]shard, shards),
}
c.trace("alloc", c.refs)
c.refs.Store(1)
c.idAlloc.Store(1)
c.trace("alloc", c.refs.Load())
for i := range c.shards {
c.shards[i] = shard{
maxSize: size / int64(len(c.shards)),
@@ -669,7 +736,7 @@ func newShards(size int64, shards int) *Cache {
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(c, func(obj interface{}) {
c := obj.(*Cache)
if v := atomic.LoadInt64(&c.refs); v != 0 {
if v := c.refs.Load(); v != 0 {
c.tr.Lock()
fmt.Fprintf(os.Stderr,
"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
@@ -683,7 +750,7 @@ func newShards(size int64, shards int) *Cache {
return c
}
func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard {
func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
@@ -698,10 +765,11 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
h ^= uint64(id & 0xff)
id >>= 8
}
fileNumVal := uint64(fileNum.FileNum())
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(fileNum & 0xff)
fileNum >>= 8
h ^= uint64(fileNumVal) & 0xff
fileNumVal >>= 8
}
for i := 0; i < 8; i++ {
h *= prime64
@@ -715,7 +783,7 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
// Ref adds a reference to the cache. The cache only remains valid as long a
// reference is maintained to it.
func (c *Cache) Ref() {
v := atomic.AddInt64(&c.refs, 1)
v := c.refs.Add(1)
if v <= 1 {
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
@@ -724,7 +792,7 @@ func (c *Cache) Ref() {
// Unref releases a reference on the cache.
func (c *Cache) Unref() {
v := atomic.AddInt64(&c.refs, -1)
v := c.refs.Add(-1)
c.trace("unref", v)
switch {
case v < 0:
@@ -738,7 +806,7 @@ func (c *Cache) Unref() {
// Get retrieves the cache value for the specified file and offset, returning
// nil if no value is present.
func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
}
@@ -746,17 +814,17 @@ func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
// existing value if present. A Handle is returned which provides faster
// retrieval of the cached value than Get (lock-free and avoidance of the map
// lookup). The value must have been allocated by Cache.Alloc.
func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
}
// Delete deletes the cached value for the specified file and offset.
func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) {
func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
}
// EvictFile evicts all of the cache values for the specified file.
func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) {
func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
@@ -784,14 +852,14 @@ func (c *Cache) Size() int64 {
// manually managed. The caller MUST either add the value to the cache (via
// Cache.Set), or release the value (via Cache.Free). Failure to do so will
// result in a memory leak.
func (c *Cache) Alloc(n int) *Value {
func Alloc(n int) *Value {
return newValue(n)
}
// Free frees the specified value. The buffer associated with the value will
// possibly be reused, making it invalid to use the buffer after calling
// Free. Do not call Free on a value that has been added to the cache.
func (c *Cache) Free(v *Value) {
func Free(v *Value) {
if n := v.refs(); n > 1 {
panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
}
@@ -828,8 +896,8 @@ func (c *Cache) Metrics() Metrics {
m.Count += int64(s.blocks.Count())
m.Size += s.sizeHot + s.sizeCold
s.mu.RUnlock()
m.Hits += atomic.LoadInt64(&s.hits)
m.Misses += atomic.LoadInt64(&s.misses)
m.Hits += s.hits.Load()
m.Misses += s.misses.Load()
}
return m
}
@@ -837,5 +905,5 @@ func (c *Cache) Metrics() Metrics {
// NewID returns a new ID to be used as a namespace for cached file
// blocks.
func (c *Cache) NewID() uint64 {
return atomic.AddUint64(&c.idAlloc, 1)
return c.idAlloc.Add(1)
}

8
vendor/github.com/cockroachdb/pebble/internal/cache/entry.go generated vendored
View File

@@ -4,6 +4,8 @@
package cache
import "sync/atomic"
type entryType int8
const (
@@ -56,7 +58,7 @@ type entry struct {
ptype entryType
// referenced is atomically set to indicate that this entry has been accessed
// since the last time one of the clock hands swept it.
referenced int32
referenced atomic.Bool
shard *shard
// Reference count for the entry. The entry is freed when the reference count
// drops to zero.
@@ -137,9 +139,7 @@ func (e *entry) setValue(v *Value) {
}
old := e.val
e.val = v
if old != nil {
old.release()
}
old.release()
}
func (e *entry) peekValue() *Value {

2
vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go generated vendored
View File

@@ -24,7 +24,7 @@ func robinHoodHash(k key, shift uint32) uint32 {
const m = 11400714819323198485
h := hashSeed
h ^= k.id * m
h ^= uint64(k.fileNum) * m
h ^= uint64(k.fileNum.FileNum()) * m
h ^= k.offset * m
return uint32(h >> shift)
}

2
vendor/github.com/cockroachdb/pebble/internal/cache/value.go generated vendored
View File

@@ -40,7 +40,7 @@ func (v *Value) acquire() {
}
func (v *Value) release() {
if v.ref.release() {
if v != nil && v.ref.release() {
v.free()
}
}

17
vendor/github.com/cockroachdb/pebble/internal/constants/constants.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package constants
const (
// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
oneIf64Bit = ^uint(0) >> 63
// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
// - MaxUint32 on 64-bit platforms;
// - MaxInt on 32-bit platforms.
// It is used when slices are limited to Uint32 on 64-bit platforms (the
// length limit for slices is naturally MaxInt on 32-bit platforms).
MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
)

20
vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go generated vendored
View File

@@ -35,11 +35,11 @@ type config struct {
suffix []string
}
// IEC produces human readable representations of integer values in IEC units.
var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}}
// Bytes produces human readable representations of byte values in IEC units.
var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
// SI produces human readable representations of integer values in SI units.
var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}}
// Count produces human readable representations of unitless values in SI units.
var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
// Int64 produces a human readable representation of the value.
func (c *config) Int64(s int64) FormattedString {
@@ -54,18 +54,6 @@ func (c *config) Uint64(s uint64) FormattedString {
return FormattedString(humanate(s, c.base, c.suffix))
}
// Int64 produces a human readable representation of the value in IEC units
// (base 1024).
func Int64(s int64) FormattedString {
return IEC.Int64(s)
}
// Uint64 produces a human readable representation of the value in IEC units
// (base 1024).
func Uint64(s uint64) FormattedString {
return IEC.Uint64(s)
}
// FormattedString represents a human readable representation of a value. It
// implements the redact.SafeValue interface to signal that it represents a
// a string that does not need to be redacted.

162
vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go generated vendored Normal file
View File

@@ -0,0 +1,162 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package invalidating
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/fastrand"
"github.com/cockroachdb/pebble/internal/invariants"
)
// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
// MaybeWrapIfInvariants does nothing in non-invariant builds.
func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
if invariants.Enabled {
if fastrand.Uint32n(10) == 1 {
return NewIter(iter)
}
}
return iter
}
// iter tests unsafe key/value slice reuse by modifying the last
// returned key/value to all 1s.
type iter struct {
iter base.InternalIterator
lastKey *base.InternalKey
lastValue base.LazyValue
ignoreKinds [base.InternalKeyKindMax + 1]bool
err error
}
// Option configures the behavior of an invalidating iterator.
type Option interface {
apply(*iter)
}
type funcOpt func(*iter)
func (f funcOpt) apply(i *iter) { f(i) }
// IgnoreKinds constructs an Option that configures an invalidating iterator to
// skip trashing k/v pairs with the provided key kinds. Some iterators provided
// key stability guarantees for specific key kinds.
func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
return funcOpt(func(i *iter) {
for _, kind := range kinds {
i.ignoreKinds[kind] = true
}
})
}
// NewIter constructs a new invalidating iterator that wraps the provided
// iterator, trashing buffers for previously returned keys.
func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
i := &iter{iter: originalIterator}
for _, opt := range opts {
opt.apply(i)
}
return i
}
func (i *iter) update(
key *base.InternalKey, value base.LazyValue,
) (*base.InternalKey, base.LazyValue) {
i.trashLastKV()
if key == nil {
i.lastKey = nil
i.lastValue = base.LazyValue{}
return nil, base.LazyValue{}
}
i.lastKey = &base.InternalKey{}
*i.lastKey = key.Clone()
i.lastValue = base.LazyValue{
ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
}
if value.Fetcher != nil {
fetcher := new(base.LazyFetcher)
*fetcher = *value.Fetcher
i.lastValue.Fetcher = fetcher
}
return i.lastKey, i.lastValue
}
func (i *iter) trashLastKV() {
if i.lastKey == nil {
return
}
if i.ignoreKinds[i.lastKey.Kind()] {
return
}
if i.lastKey != nil {
for j := range i.lastKey.UserKey {
i.lastKey.UserKey[j] = 0xff
}
i.lastKey.Trailer = 0xffffffffffffffff
}
for j := range i.lastValue.ValueOrHandle {
i.lastValue.ValueOrHandle[j] = 0xff
}
if i.lastValue.Fetcher != nil {
// Not all the LazyFetcher fields are visible, so we zero out the last
// value's Fetcher struct entirely.
*i.lastValue.Fetcher = base.LazyFetcher{}
}
}
func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekGE(key, flags))
}
func (i *iter) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
}
func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekLT(key, flags))
}
func (i *iter) First() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.First())
}
func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Last())
}
func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Next())
}
func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Prev())
}
func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.NextPrefix(succKey))
}
func (i *iter) Error() error {
if err := i.iter.Error(); err != nil {
return err
}
return i.err
}
func (i *iter) Close() error {
return i.iter.Close()
}
func (i *iter) SetBounds(lower, upper []byte) {
i.iter.SetBounds(lower, upper)
}
func (i *iter) String() string {
return i.iter.String()
}

38
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored
View File

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
// fragments before this span. Defragment backward to find the start of the
// defragmented span.
i.defragmentBackward()
// Defragmenting backward may have stopped because it encountered an error.
// If so, we must not continue so that i.iter.Error() (and thus i.Error())
// yields the error.
if i.iterSpan == nil && i.iter.Error() != nil {
return nil
}
if i.iterPos == iterPosPrev {
// Next once back onto the span.
i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
// fragments after this span. Defragment forward to find the end of the
// defragmented span.
i.defragmentForward()
// Defragmenting forward may have stopped because it encountered an error.
// If so, we must not continue so that i.iter.Error() (and thus i.Error())
// yields the error.
if i.iterSpan == nil && i.iter.Error() != nil {
return nil
}
if i.iterPos == iterPosNext {
// Prev once back onto the span.
i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
// Next once to move onto y, defragment forward to land on the first z
// position.
i.iterSpan = i.iter.Next()
if invariants.Enabled && i.iterSpan == nil {
if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
panic("pebble: invariant violation: no next span while switching directions")
}
// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
// Prev once to move onto y, defragment backward to land on the last x
// position.
i.iterSpan = i.iter.Prev()
if invariants.Enabled && i.iterSpan == nil {
if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
panic("pebble: invariant violation: no previous span while switching directions")
}
// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Next()
}
// i.iterSpan == nil
//
// The inner iterator may return nil when it encounters an error. If there
// was an error, we don't know whether there is another span we should
// defragment or not. Return nil so that the caller knows they should check
// Error().
if i.iter.Error() != nil {
return nil
}
i.curr.Keys = i.keysBuf
return &i.curr
}
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Prev()
}
// i.iterSpan == nil
//
// The inner iterator may return nil when it encounters an error. If there
// was an error, we don't know whether there is another span we should
// defragment or not. Return nil so that the caller knows they should check
// Error().
if i.iter.Error() != nil {
return nil
}
i.curr.Keys = i.keysBuf
return &i.curr
}

25
vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go generated vendored
View File

@@ -4,6 +4,8 @@
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// FilterFunc defines a transform from the input Span into the output Span. The
// function returns true if the Span should be returned by the iterator, and
// false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
type filteringIter struct {
iter FragmentIterator
filterFn FilterFunc
cmp base.Compare
// span is a mutable Span passed to the filterFn. The filterFn is free to
// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
// Filter returns a new filteringIter that will filter the Spans from the
// provided child iterator using the provided FilterFunc.
func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter}
func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
}
// SeekGE implements FragmentIterator.
func (i *filteringIter) SeekGE(key []byte) *Span {
return i.filter(i.iter.SeekGE(key), +1)
span := i.filter(i.iter.SeekGE(key), +1)
// i.filter could return a span that's less than key, _if_ the filterFunc
// (which has no knowledge of the seek key) mutated the span to end at a key
// less than or equal to `key`. Detect this case and next/invalidate the iter.
if span != nil && i.cmp(span.End, key) <= 0 {
return i.Next()
}
return span
}
// SeekLT implements FragmentIterator.
func (i *filteringIter) SeekLT(key []byte) *Span {
return i.filter(i.iter.SeekLT(key), -1)
span := i.filter(i.iter.SeekLT(key), -1)
// i.filter could return a span that's >= key, _if_ the filterFunc (which has
// no knowledge of the seek key) mutated the span to start at a key greater
// than or equal to `key`. Detect this case and prev/invalidate the iter.
if span != nil && i.cmp(span.Start, key) >= 0 {
return i.Prev()
}
return span
}
// First implements FragmentIterator.

925
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored
View File

File diff suppressed because it is too large Load Diff

4
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored
View File

@@ -51,6 +51,8 @@ type FragmentIterator interface {
Prev() *Span
// Error returns any accumulated error.
//
// TODO(jackson): Lift errors into return values on the positioning methods.
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
@@ -62,7 +64,7 @@ type FragmentIterator interface {
// TableNewSpanIter creates a new iterator for range key spans for the given
// file.
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
// per-sstable span iterators.

4
vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go generated vendored
View File

@@ -98,7 +98,7 @@ func (l *LevelIter) Init(
) {
l.err = nil
l.level = level
l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
l.tableOpts = opts
l.cmp = cmp
l.iterFile = nil
l.newIter = newIter
@@ -179,7 +179,7 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur
return noFileLoaded
}
if indicator != fileAlreadyLoaded {
l.iter, l.err = l.newIter(file, &l.tableOpts)
l.iter, l.err = l.newIter(file, l.tableOpts)
indicator = newFileLoaded
}
if l.err != nil {

43
vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go generated vendored
View File

@@ -25,49 +25,6 @@ import (
// MergingIter implementation, but will require a bit of plumbing to thread the
// Equal function.
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
// provided output *Span pointer. The output Span's Keys slice may be reused
// by Transform to reduce allocations.
Transform(cmp base.Compare, in Span, out *Span) error
}
// The TransformerFunc type is an adapter to allow the use of ordinary functions
// as Transformers. If f is a function with the appropriate signature,
// TransformerFunc(f) is a Transformer that calls f.
type TransformerFunc func(base.Compare, Span, *Span) error
// Transform calls f(cmp, in, out).
func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
return tf(cmp, in, out)
}
var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
})
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
for _, k := range s.Keys {
// NB: The InternalKeySeqNumMax value is used for the batch snapshot
// because a batch's visible span keys are filtered when they're
// fragmented. There's no requirement to enforce visibility at
// iteration time.
if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
dst.Keys = append(dst.Keys, k)
}
}
return nil
})
}
// MergingIter merges spans across levels of the LSM, exposing an iterator over
// spans that yields sets of spans fragmented at unique user key boundaries.
//

50
vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go generated vendored Normal file
View File

@@ -0,0 +1,50 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
// provided output *Span pointer. The output Span's Keys slice may be reused
// by Transform to reduce allocations.
Transform(cmp base.Compare, in Span, out *Span) error
}
// The TransformerFunc type is an adapter to allow the use of ordinary functions
// as Transformers. If f is a function with the appropriate signature,
// TransformerFunc(f) is a Transformer that calls f.
type TransformerFunc func(base.Compare, Span, *Span) error
// Transform calls f(cmp, in, out).
func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
return tf(cmp, in, out)
}
var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
})
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
for _, k := range s.Keys {
// NB: The InternalKeySeqNumMax value is used for the batch snapshot
// because a batch's visible span keys are filtered when they're
// fragmented. There's no requirement to enforce visibility at
// iteration time.
if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
dst.Keys = append(dst.Keys, k)
}
}
return nil
})
}

19
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored
View File

@@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base"
// truncated to be contained within the range [lower, upper). If start and end
// are specified, filter out any spans that are completely outside those bounds.
func Truncate(
cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
cmp base.Compare,
iter FragmentIterator,
lower, upper []byte,
start, end *base.InternalKey,
panicOnUpperTruncate bool,
) FragmentIterator {
return Filter(iter, func(in *Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
// Ignore this span if it lies completely outside [start, end].
// Ignore this span if it lies completely outside start, end. Note that
// end endInclusive indicated whether end is inclusive.
//
// The comparison between s.End and start is by user key only, as
// the span is exclusive at s.End, so comparing by user keys
@@ -48,13 +53,21 @@ func Truncate(
// Wholly within the end bound. Keep it.
}
}
var truncated bool
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
out.Start = lower
}
if cmp(in.End, upper) > 0 {
truncated = true
out.End = upper
}
if panicOnUpperTruncate && truncated {
panic("pebble: upper bound should not be truncated")
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}, cmp)
}

34
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored
View File

@@ -108,7 +108,7 @@ type annotation struct {
}
type leafNode struct {
ref int32
ref atomic.Int32
count int16
leaf bool
// subtreeCount holds the count of files in the entire subtree formed by
@@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node {
func newLeafNode() *node {
n := leafToNode(new(leafNode))
n.leaf = true
n.ref = 1
n.ref.Store(1)
return n
}
func newNode() *node {
n := new(node)
n.ref = 1
n.ref.Store(1)
return n
}
@@ -159,7 +159,7 @@ func newNode() *node {
// When a node is cloned, the provided pointer will be redirected to the new
// mutable node.
func mut(n **node) *node {
if atomic.LoadInt32(&(*n).ref) == 1 {
if (*n).ref.Load() == 1 {
// Exclusive ownership. Can mutate in place.
// Whenever a node will be mutated, reset its annotations to be marked
@@ -186,7 +186,7 @@ func mut(n **node) *node {
// incRef acquires a reference to the node.
func (n *node) incRef() {
atomic.AddInt32(&n.ref, 1)
n.ref.Add(1)
}
// decRef releases a reference to the node. If requested, the method will unref
@@ -195,8 +195,8 @@ func (n *node) incRef() {
// new nodes pass contentsToo=false to preserve existing reference counts during
// operations that should yield a net-zero change to descendant refcounts.
// When a node is released, its contained files are dereferenced.
func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if atomic.AddInt32(&n.ref, -1) > 0 {
func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
if n.ref.Add(-1) > 0 {
// Other references remain. Can't free.
return
}
@@ -207,7 +207,7 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
// nodes, and they want to preserve the existing reference count.
if contentsToo {
for _, f := range n.items[:n.count] {
if atomic.AddInt32(&f.Refs, -1) == 0 {
if f.Unref() == 0 {
// There are two sources of node dereferences: tree mutations
// and Version dereferences. Files should only be made obsolete
// during Version dereferences, during which `obsolete` will be
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if obsolete == nil {
panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
}
*obsolete = append(*obsolete, f)
// Reference counting is performed on the FileBacking. In the case
// of a virtual sstable, this reference counting is performed on
// a FileBacking which is shared by every single virtual sstable
// with the same backing sstable. If the reference count hits 0,
// then we know that the FileBacking won't be required by any
// sstable in Pebble, and that the backing sstable can be deleted.
*obsolete = append(*obsolete, f.FileBacking)
}
}
if !n.leaf {
@@ -241,7 +247,7 @@ func (n *node) clone() *node {
c.subtreeCount = n.subtreeCount
// Increase the refcount of each contained item.
for _, f := range n.items[:n.count] {
atomic.AddInt32(&f.Refs, 1)
f.Ref()
}
if !c.leaf {
// Copy children and increase each refcount.
@@ -761,8 +767,8 @@ type btree struct {
// Release dereferences and clears the root node of the btree, removing all
// items from the btree. In doing so, it decrements contained file counts.
// It returns a slice of newly obsolete files, if any.
func (t *btree) Release() (obsolete []*FileMetadata) {
// It returns a slice of newly obsolete backing files, if any.
func (t *btree) Release() (obsolete []*FileBacking) {
if t.root != nil {
t.root.decRef(true /* contentsToo */, &obsolete)
t.root = nil
@@ -800,7 +806,7 @@ func (t *btree) Delete(item *FileMetadata) (obsolete bool) {
return false
}
if out := mut(&t.root).Remove(t.cmp, item); out != nil {
obsolete = atomic.AddInt32(&out.Refs, -1) == 0
obsolete = out.Unref() == 0
}
if invariants.Enabled {
t.root.verifyInvariants()
@@ -832,7 +838,7 @@ func (t *btree) Insert(item *FileMetadata) error {
newRoot.subtreeCount = t.root.subtreeCount + splitNode.subtreeCount + 1
t.root = newRoot
}
atomic.AddInt32(&item.Refs, 1)
item.Ref()
err := mut(&t.root).Insert(t.cmp, item)
if invariants.Enabled {
t.root.verifyInvariants()

77
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored
View File

@@ -15,20 +15,28 @@ import (
// LevelMetadata contains metadata for all of the files within
// a level of the LSM.
type LevelMetadata struct {
level int
tree btree
level int
totalSize uint64
// NumVirtual is the number of virtual sstables in the level.
NumVirtual uint64
// VirtualSize is the size of the virtual sstables in the level.
VirtualSize uint64
tree btree
}
// clone makes a copy of the level metadata, implicitly increasing the ref
// count of every file contained within lm.
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
tree: lm.tree.Clone(),
level: lm.level,
totalSize: lm.totalSize,
NumVirtual: lm.NumVirtual,
VirtualSize: lm.VirtualSize,
tree: lm.tree.Clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
return lm.tree.Release()
}
@@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
var lm LevelMetadata
lm.level = level
lm.tree, _ = makeBTree(bcmp, files)
for _, f := range files {
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
}
return lm
}
@@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
return t, newLevelSlice(t.Iter())
}
func (lm *LevelMetadata) insert(f *FileMetadata) error {
if err := lm.tree.Insert(f); err != nil {
return err
}
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
return nil
}
func (lm *LevelMetadata) remove(f *FileMetadata) bool {
lm.totalSize -= f.Size
if f.Virtual {
lm.NumVirtual--
lm.VirtualSize -= f.Size
}
return lm.tree.Delete(f)
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.Count() == 0
@@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int {
return lm.tree.Count()
}
// Size returns the cumulative size of all the files within the level.
func (lm *LevelMetadata) Size() uint64 {
return lm.totalSize
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.Iter()}
@@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
return sum
}
// NumVirtual returns the number of virtual sstables in the level. Its runtime is
// linear in the length of the slice.
func (ls *LevelSlice) NumVirtual() uint64 {
var n uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
n++
}
}
return n
}
// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
// level.
func (ls *LevelSlice) VirtualSizeSum() uint64 {
var sum uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
sum += f.Size
}
}
return sum
}
// Reslice constructs a new slice backed by the same underlying level, with
// new start and end positions. Reslice invokes the provided function, passing
// two LevelIterators: one positioned to i's inclusive start and one

464
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored
View File

@@ -49,6 +49,9 @@ type TableStats struct {
// The number of point and range deletion entries in the table.
NumDeletions uint64
// NumRangeKeySets is the total number of range key sets in the table.
//
// NB: If there's a chance that the sstable contains any range key sets,
// then NumRangeKeySets must be > 0.
NumRangeKeySets uint64
// Estimate of the total disk space that may be dropped by this table's
// point deletions by compacting them.
@@ -60,9 +63,11 @@ type TableStats struct {
// overlapping data in L0 and ignores L0 sublevels, but the error that
// introduces is expected to be small.
//
// Tables in the bottommost level of the LSM may have a nonzero estimate
// if snapshots or move compactions prevented the elision of their range
// tombstones.
// Tables in the bottommost level of the LSM may have a nonzero estimate if
// snapshots or move compactions prevented the elision of their range
// tombstones. A table in the bottommost level that was ingested into L6
// will have a zero estimate, because the file's sequence numbers indicate
// that the tombstone cannot drop any data contained within the file itself.
RangeDeletionsBytesEstimate uint64
// Total size of value blocks and value index block.
ValueBlocksSize uint64
@@ -119,43 +124,72 @@ func (s CompactionState) String() string {
}
}
// FileMetadata holds the metadata for an on-disk table.
// FileMetadata is maintained for leveled-ssts, i.e., they belong to a level of
// some version. FileMetadata does not contain the actual level of the sst,
// since such leveled-ssts can move across levels in different versions, while
// sharing the same FileMetadata. There are two kinds of leveled-ssts, physical
// and virtual. Underlying both leveled-ssts is a backing-sst, for which the
// only state is FileBacking. A backing-sst is level-less. It is possible for a
// backing-sst to be referred to by a physical sst in one version and by one or
// more virtual ssts in one or more versions. A backing-sst becomes obsolete
// and can be deleted once it is no longer required by any physical or virtual
// sst in any version.
//
// We maintain some invariants:
//
// 1. Each physical and virtual sst will have a unique FileMetadata.FileNum,
// and there will be exactly one FileMetadata associated with the FileNum.
//
// 2. Within a version, a backing-sst is either only referred to by one
// physical sst or one or more virtual ssts.
//
// 3. Once a backing-sst is referred to by a virtual sst in the latest version,
// it cannot go back to being referred to by a physical sst in any future
// version.
//
// Once a physical sst is no longer needed by any version, we will no longer
// maintain the file metadata associated with it. We will still maintain the
// FileBacking associated with the physical sst if the backing sst is required
// by any virtual ssts in any version.
type FileMetadata struct {
// Atomic contains fields which are accessed atomically. Go allocations
// are guaranteed to be 64-bit aligned which we take advantage of by
// placing the 64-bit fields which we access atomically at the beginning
// of the FileMetadata struct. For more information, see
// https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
Atomic struct {
// AllowedSeeks is used to determine if a file should be picked for
// a read triggered compaction. It is decremented when read sampling
// in pebble.Iterator after every after every positioning operation
// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
AllowedSeeks int64
// AllowedSeeks is used to determine if a file should be picked for
// a read triggered compaction. It is decremented when read sampling
// in pebble.Iterator after every after every positioning operation
// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
AllowedSeeks atomic.Int64
// statsValid is 1 if stats have been loaded for the table. The
// TableStats structure is populated only if valid is 1.
statsValid uint32
}
// statsValid indicates if stats have been loaded for the table. The
// TableStats structure is populated only if valid is true.
statsValid atomic.Bool
// FileBacking is the state which backs either a physical or virtual
// sstables.
FileBacking *FileBacking
// InitAllowedSeeks is the inital value of allowed seeks. This is used
// to re-set allowed seeks on a file once it hits 0.
InitAllowedSeeks int64
// Reference count for the file: incremented when a file is added to a
// version and decremented when the version is unreferenced. The file is
// obsolete when the reference count falls to zero.
Refs int32
// FileNum is the file number.
//
// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
FileNum base.FileNum
// Size is the size of the file, in bytes.
// Size is the size of the file, in bytes. Size is an approximate value for
// virtual sstables.
//
// INVARIANTS:
// - When !FileMetadata.Virtual, Size == FileBacking.Size.
// - Size should be non-zero. Size 0 virtual sstables must not be created.
Size uint64
// File creation time in seconds since the epoch (1970-01-01 00:00:00
// UTC). For ingested sstables, this corresponds to the time the file was
// ingested.
// ingested. For virtual sstables, this corresponds to the wall clock time
// when the FileMetadata for the virtual sstable was first created.
CreationTime int64
// Smallest and largest sequence numbers in the table, across both point and
// range keys.
// Lower and upper bounds for the smallest and largest sequence numbers in
// the table, across both point and range keys. For physical sstables, these
// values are tight bounds. For virtual sstables, there is no guarantee that
// there will be keys with SmallestSeqNum or LargestSeqNum within virtual
// sstable bounds.
SmallestSeqNum uint64
LargestSeqNum uint64
// SmallestPointKey and LargestPointKey are the inclusive bounds for the
@@ -178,8 +212,17 @@ type FileMetadata struct {
Smallest InternalKey
Largest InternalKey
// Stats describe table statistics. Protected by DB.mu.
//
// For virtual sstables, set stats upon virtual sstable creation as
// asynchronous computation of stats is not currently supported.
//
// TODO(bananabrick): To support manifest replay for virtual sstables, we
// probably need to compute virtual sstable stats asynchronously. Otherwise,
// we'd have to write virtual sstable stats to the version edit.
Stats TableStats
// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
// pick L0 compactions. Only accurate for the most recent Version.
SubLevel int
L0Index int
minIntervalIndex int
@@ -188,9 +231,6 @@ type FileMetadata struct {
// NB: the alignment of this struct is 8 bytes. We pack all the bools to
// ensure an optimal packing.
// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
// pick L0 compactions. Only accurate for the most recent Version.
//
// IsIntraL0Compacting is set to True if this file is part of an intra-L0
// compaction. When it's true, IsCompacting must also return true. If
// Compacting is true and IsIntraL0Compacting is false for an L0 file, the
@@ -226,6 +266,212 @@ type FileMetadata struct {
// key type (point or range) corresponds to the smallest and largest overall
// table bounds.
boundTypeSmallest, boundTypeLargest boundType
// Virtual is true if the FileMetadata belongs to a virtual sstable.
Virtual bool
}
// PhysicalFileMeta is used by functions which want a guarantee that their input
// belongs to a physical sst and not a virtual sst.
//
// NB: This type should only be constructed by calling
// FileMetadata.PhysicalMeta.
type PhysicalFileMeta struct {
*FileMetadata
}
// VirtualFileMeta is used by functions which want a guarantee that their input
// belongs to a virtual sst and not a physical sst.
//
// A VirtualFileMeta inherits all the same fields as a FileMetadata. These
// fields have additional invariants imposed on them, and/or slightly varying
// meanings:
// - Smallest and Largest (and their counterparts
// {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a
// key at that exact bound. We make the effort to determine the next smallest
// or largest key in an sstable after virtualizing it, to maintain this
// tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it
// could mean that a rangedel or range key ends at that user key, or has been
// truncated to that user key.
// - One invariant is that if a rangedel or range key is truncated on its
// upper bound, the virtual sstable *must* have a rangedel or range key
// sentinel key as its upper bound. This is because truncation yields
// an exclusive upper bound for the rangedel/rangekey, and if there are
// any points at that exclusive upper bound within the same virtual
// sstable, those could get uncovered by this truncation. We enforce this
// invariant in calls to keyspan.Truncate.
// - Size is an estimate of the size of the virtualized portion of this sstable.
// The underlying file's size is stored in FileBacking.Size, though it could
// also be estimated or could correspond to just the referenced portion of
// a file (eg. if the file originated on another node).
// - Size must be > 0.
// - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
// This means that all keys in the virtual sstable must have seqnums within
// [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
// a key with a seqnum at either of the bounds. Calculating tight seqnum
// bounds would be too expensive and deliver little value.
//
// NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
type VirtualFileMeta struct {
*FileMetadata
}
// PhysicalMeta should be the only source of creating the PhysicalFileMeta
// wrapper type.
func (m *FileMetadata) PhysicalMeta() PhysicalFileMeta {
if m.Virtual {
panic("pebble: file metadata does not belong to a physical sstable")
}
return PhysicalFileMeta{
m,
}
}
// VirtualMeta should be the only source of creating the VirtualFileMeta wrapper
// type.
func (m *FileMetadata) VirtualMeta() VirtualFileMeta {
if !m.Virtual {
panic("pebble: file metadata does not belong to a virtual sstable")
}
return VirtualFileMeta{
m,
}
}
// FileBacking either backs a single physical sstable, or one or more virtual
// sstables.
//
// See the comment above the FileMetadata type for sstable terminology.
type FileBacking struct {
// Reference count for the backing file on disk: incremented when a
// physical or virtual sstable which is backed by the FileBacking is
// added to a version and decremented when the version is unreferenced.
// We ref count in order to determine when it is safe to delete a
// backing sst file from disk. The backing file is obsolete when the
// reference count falls to zero.
refs atomic.Int32
// latestVersionRefs are the references to the FileBacking in the
// latest version. This reference can be through a single physical
// sstable in the latest version, or one or more virtual sstables in the
// latest version.
//
// INVARIANT: latestVersionRefs <= refs.
latestVersionRefs atomic.Int32
// VirtualizedSize is set iff the backing sst is only referred to by
// virtual ssts in the latest version. VirtualizedSize is the sum of the
// virtual sstable sizes of all of the virtual sstables in the latest
// version which are backed by the physical sstable. When a virtual
// sstable is removed from the latest version, we will decrement the
// VirtualizedSize. During compaction picking, we'll compensate a
// virtual sstable file size by
// (FileBacking.Size - FileBacking.VirtualizedSize) / latestVersionRefs.
// The intuition is that if FileBacking.Size - FileBacking.VirtualizedSize
// is high, then the space amplification due to virtual sstables is
// high, and we should pick the virtual sstable with a higher priority.
//
// TODO(bananabrick): Compensate the virtual sstable file size using
// the VirtualizedSize during compaction picking and test.
VirtualizedSize atomic.Uint64
DiskFileNum base.DiskFileNum
Size uint64
}
// InitPhysicalBacking allocates and sets the FileBacking which is required by a
// physical sstable FileMetadata.
//
// Ensure that the state required by FileBacking, such as the FileNum, is
// already set on the FileMetadata before InitPhysicalBacking is called.
// Calling InitPhysicalBacking only after the relevant state has been set in the
// FileMetadata is not necessary in tests which don't rely on FileBacking.
func (m *FileMetadata) InitPhysicalBacking() {
if m.Virtual {
panic("pebble: virtual sstables should use a pre-existing FileBacking")
}
if m.FileBacking == nil {
m.FileBacking = &FileBacking{Size: m.Size, DiskFileNum: m.FileNum.DiskFileNum()}
}
}
// InitProviderBacking creates a new FileBacking for a file backed by
// an objstorage.Provider.
func (m *FileMetadata) InitProviderBacking(fileNum base.DiskFileNum) {
if !m.Virtual {
panic("pebble: provider-backed sstables must be virtual")
}
if m.FileBacking == nil {
m.FileBacking = &FileBacking{DiskFileNum: fileNum}
}
}
// ValidateVirtual should be called once the FileMetadata for a virtual sstable
// is created to verify that the fields of the virtual sstable are sound.
func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {
if !m.Virtual {
panic("pebble: invalid virtual sstable")
}
if createdFrom.SmallestSeqNum != m.SmallestSeqNum {
panic("pebble: invalid smallest sequence number for virtual sstable")
}
if createdFrom.LargestSeqNum != m.LargestSeqNum {
panic("pebble: invalid largest sequence number for virtual sstable")
}
if createdFrom.FileBacking != nil && createdFrom.FileBacking != m.FileBacking {
panic("pebble: invalid physical sstable state for virtual sstable")
}
if m.Size == 0 {
panic("pebble: virtual sstable size must be set upon creation")
}
}
// Refs returns the refcount of backing sstable.
func (m *FileMetadata) Refs() int32 {
return m.FileBacking.refs.Load()
}
// Ref increments the ref count associated with the backing sstable.
func (m *FileMetadata) Ref() {
m.FileBacking.refs.Add(1)
}
// Unref decrements the ref count associated with the backing sstable.
func (m *FileMetadata) Unref() int32 {
v := m.FileBacking.refs.Add(-1)
if invariants.Enabled && v < 0 {
panic("pebble: invalid FileMetadata refcounting")
}
return v
}
// LatestRef increments the latest ref count associated with the backing
// sstable.
func (m *FileMetadata) LatestRef() {
m.FileBacking.latestVersionRefs.Add(1)
if m.Virtual {
m.FileBacking.VirtualizedSize.Add(m.Size)
}
}
// LatestUnref decrements the latest ref count associated with the backing
// sstable.
func (m *FileMetadata) LatestUnref() int32 {
if m.Virtual {
m.FileBacking.VirtualizedSize.Add(-m.Size)
}
v := m.FileBacking.latestVersionRefs.Add(-1)
if invariants.Enabled && v < 0 {
panic("pebble: invalid FileMetadata latest refcounting")
}
return v
}
// LatestRefs returns the latest ref count associated with the backing sstable.
func (m *FileMetadata) LatestRefs() int32 {
return m.FileBacking.latestVersionRefs.Load()
}
// SetCompactionState transitions this file's compaction state to the given
@@ -263,22 +509,14 @@ func (m *FileMetadata) IsCompacting() bool {
// returns true, the Stats field may be read (with or without holding the
// database mutex).
func (m *FileMetadata) StatsValid() bool {
return atomic.LoadUint32(&m.Atomic.statsValid) == 1
}
// StatsValidLocked returns true if the table stats have been populated.
// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the
// overhead of an atomic load. This is possible because table stats validity is
// only set while DB.mu is held.
func (m *FileMetadata) StatsValidLocked() bool {
return m.Atomic.statsValid == 1
return m.statsValid.Load()
}
// StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu
// while populating TableStats and calling StatsMarkValud. Once stats are
// populated, they must not be mutated.
func (m *FileMetadata) StatsMarkValid() {
atomic.StoreUint32(&m.Atomic.statsValid, 1)
m.statsValid.Store(true)
}
// ExtendPointKeyBounds attempts to extend the lower and upper point key bounds
@@ -369,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive
return true
}
// ContainedWithinSpan returns true if the file key range completely overlaps with the
// given range ("end" is assumed to exclusive).
func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool {
lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end)
return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel()))
}
// ContainsKeyType returns whether or not the file contains keys of the provided
// type.
func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
@@ -466,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
if !verbose {
return b.String()
}
fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum)
if m.HasPointKeys {
fmt.Fprintf(&b, " points:[%s-%s]",
m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format))
@@ -479,9 +725,9 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
// ParseFileMetadataDebug parses a FileMetadata from its DebugString
// representation.
func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
// Split lines of the form:
// 000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
// 000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...]
fields := strings.FieldsFunc(s, func(c rune) bool {
switch c {
case ':', '[', '-', ']':
@@ -491,10 +737,24 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
}
})
if len(fields)%3 != 0 {
return m, errors.Newf("malformed input: %s", s)
return nil, errors.Newf("malformed input: %s", s)
}
m := &FileMetadata{}
for len(fields) > 0 {
prefix := fields[0]
if prefix == "seqnums" {
smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum
fields = fields[3:]
continue
}
smallest := base.ParsePrettyInternalKey(fields[1])
largest := base.ParsePrettyInternalKey(fields[2])
switch prefix {
@@ -522,7 +782,8 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
m.HasPointKeys = true
}
return
m.InitPhysicalBacking()
return m, nil
}
// Validate validates the metadata for consistency with itself, returning an
@@ -584,6 +845,11 @@ func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error {
}
}
// Ensure that FileMetadata.Init was called.
if m.FileBacking == nil {
return base.CorruptionErrorf("file metadata FileBacking not set")
}
return nil
}
@@ -756,6 +1022,9 @@ func NewVersion(
} else {
v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp)
}
for _, f := range files[l] {
v.Levels[l].totalSize += f.Size
}
}
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
panic(err)
@@ -788,7 +1057,7 @@ func NewVersion(
// key in a higher level table that has both the same user key and a higher
// sequence number.
type Version struct {
refs int32
refs atomic.Int32
// The level 0 sstables are organized in a series of sublevels. Similar to
// the seqnum invariant in normal levels, there is no internal key in a
@@ -805,7 +1074,7 @@ type Version struct {
// usage.
//
// L0Sublevels.Levels contains L0 files ordered by sublevels. All the files
// in Files[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
// in Levels[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
// a reference to that slice, as that slice is necessary for iterator
// creation and needs to outlast L0Sublevels.
L0Sublevels *L0Sublevels
@@ -820,7 +1089,7 @@ type Version struct {
// The callback to invoke when the last reference to a version is
// removed. Will be called with list.mu held.
Deleted func(obsolete []*FileMetadata)
Deleted func(obsolete []*FileBacking)
// Stats holds aggregated stats about the version maintained from
// version to version.
@@ -849,15 +1118,21 @@ func (v *Version) DebugString(format base.FormatKey) string {
return v.string(format, true)
}
func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string {
var buf bytes.Buffer
for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- {
fmt.Fprintf(&buf, "0.%d:\n", sublevel)
sublevels[sublevel].Each(func(f *FileMetadata) {
fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose))
})
}
return buf.String()
}
func (v *Version) string(format base.FormatKey, verbose bool) string {
var buf bytes.Buffer
if len(v.L0SublevelFiles) > 0 {
for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
fmt.Fprintf(&buf, "0.%d:\n", sublevel)
v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) {
fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose))
})
}
fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles))
}
for level := 1; level < NumLevels; level++ {
if v.Levels[level].Empty() {
@@ -898,7 +1173,7 @@ func ParseVersionDebug(
m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
m.HasPointKeys = true
}
files[level] = append(files[level], &m)
files[level] = append(files[level], m)
}
}
// Reverse the order of L0 files. This ensures we construct the same
@@ -912,12 +1187,12 @@ func ParseVersionDebug(
// Refs returns the number of references to the version.
func (v *Version) Refs() int32 {
return atomic.LoadInt32(&v.refs)
return v.refs.Load()
}
// Ref increments the version refcount.
func (v *Version) Ref() {
atomic.AddInt32(&v.refs, 1)
v.refs.Add(1)
}
// Unref decrements the version refcount. If the last reference to the version
@@ -925,12 +1200,11 @@ func (v *Version) Ref() {
// Deleted callback is invoked. Requires that the VersionList mutex is NOT
// locked.
func (v *Version) Unref() {
if atomic.AddInt32(&v.refs, -1) == 0 {
obsolete := v.unrefFiles()
if v.refs.Add(-1) == 0 {
l := v.list
l.mu.Lock()
l.Remove(v)
v.Deleted(obsolete)
v.Deleted(v.unrefFiles())
l.mu.Unlock()
}
}
@@ -940,14 +1214,14 @@ func (v *Version) Unref() {
// the Deleted callback is invoked. Requires that the VersionList mutex is
// already locked.
func (v *Version) UnrefLocked() {
if atomic.AddInt32(&v.refs, -1) == 0 {
if v.refs.Add(-1) == 0 {
v.list.Remove(v)
v.Deleted(v.unrefFiles())
}
}
func (v *Version) unrefFiles() []*FileMetadata {
var obsolete []*FileMetadata
func (v *Version) unrefFiles() []*FileBacking {
var obsolete []*FileBacking
for _, lm := range v.Levels {
obsolete = append(obsolete, lm.release()...)
}
@@ -1079,16 +1353,20 @@ func (v *Version) Overlaps(
// CheckOrdering checks that the files are consistent with respect to
// increasing file numbers (for level 0 files) and increasing and non-
// overlapping internal key ranges (for level non-0 files).
func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
func (v *Version) CheckOrdering(
cmp Compare, format base.FormatKey, order OrderingInvariants,
) error {
for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
sublevelIter := v.L0SublevelFiles[sublevel].Iter()
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
// Sublevels have NEVER allowed split user keys, so we can pass
// ProhibitSplitUserKeys.
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
for level, lm := range v.Levels {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
@@ -1157,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
v.list = nil // avoid memory leaks
}
// OrderingInvariants dictates the file ordering invariants active.
type OrderingInvariants int8
const (
// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
// contain the same user key.
ProhibitSplitUserKeys OrderingInvariants = iota
// AllowSplitUserKeys indicates that adjacent files within a level may
// contain the same user key. This is only allowed by historical format
// major versions.
//
// TODO(jackson): Remove.
AllowSplitUserKeys
)
// CheckOrdering checks that the files are consistent with respect to
// seqnums (for level 0 files -- see detailed comment below) and increasing and non-
// overlapping internal key ranges (for non-level 0 files).
func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
//
// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
// inclusive of the same user key. Pebble no longer creates version edits
// installing such files, and Pebble databases with sufficiently high format
// major version should no longer have any such files within their LSM.
// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
// earlier format major versions.
func CheckOrdering(
cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
) error {
// The invariants to check for L0 sublevels are the same as the ones to
// check for all other levels. However, if L0 is not organized into
// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1238,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
// What's considered "overlapping" is dependent on the format
// major version. If ordering=ProhibitSplitUserKeys, then both
// files cannot contain keys with the same user keys. If the
// bounds have the same user key, the previous file's boundary
// must have a Trailer indicating that it's exclusive.
switch ordering {
case AllowSplitUserKeys:
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
case ProhibitSplitUserKeys:
if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
default:
panic("unreachable")
}
}
}

403
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored
View File

@@ -11,11 +11,11 @@ import (
"fmt"
"io"
"sort"
"sync/atomic"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// TODO(peter): describe the MANIFEST file format, independently of the C++
@@ -51,7 +51,9 @@ const (
tagMaxColumnFamily = 203
// Pebble tags.
tagNewFile5 = 104 // Range keys.
tagNewFile5 = 104 // Range keys.
tagCreatedBackingTable = 105
tagRemovedBackingTable = 106
// The custom tags sub-format used by tagNewFile4 and above.
customTagTerminate = 1
@@ -59,6 +61,7 @@ const (
customTagCreationTime = 6
customTagPathID = 65
customTagNonSafeIgnoreMask = 1 << 6
customTagVirtual = 66
)
// DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -73,6 +76,9 @@ type DeletedFileEntry struct {
type NewFileEntry struct {
Level int
Meta *FileMetadata
// BackingFileNum is only set during manifest replay, and only for virtual
// sstables.
BackingFileNum base.DiskFileNum
}
// VersionEdit holds the state for an edit to a Version along with other
@@ -110,9 +116,42 @@ type VersionEdit struct {
// found that there was no overlapping file at the higher level).
DeletedFiles map[DeletedFileEntry]*FileMetadata
NewFiles []NewFileEntry
// CreatedBackingTables can be used to preserve the FileBacking associated
// with a physical sstable. This is useful when virtual sstables in the
// latest version are reconstructed during manifest replay, and we also need
// to reconstruct the FileBacking which is required by these virtual
// sstables.
//
// INVARIANT: The FileBacking associated with a physical sstable must only
// be added as a backing file in the same version edit where the physical
// sstable is first virtualized. This means that the physical sstable must
// be present in DeletedFiles and that there must be at least one virtual
// sstable with the same FileBacking as the physical sstable in NewFiles. A
// file must be present in CreatedBackingTables in exactly one version edit.
// The physical sstable associated with the FileBacking must also not be
// present in NewFiles.
CreatedBackingTables []*FileBacking
// RemovedBackingTables is used to remove the FileBacking associated with a
// virtual sstable. Note that a backing sstable can be removed as soon as
// there are no virtual sstables in the latest version which are using the
// backing sstable, but the backing sstable doesn't necessarily have to be
// removed atomically with the version edit which removes the last virtual
// sstable associated with the backing sstable. The removal can happen in a
// future version edit.
//
// INVARIANT: A file must only be added to RemovedBackingTables if it was
// added to CreateBackingTables in a prior version edit. The same version
// edit also cannot have the same file present in both CreateBackingTables
// and RemovedBackingTables. A file must be present in RemovedBackingTables
// in exactly one version edit.
RemovedBackingTables []base.DiskFileNum
}
// Decode decodes an edit from the specified reader.
//
// Note that the Decode step will not set the FileBacking for virtual sstables
// and the responsibility is left to the caller. However, the Decode step will
// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles.
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(byteReader)
if !ok {
@@ -165,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
// NB: RocksDB does not use compaction pointers anymore.
case tagRemovedBackingTable:
n, err := d.readUvarint()
if err != nil {
return err
}
v.RemovedBackingTables = append(
v.RemovedBackingTables, base.FileNum(n).DiskFileNum(),
)
case tagCreatedBackingTable:
dfn, err := d.readUvarint()
if err != nil {
return err
}
size, err := d.readUvarint()
if err != nil {
return err
}
fileBacking := &FileBacking{
DiskFileNum: base.FileNum(dfn).DiskFileNum(),
Size: size,
}
v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking)
case tagDeletedFile:
level, err := d.readLevel()
if err != nil {
@@ -270,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
var markedForCompaction bool
var creationTime uint64
virtualState := struct {
virtual bool
backingFileNum uint64
}{}
if tag == tagNewFile4 || tag == tagNewFile5 {
for {
customTag, err := d.readUvarint()
@@ -278,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
if customTag == customTagTerminate {
break
} else if customTag == customTagVirtual {
virtualState.virtual = true
n, err := d.readUvarint()
if err != nil {
return err
}
virtualState.backingFileNum = n
continue
}
field, err := d.readBytes()
if err != nil {
return err
@@ -314,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
SmallestSeqNum: smallestSeqNum,
LargestSeqNum: largestSeqNum,
MarkedForCompaction: markedForCompaction,
Virtual: virtualState.virtual,
}
if tag != tagNewFile5 { // no range keys present
m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
@@ -345,10 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
}
m.boundsSet = true
v.NewFiles = append(v.NewFiles, NewFileEntry{
if !virtualState.virtual {
m.InitPhysicalBacking()
}
nfe := NewFileEntry{
Level: level,
Meta: m,
})
}
if virtualState.virtual {
nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum()
}
v.NewFiles = append(v.NewFiles, nfe)
case tagPrevLogNumber:
n, err := d.readUvarint()
@@ -367,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
return nil
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
var buf bytes.Buffer
if v.ComparerName != "" {
fmt.Fprintf(&buf, " comparer: %s", v.ComparerName)
@@ -399,7 +481,12 @@ func (v *VersionEdit) String() string {
fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum)
}
for _, nf := range v.NewFiles {
fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String())
fmt.Fprintf(&buf, " added: L%d", nf.Level)
if verbose {
fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
} else {
fmt.Fprintf(&buf, " %s", nf.Meta.String())
}
if nf.Meta.CreationTime != 0 {
fmt.Fprintf(&buf, " (%s)",
time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -409,6 +496,16 @@ func (v *VersionEdit) String() string {
return buf.String()
}
// DebugString is a more verbose version of String(). Use this in tests.
func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
return v.string(true /* verbose */, fmtKey)
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
return v.string(false /* verbose */, base.DefaultFormatter)
}
// Encode encodes an edit to the specified writer.
func (v *VersionEdit) Encode(w io.Writer) error {
e := versionEditEncoder{new(bytes.Buffer)}
@@ -429,6 +526,15 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(tagNextFileNumber)
e.writeUvarint(uint64(v.NextFileNum))
}
for _, dfn := range v.RemovedBackingTables {
e.writeUvarint(tagRemovedBackingTable)
e.writeUvarint(uint64(dfn.FileNum()))
}
for _, fileBacking := range v.CreatedBackingTables {
e.writeUvarint(tagCreatedBackingTable)
e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum()))
e.writeUvarint(fileBacking.Size)
}
// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
// even though its value is zero. We detect this by encoding LastSeqNum when
// ComparerName is set.
@@ -442,7 +548,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(uint64(x.FileNum))
}
for _, x := range v.NewFiles {
customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
var tag uint64
switch {
case x.Meta.HasRangeKeys:
@@ -495,6 +601,10 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(customTagNeedsCompaction)
e.writeBytes([]byte{1})
}
if x.Meta.Virtual {
e.writeUvarint(customTagVirtual)
e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum()))
}
e.writeUvarint(customTagTerminate)
}
}
@@ -502,6 +612,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
return err
}
// versionEditDecoder should be used to decode version edits.
type versionEditDecoder struct {
byteReader
}
@@ -581,10 +692,30 @@ func (e versionEditEncoder) writeUvarint(u uint64) {
// BulkVersionEdit summarizes the files added and deleted from a set of version
// edits.
//
// INVARIANTS:
// No file can be added to a level more than once. This is true globally, and
// also true for all of the calls to Accumulate for a single bulk version edit.
//
// No file can be removed from a level more than once. This is true globally,
// and also true for all of the calls to Accumulate for a single bulk version
// edit.
//
// A file must not be added and removed from a given level in the same version
// edit.
//
// A file that is being removed from a level must have been added to that level
// before (in a prior version edit). Note that a given file can be deleted from
// a level and added to another level in a single version edit
type BulkVersionEdit struct {
Added [NumLevels][]*FileMetadata
Added [NumLevels]map[base.FileNum]*FileMetadata
Deleted [NumLevels]map[base.FileNum]*FileMetadata
// AddedFileBacking is a map to support lookup so that we can populate the
// FileBacking of virtual sstables during manifest replay.
AddedFileBacking map[base.DiskFileNum]*FileBacking
RemovedFileBacking []base.DiskFileNum
// AddedByFileNum maps file number to file metadata for all added files
// from accumulated version edits. AddedByFileNum is only populated if set
// to non-nil by a caller. It must be set to non-nil when replaying
@@ -603,6 +734,19 @@ type BulkVersionEdit struct {
// Accumulate adds the file addition and deletions in the specified version
// edit to the bulk edit's internal state.
//
// INVARIANTS:
// If a file is added to a given level in a call to Accumulate and then removed
// from that level in a subsequent call, the file will not be present in the
// resulting BulkVersionEdit.Deleted for that level.
//
// After accumulation of version edits, the bulk version edit may have
// information about a file which has been deleted from a level, but it may
// not have information about the same file added to the same level. The add
// could've occurred as part of a previous bulk version edit. In this case,
// the deleted file must be present in BulkVersionEdit.Deleted, at the end
// of the accumulation, because we need to decrease the refcount of the
// deleted file in Apply.
func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
for df, m := range ve.DeletedFiles {
dmap := b.Deleted[df.Level]
@@ -624,7 +768,28 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
if m.MarkedForCompaction {
b.MarkedForCompactionCountDiff--
}
dmap[df.FileNum] = m
if _, ok := b.Added[df.Level][df.FileNum]; !ok {
dmap[df.FileNum] = m
} else {
// Present in b.Added for the same level.
delete(b.Added[df.Level], df.FileNum)
}
}
// Generate state for Added backing files. Note that these must be generated
// before we loop through the NewFiles, because we need to populate the
// FileBackings which might be used by the NewFiles loop.
if b.AddedFileBacking == nil {
b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
}
for _, fb := range ve.CreatedBackingTables {
if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
// There is already a FileBacking associated with fb.DiskFileNum.
// This should never happen. There must always be only one FileBacking
// associated with a backing sstable.
panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
}
b.AddedFileBacking[fb.DiskFileNum] = fb
}
for _, nf := range ve.NewFiles {
@@ -635,7 +800,21 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
}
}
b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta)
if nf.Meta.Virtual && nf.Meta.FileBacking == nil {
// FileBacking for a virtual sstable must only be nil if we're performing
// manifest replay.
nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum]
if nf.Meta.FileBacking == nil {
return errors.Errorf("FileBacking for virtual sstable must not be nil")
}
} else if nf.Meta.FileBacking == nil {
return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum)
}
if b.Added[nf.Level] == nil {
b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
}
b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta
if b.AddedByFileNum != nil {
b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta
}
@@ -643,36 +822,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
b.MarkedForCompactionCountDiff++
}
}
// Since a file can be removed from backing files in exactly one version
// edit it is safe to just append without any de-duplication.
b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)
return nil
}
// AccumulateIncompleteAndApplySingleVE should be called if a single version edit
// is to be applied to the provided curr Version and if the caller needs to
// update the versionSet.zombieTables map. This function exists separately from
// BulkVersionEdit.Apply because it is easier to reason about properties
// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we
// know that exactly one version edit is being accumulated.
//
// Note that the version edit passed into this function may be incomplete
// because compactions don't have the ref counting information necessary to
// populate VersionEdit.RemovedBackingTables. This function will complete such a
// version edit by populating RemovedBackingTables.
//
// Invariant: Any file being deleted through ve must belong to the curr Version.
// We can't have a delete for some arbitrary file which does not exist in curr.
func AccumulateIncompleteAndApplySingleVE(
ve *VersionEdit,
curr *Version,
cmp Compare,
formatKey base.FormatKey,
flushSplitBytes int64,
readCompactionRate int64,
backingStateMap map[base.DiskFileNum]*FileBacking,
addBackingFunc func(*FileBacking),
removeBackingFunc func(base.DiskFileNum),
orderingInvariants OrderingInvariants,
) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
if len(ve.RemovedBackingTables) != 0 {
panic("pebble: invalid incomplete version edit")
}
var b BulkVersionEdit
err := b.Accumulate(ve)
if err != nil {
return nil, nil, err
}
zombies = make(map[base.DiskFileNum]uint64)
v, err := b.Apply(
curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
)
if err != nil {
return nil, nil, err
}
for _, s := range b.AddedFileBacking {
addBackingFunc(s)
}
for fileNum := range zombies {
if _, ok := backingStateMap[fileNum]; ok {
// This table was backing some virtual sstable in the latest version,
// but is now a zombie. We add RemovedBackingTables entries for
// these, before the version edit is written to disk.
ve.RemovedBackingTables = append(
ve.RemovedBackingTables, fileNum,
)
removeBackingFunc(fileNum)
}
}
return v, zombies, nil
}
// Apply applies the delta b to the current version to produce a new
// version. The new version is consistent with respect to the comparer cmp.
//
// curr may be nil, which is equivalent to a pointer to a zero version.
//
// On success, a map of zombie files containing the file numbers and sizes of
// deleted files is returned. These files are considered zombies because they
// are no longer referenced by the returned Version, but cannot be deleted from
// disk as they are still in use by the incoming Version.
// On success, if a non-nil zombies map is provided to Apply, the map is updated
// with file numbers and files sizes of deleted files. These files are
// considered zombies because they are no longer referenced by the returned
// Version, but cannot be deleted from disk as they are still in use by the
// incoming Version.
func (b *BulkVersionEdit) Apply(
curr *Version,
cmp Compare,
formatKey base.FormatKey,
flushSplitBytes int64,
readCompactionRate int64,
) (_ *Version, zombies map[base.FileNum]uint64, _ error) {
addZombie := func(fileNum base.FileNum, size uint64) {
if zombies == nil {
zombies = make(map[base.FileNum]uint64)
}
zombies[fileNum] = size
}
// The remove zombie function is used to handle tables that are moved from
// one level to another during a version edit (i.e. a "move" compaction).
removeZombie := func(fileNum base.FileNum) {
zombies map[base.DiskFileNum]uint64,
orderingInvariants OrderingInvariants,
) (*Version, error) {
addZombie := func(state *FileBacking) {
if zombies != nil {
delete(zombies, fileNum)
zombies[state.DiskFileNum] = state.Size
}
}
removeZombie := func(state *FileBacking) {
if zombies != nil {
delete(zombies, state.DiskFileNum)
}
}
@@ -684,7 +928,7 @@ func (b *BulkVersionEdit) Apply(
}
v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff
if v.Stats.MarkedForCompaction < 0 {
return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
}
for level := range v.Levels {
@@ -705,7 +949,7 @@ func (b *BulkVersionEdit) Apply(
// Initialize L0Sublevels.
if curr == nil || curr.L0Sublevels == nil {
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
} else {
v.L0Sublevels = curr.L0Sublevels
@@ -718,47 +962,70 @@ func (b *BulkVersionEdit) Apply(
// Some edits on this level.
lm := &v.Levels[level]
lmRange := &v.RangeKeyLevels[level]
addedFiles := b.Added[level]
deletedMap := b.Deleted[level]
if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
return nil, nil, base.CorruptionErrorf(
addedFilesMap := b.Added[level]
deletedFilesMap := b.Deleted[level]
if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 {
return nil, base.CorruptionErrorf(
"pebble: internal error: No current or added files but have deleted files: %d",
errors.Safe(len(deletedMap)))
errors.Safe(len(deletedFilesMap)))
}
// NB: addedFiles may be empty and it also is not necessarily
// internally consistent: it does not reflect deletions in deletedMap.
// NB: addedFilesMap may be empty. If a file is present in addedFilesMap
// for a level, it won't be present in deletedFilesMap for the same
// level.
for _, f := range deletedMap {
addZombie(f.FileNum, f.Size)
if obsolete := v.Levels[level].tree.Delete(f); obsolete {
for _, f := range deletedFilesMap {
if obsolete := v.Levels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
// file's reference count dropping to zero.
err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
return nil, nil, err
return nil, err
}
if f.HasRangeKeys {
if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
// file's reference count dropping to zero.
err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum)
return nil, nil, err
return nil, err
}
}
// Note that a backing sst will only become a zombie if the
// references to it in the latest version is 0. We will remove the
// backing sst from the zombie list in the next loop if one of the
// addedFiles in any of the levels is referencing the backing sst.
// This is possible if a physical sstable is virtualized, or if it
// is moved.
latestRefCount := f.LatestRefs()
if latestRefCount <= 0 {
// If a file is present in deletedFilesMap for a level, then it
// must have already been added to the level previously, which
// means that its latest ref count cannot be 0.
err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum)
return nil, err
} else if f.LatestUnref() == 0 {
addZombie(f.FileBacking)
}
}
addedFiles := make([]*FileMetadata, 0, len(addedFilesMap))
for _, f := range addedFilesMap {
addedFiles = append(addedFiles, f)
}
// Sort addedFiles by file number. This isn't necessary, but tests which
// replay invalid manifests check the error output, and the error output
// depends on the order in which files are added to the btree.
sort.Slice(addedFiles, func(i, j int) bool {
return addedFiles[i].FileNum < addedFiles[j].FileNum
})
var sm, la *FileMetadata
for _, f := range addedFiles {
if _, ok := deletedMap[f.FileNum]; ok {
// Already called addZombie on this file in the preceding
// loop, so we don't need to do it here.
continue
}
// NB: allowedSeeks is used for read triggered compactions. It is set using
// Options.Experimental.ReadCompactionRate which defaults to 32KB.
var allowedSeeks int64
@@ -768,20 +1035,23 @@ func (b *BulkVersionEdit) Apply(
if allowedSeeks < 100 {
allowedSeeks = 100
}
atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks)
f.AllowedSeeks.Store(allowedSeeks)
f.InitAllowedSeeks = allowedSeeks
err := lm.tree.Insert(f)
err := lm.insert(f)
// We're adding this file to the new version, so increment the
// latest refs count.
f.LatestRef()
if err != nil {
return nil, nil, errors.Wrap(err, "pebble")
return nil, errors.Wrap(err, "pebble")
}
if f.HasRangeKeys {
err = lmRange.tree.Insert(f)
err = lmRange.insert(f)
if err != nil {
return nil, nil, errors.Wrap(err, "pebble")
return nil, errors.Wrap(err, "pebble")
}
}
removeZombie(f.FileNum)
removeZombie(f.FileBacking)
// Track the keys with the smallest and largest keys, so that we can
// check consistency of the modified span.
if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 {
@@ -793,27 +1063,36 @@ func (b *BulkVersionEdit) Apply(
}
if level == 0 {
if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 {
if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 {
// Flushes and ingestions that do not delete any L0 files do not require
// a regeneration of L0Sublevels from scratch. We can instead generate
// it incrementally.
var err error
// AddL0Files requires addedFiles to be sorted in seqnum order.
addedFiles = append([]*FileMetadata(nil), addedFiles...)
SortBySeqNum(addedFiles)
v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0])
if errors.Is(err, errInvalidL0SublevelsOpt) {
err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes)
} else if invariants.Enabled && err == nil {
copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes)
if err != nil {
panic(fmt.Sprintf("error when regenerating sublevels: %s", err))
}
s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels)
s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels)
if s1 != s2 {
panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2))
}
}
if err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
v.L0SublevelFiles = v.L0Sublevels.Levels
} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
continue
}
@@ -833,10 +1112,10 @@ func (b *BulkVersionEdit) Apply(
end.Prev()
}
})
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
}
}
return v, zombies, nil
return v, nil
}

2
vendor/github.com/cockroachdb/pebble/internal/private/sstable.go generated vendored
View File

@@ -8,7 +8,7 @@ import "github.com/cockroachdb/pebble/internal/base"
// SSTableCacheOpts is a hook for specifying cache options to
// sstable.NewReader.
var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{}
var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{}
// SSTableRawTombstonesOpt is a sstable.Reader option for disabling
// fragmentation of the range tombstones returned by

71
vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go generated vendored
View File

@@ -52,15 +52,15 @@ import (
// │
// ╰── <?>.Next
type UserIteratorConfig struct {
snapshot uint64
comparer *base.Comparer
miter keyspan.MergingIter
biter keyspan.BoundedIter
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
onlySets bool
bufs *Buffers
snapshot uint64
comparer *base.Comparer
miter keyspan.MergingIter
biter keyspan.BoundedIter
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
internalKeys bool
bufs *Buffers
}
// Buffers holds various buffers used for range key iteration. They're exposed
@@ -79,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
// Init initializes the range key iterator stack for user iteration. The
// resulting fragment iterator applies range key semantics, defragments spans
// according to their user-observable state and, if onlySets = true, removes all
// according to their user-observable state and, if !internalKeys, removes all
// Keys other than RangeKeySets describing the current state of range keys. The
// resulting spans contain Keys sorted by Suffix.
// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
// in which case they remain sorted by trailer descending).
//
// The snapshot sequence number parameter determines which keys are visible. Any
// keys not visible at the provided snapshot are ignored.
@@ -91,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
lower, upper []byte,
hasPrefix *bool,
prefix *[]byte,
onlySets bool,
internalKeys bool,
bufs *Buffers,
iters ...keyspan.FragmentIterator,
) keyspan.FragmentIterator {
ui.snapshot = snapshot
ui.comparer = comparer
ui.onlySets = onlySets
ui.internalKeys = internalKeys
ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
if internalKeys {
ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
} else {
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
}
ui.litersUsed = 0
ui.bufs = bufs
return &ui.diter
@@ -136,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
// of unset keys, removal of keys overwritten by a set at the same suffix, etc)
// and then non-RangeKeySet keys are removed. The resulting transformed spans
// only contain RangeKeySets describing the state visible at the provided
// sequence number, and hold their Keys sorted by Suffix.
// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
// is true, then keys remain sorted by trailer.
func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
// Apply shadowing of keys.
dst.Start = s.Start
@@ -148,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
return err
}
// During user iteration over range keys, unsets and deletes don't matter.
// Remove them if onlySets = true. This step helps logical defragmentation
// during iteration.
if ui.internalKeys {
if s.KeysOrder != keyspan.ByTrailerDesc {
panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
}
dst.Keys = ui.bufs.sortBuf.Keys
keyspan.SortKeysByTrailer(&dst.Keys)
return nil
}
// During user iteration over range keys, unsets and deletes don't matter. This
// step helps logical defragmentation during iteration.
keys := ui.bufs.sortBuf.Keys
dst.Keys = dst.Keys[:0]
for i := range keys {
@@ -164,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
// Skip.
continue
case base.InternalKeyKindRangeKeyDelete:
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
// Skip.
continue
default:
return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
}
@@ -193,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
// sequence numbers). It's intended for use during user iteration, when the
// wrapped keyspan iterator is merging spans across all levels of the LSM.
func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
// This method is not called with internalKeys = true.
if ui.internalKeys {
panic("unexpected call to ShouldDefragment with internalKeys = true")
}
// This implementation must only be used on spans that have transformed by
// ui.Transform. The transform applies shadowing, removes all keys besides
// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -208,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
ret := true
for i := range a.Keys {
if invariants.Enabled {
if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
panic("pebble: unexpected non-RangeKeySet during defragmentation")
}
if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||

387
vendor/github.com/cockroachdb/pebble/internal/rate/rate.go generated vendored
View File

@@ -1,387 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package rate provides a rate limiter.
package rate // import "github.com/cockroachdb/pebble/internal/rate"
import (
"context"
"math"
"sync"
"time"
"github.com/cockroachdb/errors"
)
// Limit defines the maximum frequency of some events.
// Limit is represented as number of events per second.
// A zero Limit allows no events.
type Limit float64
// Inf is the infinite rate limit; it allows all events (even if burst is zero).
const Inf = Limit(math.MaxFloat64)
// Every converts a minimum time interval between events to a Limit.
func Every(interval time.Duration) Limit {
if interval <= 0 {
return Inf
}
return 1 / Limit(interval.Seconds())
}
// A Limiter controls how frequently events are allowed to happen.
// It implements a "token bucket" of size b, initially full and refilled
// at rate r tokens per second.
// Informally, in any large enough time interval, the Limiter limits the
// rate to r tokens per second, with a maximum burst size of b events.
// As a special case, if r == Inf (the infinite rate), b is ignored.
// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
//
// The zero value is a valid Limiter, but it will reject all events.
// Use NewLimiter to create non-zero Limiters.
//
// Limiter has three main methods, Allow, Reserve, and Wait.
// Most callers should use Wait.
//
// Each of the three methods consumes a single token.
// They differ in their behavior when no token is available.
// If no token is available, Allow returns false.
// If no token is available, Reserve returns a reservation for a future token
// and the amount of time the caller must wait before using it.
// If no token is available, Wait blocks until one can be obtained
// or its associated context.Context is canceled.
//
// The methods AllowN, ReserveN, and WaitN consume n tokens.
type Limiter struct {
limit Limit
burst int
mu sync.Mutex
tokens float64
// last is the last time the limiter's tokens field was updated
last time.Time
// lastEvent is the latest time of a rate-limited event (past or future)
lastEvent time.Time
}
// Limit returns the maximum overall event rate.
func (lim *Limiter) Limit() Limit {
lim.mu.Lock()
defer lim.mu.Unlock()
return lim.limit
}
// Burst returns the maximum burst size. Burst is the maximum number of tokens
// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
// Burst values allow more events to happen at once.
// A zero Burst allows no events, unless limit == Inf.
func (lim *Limiter) Burst() int {
return lim.burst
}
// NewLimiter returns a new Limiter that allows events up to rate r and permits
// bursts of at most b tokens.
func NewLimiter(r Limit, b int) *Limiter {
return &Limiter{
limit: r,
burst: b,
}
}
// Allow is shorthand for AllowN(time.Now(), 1).
func (lim *Limiter) Allow() bool {
return lim.AllowN(time.Now(), 1)
}
// AllowN reports whether n events may happen at time now.
// Use this method if you intend to drop / skip events that exceed the rate limit.
// Otherwise use Reserve or Wait.
func (lim *Limiter) AllowN(now time.Time, n int) bool {
return lim.reserveN(now, n, 0).ok
}
// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
// A Reservation may be canceled, which may enable the Limiter to permit additional events.
type Reservation struct {
ok bool
lim *Limiter
tokens int
timeToAct time.Time
// This is the Limit at reservation time, it can change later.
limit Limit
}
// OK returns whether the limiter can provide the requested number of tokens
// within the maximum wait time. If OK is false, Delay returns InfDuration, and
// Cancel does nothing.
func (r *Reservation) OK() bool {
return r.ok
}
// Delay is shorthand for DelayFrom(time.Now()).
func (r *Reservation) Delay() time.Duration {
return r.DelayFrom(time.Now())
}
// InfDuration is the duration returned by Delay when a Reservation is not OK.
const InfDuration = time.Duration(1<<63 - 1)
// DelayFrom returns the duration for which the reservation holder must wait
// before taking the reserved action. Zero duration means act immediately.
// InfDuration means the limiter cannot grant the tokens requested in this
// Reservation within the maximum wait time.
func (r *Reservation) DelayFrom(now time.Time) time.Duration {
if !r.ok {
return InfDuration
}
delay := r.timeToAct.Sub(now)
if delay < 0 {
return 0
}
return delay
}
// Cancel is shorthand for CancelAt(time.Now()).
func (r *Reservation) Cancel() {
r.CancelAt(time.Now())
}
// CancelAt indicates that the reservation holder will not perform the reserved action
// and reverses the effects of this Reservation on the rate limit as much as possible,
// considering that other reservations may have already been made.
func (r *Reservation) CancelAt(now time.Time) {
if !r.ok {
return
}
r.lim.mu.Lock()
defer r.lim.mu.Unlock()
if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
return
}
// calculate tokens to restore
// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
// after r was obtained. These tokens should not be restored.
restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
if restoreTokens <= 0 {
return
}
// advance time to now
now, _, tokens := r.lim.advance(now)
// calculate new number of tokens
tokens += restoreTokens
if burst := float64(r.lim.burst); tokens > burst {
tokens = burst
}
// update state
r.lim.last = now
r.lim.tokens = tokens
if r.timeToAct == r.lim.lastEvent {
prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
if !prevEvent.Before(now) {
r.lim.lastEvent = prevEvent
}
}
}
// Reserve is shorthand for ReserveN(time.Now(), 1).
func (lim *Limiter) Reserve() Reservation {
return lim.ReserveN(time.Now(), 1)
}
// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
// The Limiter takes this Reservation into account when allowing future events.
// ReserveN returns false if n exceeds the Limiter's burst size.
// Usage example:
//
// r := lim.ReserveN(time.Now(), 1)
// if !r.OK() {
// // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
// return
// }
// time.Sleep(r.Delay())
// Act()
//
// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
// If you need to respect a deadline or cancel the delay, use Wait instead.
// To drop or skip events exceeding rate limit, use Allow instead.
func (lim *Limiter) ReserveN(now time.Time, n int) Reservation {
return lim.reserveN(now, n, InfDuration)
}
// Wait is shorthand for WaitN(ctx, 1).
func (lim *Limiter) Wait(ctx context.Context) (err error) {
return lim.WaitN(ctx, 1)
}
// WaitN blocks until lim permits n events to happen.
// It returns an error if n exceeds the Limiter's burst size, the Context is
// canceled, or the expected wait time exceeds the Context's Deadline.
func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
if n > lim.burst && lim.limit != Inf {
return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst))
}
// Check if ctx is already cancelled
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// Determine wait limit
now := time.Now()
waitLimit := InfDuration
if deadline, ok := ctx.Deadline(); ok {
waitLimit = deadline.Sub(now)
}
// Reserve
r := lim.reserveN(now, n, waitLimit)
if !r.ok {
return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n))
}
// Wait
d := r.DelayFrom(now)
if d <= 0 {
return nil
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
// We can proceed.
return nil
case <-ctx.Done():
// Context was canceled before we could proceed. Cancel the
// reservation, which may permit other events to proceed sooner.
r.Cancel()
return ctx.Err()
}
}
// Delay is shorthand for DelayN(time.Now(), 1).
func (lim *Limiter) Delay() time.Duration {
return lim.DelayN(time.Now(), 1)
}
// DelayN returns the delay to wait to permit n events to happen. Zero duration
// means act immediately. InfDuration means the limiter cannot grant the tokens
// requested within the maximum wait time.
func (lim *Limiter) DelayN(now time.Time, n int) time.Duration {
r := lim.reserveN(now, n, InfDuration)
return r.DelayFrom(now)
}
// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
func (lim *Limiter) SetLimit(newLimit Limit) {
lim.SetLimitAt(time.Now(), newLimit)
}
// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
// before SetLimitAt was called.
func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
lim.last = now
lim.tokens = tokens
lim.limit = newLimit
}
// reserveN is a helper method for AllowN, ReserveN, and WaitN.
// maxFutureReserve specifies the maximum reservation wait duration allowed.
// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
lim.mu.Lock()
if lim.limit == Inf {
lim.mu.Unlock()
return Reservation{
ok: true,
lim: lim,
tokens: n,
timeToAct: now,
}
}
now, last, tokens := lim.advance(now)
// Calculate the remaining number of tokens resulting from the request.
tokens -= float64(n)
// Calculate the wait duration
var waitDuration time.Duration
if tokens < 0 {
waitDuration = lim.limit.durationFromTokens(-tokens)
}
// Decide result
ok := n <= lim.burst && waitDuration <= maxFutureReserve
// Prepare reservation
r := Reservation{
ok: ok,
lim: lim,
limit: lim.limit,
}
if ok {
r.tokens = n
r.timeToAct = now.Add(waitDuration)
}
// Update state
if ok {
lim.last = now
lim.tokens = tokens
lim.lastEvent = r.timeToAct
} else {
lim.last = last
}
lim.mu.Unlock()
return r
}
// advance calculates and returns an updated state for lim resulting from the passage of time.
// lim is not changed.
func (lim *Limiter) advance(
now time.Time,
) (newNow time.Time, newLast time.Time, newTokens float64) {
last := lim.last
if now.Before(last) {
last = now
}
// Avoid making delta overflow below when last is very old.
maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
elapsed := now.Sub(last)
if elapsed > maxElapsed {
elapsed = maxElapsed
}
// Calculate the new number of tokens, due to time that passed.
delta := lim.limit.tokensFromDuration(elapsed)
tokens := lim.tokens + delta
if burst := float64(lim.burst); tokens > burst {
tokens = burst
}
return now, last, tokens
}
// durationFromTokens is a unit conversion function from the number of tokens to the duration
// of time it takes to accumulate them at a rate of limit tokens per second.
func (limit Limit) durationFromTokens(tokens float64) time.Duration {
seconds := tokens / float64(limit)
return time.Nanosecond * time.Duration(1e9*seconds)
}
// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
// which could be accumulated during that duration at a rate of limit tokens per second.
func (limit Limit) tokensFromDuration(d time.Duration) float64 {
return d.Seconds() * float64(limit)
}

229
vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go generated vendored
View File

@@ -20,17 +20,27 @@ import (
"strings"
"github.com/cockroachdb/pebble/internal/base"
"golang.org/x/exp/constraints"
"golang.org/x/exp/rand"
)
const alpha = "abcdefghijklmnopqrstuvwxyz"
const suffixDelim = '@'
var inverseAlphabet = make(map[byte]int64, len(alpha))
func init() {
for i := range alpha {
inverseAlphabet[alpha[i]] = int64(i)
}
}
// MaxSuffixLen is the maximum length of a suffix generated by this package.
var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64))
var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64)))
// Comparer is the comparer for test keys generated by this package.
var Comparer *base.Comparer = &base.Comparer{
var Comparer = &base.Comparer{
Compare: compare,
Equal: func(a, b []byte) bool { return compare(a, b) == 0 },
AbbreviatedKey: func(k []byte) uint64 {
@@ -136,7 +146,7 @@ func compareTimestamps(a, b []byte) int {
// Keyspace describes a finite keyspace of unsuffixed test keys.
type Keyspace interface {
// Count returns the number of keys that exist within this keyspace.
Count() int
Count() int64
// MaxLen returns the maximum length, in bytes, of a key within this
// keyspace. This is only guaranteed to return an upper bound.
@@ -144,20 +154,21 @@ type Keyspace interface {
// Slice returns the sub-keyspace from index i, inclusive, to index j,
// exclusive. The receiver is unmodified.
Slice(i, j int) Keyspace
Slice(i, j int64) Keyspace
// EveryN returns a key space that includes 1 key for every N keys in the
// original keyspace. The receiver is unmodified.
EveryN(n int) Keyspace
EveryN(n int64) Keyspace
key(buf []byte, i int) int
// key writes the i-th key to the buffer and returns the length.
key(buf []byte, i int64) int
}
// Divvy divides the provided keyspace into N equal portions, containing
// disjoint keys evenly distributed across the keyspace.
func Divvy(ks Keyspace, n int) []Keyspace {
func Divvy(ks Keyspace, n int64) []Keyspace {
ret := make([]Keyspace, n)
for i := 0; i < n; i++ {
for i := int64(0); i < n; i++ {
ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
}
return ret
@@ -175,7 +186,7 @@ func Alpha(maxLength int) Keyspace {
// KeyAt returns the i-th key within the keyspace with a suffix encoding the
// timestamp t.
func KeyAt(k Keyspace, i int, t int) []byte {
func KeyAt(k Keyspace, i int64, t int64) []byte {
b := make([]byte, k.MaxLen()+MaxSuffixLen)
return b[:WriteKeyAt(b, k, i, t)]
}
@@ -183,20 +194,20 @@ func KeyAt(k Keyspace, i int, t int) []byte {
// WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
// suffix encoding the timestamp t suffix. It returns the number of bytes
// written.
func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int {
func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int {
n := WriteKey(dst, k, i)
n += WriteSuffix(dst[n:], t)
return n
}
// Suffix returns the test keys suffix representation of timestamp t.
func Suffix(t int) []byte {
func Suffix(t int64) []byte {
b := make([]byte, MaxSuffixLen)
return b[:WriteSuffix(b, t)]
}
// SuffixLen returns the exact length of the given suffix when encoded.
func SuffixLen(t int) int {
func SuffixLen(t int64) int {
// Begin at 1 for the '@' delimiter, 1 for a single digit.
n := 2
t /= 10
@@ -208,42 +219,42 @@ func SuffixLen(t int) int {
}
// ParseSuffix returns the integer representation of the encoded suffix.
func ParseSuffix(s []byte) (int, error) {
return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
func ParseSuffix(s []byte) (int64, error) {
return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64)
}
// WriteSuffix writes the test keys suffix representation of timestamp t to dst,
// returning the number of bytes written.
func WriteSuffix(dst []byte, t int) int {
func WriteSuffix(dst []byte, t int64) int {
dst[0] = suffixDelim
n := 1
n += len(strconv.AppendInt(dst[n:n], int64(t), 10))
n += len(strconv.AppendInt(dst[n:n], t, 10))
return n
}
// Key returns the i-th unsuffixed key within the keyspace.
func Key(k Keyspace, i int) []byte {
func Key(k Keyspace, i int64) []byte {
b := make([]byte, k.MaxLen())
return b[:k.key(b, i)]
}
// WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
// returns the number of bytes written.
func WriteKey(dst []byte, k Keyspace, i int) int {
func WriteKey(dst []byte, k Keyspace, i int64) int {
return k.key(dst, i)
}
type alphabet struct {
alphabet []byte
maxLength int
headSkip int
tailSkip int
increment int
headSkip int64
tailSkip int64
increment int64
}
func (a alphabet) Count() int {
func (a alphabet) Count() int64 {
// Calculate the total number of keys, ignoring the increment.
total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip)
total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip
// The increment dictates that we take every N keys, where N = a.increment.
// Consider a total containing the 5 keys:
@@ -263,20 +274,25 @@ func (a alphabet) MaxLen() int {
return a.maxLength
}
func (a alphabet) Slice(i, j int) Keyspace {
func (a alphabet) Slice(i, j int64) Keyspace {
s := a
s.headSkip += i
s.tailSkip += a.Count() - j
return s
}
func (a alphabet) EveryN(n int) Keyspace {
func (a alphabet) EveryN(n int64) Keyspace {
s := a
s.increment *= n
return s
}
func keyCount(n, l int) int {
func keyCount(n, l int) int64 {
if n == 0 {
return 0
} else if n == 1 {
return int64(l)
}
// The number of representable keys in the keyspace is a function of the
// length of the alphabet n and the max key length l. Consider how the
// number of representable keys grows as l increases:
@@ -286,10 +302,10 @@ func keyCount(n, l int) int {
// l = 3: n + n^2 + n^3
// ...
// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1)
return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1)
}
func (a alphabet) key(buf []byte, idx int) int {
func (a alphabet) key(buf []byte, idx int64) int {
// This function generates keys of length 1..maxKeyLength, pulling
// characters from the alphabet. The idx determines which key to generate,
// generating the i-th lexicographically next key.
@@ -308,14 +324,14 @@ func (a alphabet) key(buf []byte, idx int) int {
keyCount(len(a.alphabet), a.maxLength))
}
func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int {
if keyCount == 0 || i > keyCount || i < 0 {
return 0
}
// Of the keyCount keys in the generative keyspace, how many are there
// starting with a particular character?
keysPerCharacter := keyCount / len(alphabet)
keysPerCharacter := keyCount / int64(len(alphabet))
// Find the character that the key at index i starts with and set it.
characterIdx := i / keysPerCharacter
@@ -347,3 +363,156 @@ func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
i = i - keysPerCharacter*characterIdx - 1
return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
}
// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey,
// returning the index of a particular key, given the provided alphabet and max
// length of a key.
//
// len(key) must be ≥ 1.
func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
i, ok := alphabet[key[0]]
if !ok {
panic(fmt.Sprintf("unrecognized alphabet character %v", key[0]))
}
// How many keys exist that start with the preceding i characters? Each of
// the i characters themselves are a key, plus the count of all the keys
// with one less character for each.
ret := i + i*keyCount(len(alphabet), n-1)
if len(key) > 1 {
ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1)
}
return ret
}
func abs(a int64) int64 {
if a < 0 {
return -a
}
return a
}
// RandomSeparator returns a random alphabetic key k such that a < k < b,
// pulling randomness from the provided random number generator. If dst is
// provided and the generated key fits within dst's capacity, the returned slice
// will use dst's memory.
//
// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
// will consist of the prefix P appended with the provided suffix. A zero suffix
// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b,
// but the generated key will not use the provided suffix. Note that it's
// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case
// RandomSeparator returns nil.
//
// If RandomSeparator generates a new prefix, the generated prefix will have
// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))).
//
// RandomSeparator panics if a or b fails to decode.
func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
if Comparer.Compare(a, b) >= 0 {
return nil
}
// Determine both keys' logical prefixes and suffixes.
ai := Comparer.Split(a)
bi := Comparer.Split(b)
ap := a[:ai]
bp := b[:bi]
maxLength = max[int](maxLength, max[int](len(ap), len(bp)))
var as, bs int64
var err error
if ai != len(a) {
as, err = ParseSuffix(a[ai:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", a))
}
}
if bi != len(b) {
bs, err = ParseSuffix(b[bi:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", b))
}
}
apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength)
bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength)
diff := bpIdx - apIdx
generatedIdx := bpIdx
if diff > 0 {
var add int64 = diff + 1
var start int64 = apIdx
if as == 1 {
// There's no expressible key with prefix a greater than a@1. So,
// exclude ap.
start = apIdx + 1
add = diff
}
if bs == 0 {
// No key with prefix b can sort before b@0. We don't want to pick b.
add--
}
// We're allowing generated id to be in the range [start, start + add - 1].
if start > start+add-1 {
return nil
}
// If we can generate a key which is actually in the middle of apIdx
// and bpIdx use it so that we don't have to bother about timestamps.
generatedIdx = rng.Int63n(add) + start
for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
generatedIdx = rng.Int63n(add) + start
}
}
switch {
case generatedIdx == apIdx && generatedIdx == bpIdx:
if abs(bs-as) <= 1 {
// There's no expressible suffix between the two, and there's no
// possible separator key.
return nil
}
// The key b is >= key a, but has the same prefix, so b must have the
// smaller timestamp, unless a has timestamp of 0.
//
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if as == 0 {
// bs > as
suffix = bs + rng.Int63n(10) + 1
} else {
// bs < as.
// Generate suffix in range [bs + 1, as - 1]
suffix = bs + 1 + rng.Int63n(as-bs-1)
}
case generatedIdx == apIdx:
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if as == 0 && suffix == 0 {
suffix++
} else if as != 0 && suffix >= as {
suffix = rng.Int63n(as)
}
case generatedIdx == bpIdx:
if suffix <= bs {
suffix = bs + rng.Int63n(10) + 1
}
}
if sz := maxLength + SuffixLen(suffix); cap(dst) < sz {
dst = make([]byte, sz)
} else {
dst = dst[:cap(dst)]
}
var w int
if suffix == 0 {
w = WriteKey(dst, Alpha(maxLength), generatedIdx)
} else {
w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
}
return dst[:w]
}
func max[I constraints.Ordered](a, b I) I {
if b > a {
return b
}
return a
}