aleesa-bot/aleesa-phrases-go
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgrade versions of vendored libs

Browse Source
This commit is contained in:
eleksir
2023-05-18 23:34:43 +03:00
parent 461ec78a7b
commit d90de40892
551 changed files with 36142 additions and 15606 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
vendor/github.com/cockroachdb/errors/.travis.yml generated vendored
View File

@@ -1,7 +0,0 @@
language: go
go:
- 1.13.x
- 1.14.x
- 1.15.x
- 1.16.x

40
vendor/github.com/cockroachdb/errors/markers/markers.go generated vendored
View File

@@ -46,10 +46,12 @@ func Is(err, reference error) bool {
return err == nil
}
isComparable := reflect.TypeOf(reference).Comparable()
// Direct reference comparison is the fastest, and most
// likely to be true, so do this first.
for c := err; c != nil; c = errbase.UnwrapOnce(c) {
if c == reference {
if isComparable && c == reference {
return true
}
// Compatibility with std go errors: if the error object itself
@@ -141,10 +143,27 @@ func If(err error, pred func(err error) (interface{}, bool)) (interface{}, bool)
// package location or a different type, ensure that
// RegisterTypeMigration() was called prior to IsAny().
func IsAny(err error, references ...error) bool {
// First try using direct reference comparison.
for c := err; ; c = errbase.UnwrapOnce(c) {
if err == nil {
for _, refErr := range references {
if c == refErr {
if refErr == nil {
return true
}
}
// The mark-based comparison below will never match anything if
// the error is nil, so don't bother with computing the marks in
// that case. This avoids the computational expense of computing
// the reference marks upfront.
return false
}
// First try using direct reference comparison.
for c := err; c != nil; c = errbase.UnwrapOnce(c) {
for _, refErr := range references {
if refErr == nil {
continue
}
isComparable := reflect.TypeOf(refErr).Comparable()
if isComparable && c == refErr {
return true
}
// Compatibility with std go errors: if the error object itself
@@ -153,19 +172,6 @@ func IsAny(err error, references ...error) bool {
return true
}
}
if c == nil {
// This special case is to support a comparison to a nil
// reference.
break
}
}
if err == nil {
// The mark-based comparison below will never match anything if
// the error is nil, so don't bother with computing the marks in
// that case. This avoids the computational expense of computing
// the reference marks upfront.
return false
}
// Try harder with marks.

57
vendor/github.com/cockroachdb/logtags/buffer.go generated vendored
View File

@@ -38,6 +38,20 @@ func (b *Buffer) Get() []Tag {
return b.tags
}
// GetTag returns the tag corresponding to the given key. If the tag doesn't
// exist, the bool return value will be false.
func (b *Buffer) GetTag(key string) (Tag, bool) {
if b == nil {
return Tag{}, false
}
for _, t := range b.tags {
if t.Key() == key {
return t, true
}
}
return Tag{}, false
}
// Add returns a new buffer with one more tag. If the tag has the same key as an
// earlier tag, that tag is overwritten.
// The receiver can be nil.
@@ -52,6 +66,25 @@ func (b *Buffer) Add(key string, value interface{}) *Buffer {
return res
}
// Remove returns a new Buffer with the tag with key `key` removed. If the tag
// does not exist, the receiver is returned (unchanged). The bool return value
// is true if the tag existed.
func (b *Buffer) Remove(key string) (*Buffer, bool) {
if b == nil {
return nil, false
}
for i, t := range b.tags {
if t.Key() == key {
res := &Buffer{}
res.init(len(b.tags)-1, 0 /* capacityHint */)
copy(res.tags, b.tags[:i])
copy(res.tags[i:], b.tags[i+1:])
return res, true
}
}
return b, false
}
// Merge returns a new buffer which contains tags from the receiver, followed by
// the tags from <other>.
//
@@ -121,12 +154,12 @@ func (b *Buffer) String() string {
}
// FormatToString emits the k/v pairs to a strings.Builder.
// - the k/v pairs are separated by commas (& no spaces).
// - if there is no value, only the key is printed.
// - if there is a value, and the key is just 1 character long,
// the key and the value are concatenated.
// This supports e.g. printing k="n", v=123 as "n123".
// - otherwise, it prints "k=v".
// - the k/v pairs are separated by commas (& no spaces).
// - if there is no value, only the key is printed.
// - if there is a value, and the key is just 1 character long,
// the key and the value are concatenated.
// This supports e.g. printing k="n", v=123 as "n123".
// - otherwise, it prints "k=v".
func (b *Buffer) FormatToString(buf *strings.Builder) {
comma := ""
for _, t := range b.Get() {
@@ -142,13 +175,19 @@ func (b *Buffer) FormatToString(buf *strings.Builder) {
}
}
func (b *Buffer) init(length, maxLenHint int) {
// init initializes b.tags to a slice of len `length` (filled with zero values).
// The slice may have capacity capacityHint. If capacityHint is less than
// length, it is ignored.
func (b *Buffer) init(length, capacityHint int) {
if length <= staticSlots {
// Even if maxLenHint is larger that staticSlots, we still want to try to
// Even if capacityHint is larger that staticSlots, we still want to try to
// avoid the allocation (especially since tags frequently get deduplicated).
b.tags = b.prealloc[:length]
} else {
b.tags = make([]Tag, length, maxLenHint)
if capacityHint < length {
capacityHint = length
}
b.tags = make([]Tag, length, capacityHint)
}
}

12
vendor/github.com/cockroachdb/logtags/context.go generated vendored
View File

@@ -41,6 +41,18 @@ func AddTag(ctx context.Context, key string, value interface{}) context.Context
return WithTags(ctx, b.Add(key, value))
}
// RemoveTag returns a context that has the tags in the given context except the
// tag with key `key`. If such a tag does not exist, the given context is
// returned.
func RemoveTag(ctx context.Context, key string) context.Context {
b := FromContext(ctx)
newB, ok := b.Remove(key)
if !ok {
return ctx
}
return WithTags(ctx, newB)
}
// AddTags returns a context that has the tags in the given context plus another
// set of tags. Tags are deduplicated (see Buffer.AddTags).
func AddTags(ctx context.Context, tags *Buffer) context.Context {

23
vendor/github.com/cockroachdb/pebble/Makefile generated vendored
View File

@@ -27,6 +27,21 @@ test:
testrace: testflags += -race -timeout 20m
testrace: test
testasan: testflags += -asan -timeout 20m
testasan: test
testmsan: export CC=clang
testmsan: testflags += -msan -timeout 20m
testmsan: test
.PHONY: testobjiotracing
testobjiotracing:
${GO} test -tags '$(TAGS) pebble_obj_io_tracing' ${testflags} -run ${TESTS} ./objstorage/objstorageprovider/objiotracing
.PHONY: lint
lint:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ./internal/lint
.PHONY: stress stressrace
stressrace: testflags += -race
stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 -test.v
@@ -48,7 +63,7 @@ crossversion-meta:
.PHONY: stress-crossversion
stress-crossversion:
STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 master
STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 crl-release-23.1 master
.PHONY: generate
generate:
@@ -81,11 +96,11 @@ endif
@${GO} mod tidy
$(MAKE) git-clean-check
# TODO(radu): switch back to @latest once bogus doc changes are
# addressed; see https://github.com/cockroachdb/crlfmt/pull/44
.PHONY: format
format:
for _file in $$(gofmt -s -l .); do \
gofmt -s -w $$_file ; \
done
go install github.com/cockroachdb/crlfmt@44a36ec7 && crlfmt -w -tab 2 .
.PHONY: format-check
format-check:

199
vendor/github.com/cockroachdb/pebble/batch.go generated vendored
View File

@@ -5,6 +5,7 @@
package pebble
import (
"context"
"encoding/binary"
"fmt"
"io"
@@ -12,6 +13,7 @@ import (
"sort"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/errors"
@@ -82,7 +84,7 @@ func (d DeferredBatchOp) Finish() error {
// consumers should use a batch per goroutine or provide their own
// synchronization.
//
// Indexing
// # Indexing
//
// Batches can be optionally indexed (see DB.NewIndexedBatch). An indexed batch
// allows iteration via an Iterator (see Batch.NewIter). The iterator provides
@@ -104,7 +106,7 @@ func (d DeferredBatchOp) Finish() error {
// significantly slower than inserting into a non-indexed batch. Only use an
// indexed batch if you require reading from it.
//
// Atomic commit
// # Atomic commit
//
// The operations in a batch are persisted by calling Batch.Commit which is
// equivalent to calling DB.Apply(batch). A batch is committed atomically by
@@ -115,7 +117,7 @@ func (d DeferredBatchOp) Finish() error {
// Batch.Commit will guarantee that the batch is persisted to disk before
// returning. See commitPipeline for more on the implementation details.
//
// Large batches
// # Large batches
//
// The size of a batch is limited only by available memory (be aware that
// indexed batches require considerably additional memory for the skiplist
@@ -140,34 +142,35 @@ func (d DeferredBatchOp) Finish() error {
// memtable have the same big-O time, but the constant factor dominates
// here. Sorting is significantly faster and uses significantly less memory.
//
// Internal representation
// # Internal representation
//
// The internal batch representation is a contiguous byte buffer with a fixed
// 12-byte header, followed by a series of records.
//
// +-------------+------------+--- ... ---+
// | SeqNum (8B) | Count (4B) | Entries |
// +-------------+------------+--- ... ---+
// +-------------+------------+--- ... ---+
// | SeqNum (8B) | Count (4B) | Entries |
// +-------------+------------+--- ... ---+
//
// Each record has a 1-byte kind tag prefix, followed by 1 or 2 length prefixed
// strings (varstring):
//
// +-----------+-----------------+-------------------+
// | Kind (1B) | Key (varstring) | Value (varstring) |
// +-----------+-----------------+-------------------+
// +-----------+-----------------+-------------------+
// | Kind (1B) | Key (varstring) | Value (varstring) |
// +-----------+-----------------+-------------------+
//
// A varstring is a varint32 followed by N bytes of data. The Kind tags are
// exactly those specified by InternalKeyKind. The following table shows the
// format for records of each kind:
//
// InternalKeyKindDelete varstring
// InternalKeyKindLogData varstring
// InternalKeyKindSet varstring varstring
// InternalKeyKindMerge varstring varstring
// InternalKeyKindRangeDelete varstring varstring
// InternalKeyKindRangeKeySet varstring varstring
// InternalKeyKindRangeKeyUnset varstring varstring
// InternalKeyKindRangeKeyDelete varstring varstring
// InternalKeyKindDelete varstring
// InternalKeyKindLogData varstring
// InternalKeyKindIngestSST varstring
// InternalKeyKindSet varstring varstring
// InternalKeyKindMerge varstring varstring
// InternalKeyKindRangeDelete varstring varstring
// InternalKeyKindRangeKeySet varstring varstring
// InternalKeyKindRangeKeyUnset varstring varstring
// InternalKeyKindRangeKeyDelete varstring varstring
//
// The intuitive understanding here are that the arguments to Delete, Set,
// Merge, DeleteRange and RangeKeyDelete are encoded into the batch. The
@@ -210,6 +213,8 @@ type Batch struct {
// The db to which the batch will be committed. Do not change this field
// after the batch has been created as it might invalidate internal state.
// Batch.memTableSize is only refreshed if Batch.db is set. Setting db to
// nil once it has been set implies that the Batch has encountered an error.
db *DB
// The count of records in the batch. This count will be stored in the batch
@@ -255,9 +260,74 @@ type Batch struct {
// memtable.
flushable *flushableBatch
// ingestedSSTBatch indicates that the batch contains one or more key kinds
// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
// then it will only contain key kinds of IngestSST.
ingestedSSTBatch bool
// Synchronous Apply uses the commit WaitGroup for both publishing the
// seqnum and waiting for the WAL fsync (if needed). Asynchronous
// ApplyNoSyncWait, which implies WriteOptions.Sync is true, uses the commit
// WaitGroup for publishing the seqnum and the fsyncWait WaitGroup for
// waiting for the WAL fsync.
//
// TODO(sumeer): if we find that ApplyNoSyncWait in conjunction with
// SyncWait is causing higher memory usage because of the time duration
// between when the sync is already done, and a goroutine calls SyncWait
// (followed by Batch.Close), we could separate out {fsyncWait, commitErr}
// into a separate struct that is allocated separately (using another
// sync.Pool), and only that struct needs to outlive Batch.Close (which
// could then be called immediately after ApplyNoSyncWait). commitStats
// will also need to be in this separate struct.
commit sync.WaitGroup
fsyncWait sync.WaitGroup
commitStats BatchCommitStats
commitErr error
applied uint32 // updated atomically
applied atomic.Bool
}
// BatchCommitStats exposes stats related to committing a batch.
//
// NB: there is no Pebble internal tracing (using LoggerAndTracer) of slow
// batch commits. The caller can use these stats to do their own tracing as
// needed.
type BatchCommitStats struct {
// TotalDuration is the time spent in DB.{Apply,ApplyNoSyncWait} or
// Batch.Commit, plus the time waiting in Batch.SyncWait. If there is a gap
// between calling ApplyNoSyncWait and calling SyncWait, that gap could
// include some duration in which real work was being done for the commit
// and will not be included here. This missing time is considered acceptable
// since the goal of these stats is to understand user-facing latency.
//
// TotalDuration includes time spent in various queues both inside Pebble
// and outside Pebble (I/O queues, goroutine scheduler queue, mutex wait
// etc.). For some of these queues (which we consider important) the wait
// times are included below -- these expose low-level implementation detail
// and are meant for expert diagnosis and subject to change. There may be
// unaccounted time after subtracting those values from TotalDuration.
TotalDuration time.Duration
// SemaphoreWaitDuration is the wait time for semaphores in
// commitPipeline.Commit.
SemaphoreWaitDuration time.Duration
// WALQueueWaitDuration is the wait time for allocating memory blocks in the
// LogWriter (due to the LogWriter not writing fast enough).
WALQueueWaitDuration time.Duration
// MemTableWriteStallDuration is the wait caused by a write stall due to too
// many memtables (due to not flushing fast enough).
MemTableWriteStallDuration time.Duration
// L0ReadAmpWriteStallDuration is the wait caused by a write stall due to
// high read amplification in L0 (due to not compacting fast enough out of
// L0).
L0ReadAmpWriteStallDuration time.Duration
// WALRotationDuration is the wait time for WAL rotation, which includes
// syncing and closing the old WAL and creating (or reusing) a new one.
WALRotationDuration time.Duration
// CommitWaitDuration is the wait for publishing the seqnum plus the
// duration for the WAL sync (if requested). The former should be tiny and
// one can assume that this is all due to the WAL sync.
CommitWaitDuration time.Duration
}
var _ Reader = (*Batch)(nil)
@@ -348,13 +418,16 @@ func (b *Batch) refreshMemTableSize() {
if !ok {
break
}
b.memTableSize += memTableEntrySize(len(key), len(value))
switch kind {
case InternalKeyKindRangeDelete:
b.countRangeDels++
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
b.countRangeKeys++
case InternalKeyKindIngestSST:
// This key kind doesn't contribute to the memtable size.
continue
}
b.memTableSize += memTableEntrySize(len(key), len(value))
}
}
@@ -362,6 +435,9 @@ func (b *Batch) refreshMemTableSize() {
//
// It is safe to modify the contents of the arguments after Apply returns.
func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
if b.ingestedSSTBatch {
panic("pebble: invalid batch application")
}
if len(batch.data) == 0 {
return nil
}
@@ -392,6 +468,8 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
b.countRangeDels++
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
b.countRangeKeys++
case InternalKeyKindIngestSST:
panic("pebble: invalid key kind for batch")
}
if b.index != nil {
var err error
@@ -797,6 +875,28 @@ func (b *Batch) LogData(data []byte, _ *WriteOptions) error {
return nil
}
// IngestSST adds the FileNum for an sstable to the batch. The data will only be
// written to the WAL (not added to memtables or sstables).
func (b *Batch) ingestSST(fileNum base.FileNum) {
if b.Empty() {
b.ingestedSSTBatch = true
} else if !b.ingestedSSTBatch {
// Batch contains other key kinds.
panic("pebble: invalid call to ingestSST")
}
origMemTableSize := b.memTableSize
var buf [binary.MaxVarintLen64]byte
length := binary.PutUvarint(buf[:], uint64(fileNum))
b.prepareDeferredKeyRecord(length, InternalKeyKindIngestSST)
copy(b.deferredOp.Key, buf[:length])
// Since IngestSST writes only to the WAL and does not affect the memtable,
// we restore b.memTableSize to its original value. Note that Batch.count
// is not reset because for the InternalKeyKindIngestSST the count is the
// number of sstable paths which have been added to the batch.
b.memTableSize = origMemTableSize
}
// Empty returns true if the batch is empty, and false otherwise.
func (b *Batch) Empty() bool {
return len(b.data) <= batchHeaderLen
@@ -845,10 +945,16 @@ func (b *Batch) SetRepr(data []byte) error {
// later mutations. Its view can be refreshed via RefreshBatchSnapshot or
// SetOptions().
func (b *Batch) NewIter(o *IterOptions) *Iterator {
return b.NewIterWithContext(context.Background(), o)
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
// tracing.
func (b *Batch) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator {
if b.index == nil {
return &Iterator{err: ErrNotIndexed}
}
return b.db.newIterInternal(b, nil /* snapshot */, o)
return b.db.newIter(ctx, b, nil /* snapshot */, o)
}
// newInternalIter creates a new internalIterator that iterates over the
@@ -1089,8 +1195,10 @@ func (b *Batch) Reset() {
b.rangeKeysSeqNum = 0
b.flushable = nil
b.commit = sync.WaitGroup{}
b.fsyncWait = sync.WaitGroup{}
b.commitStats = BatchCommitStats{}
b.commitErr = nil
atomic.StoreUint32(&b.applied, 0)
b.applied.Store(false)
if b.data != nil {
if cap(b.data) > batchMaxRetainedSize {
// If the capacity of the buffer is larger than our maximum
@@ -1159,7 +1267,9 @@ func (b *Batch) setCount(v uint32) {
}
// Count returns the count of memtable-modifying operations in this batch. All
// operations with the except of LogData increment this count.
// operations with the except of LogData increment this count. For IngestSSTs,
// count is only used to indicate the number of SSTs ingested in the record, the
// batch isn't applied to the memtable.
func (b *Batch) Count() uint32 {
if b.count > math.MaxUint32 {
panic(ErrInvalidBatch)
@@ -1205,6 +1315,26 @@ func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) {
return data[v:], data[:v], true
}
// SyncWait is to be used in conjunction with DB.ApplyNoSyncWait.
func (b *Batch) SyncWait() error {
now := time.Now()
b.fsyncWait.Wait()
if b.commitErr != nil {
b.db = nil // prevent batch reuse on error
}
waitDuration := time.Since(now)
b.commitStats.CommitWaitDuration += waitDuration
b.commitStats.TotalDuration += waitDuration
return b.commitErr
}
// CommitStats returns stats related to committing the batch. Should be called
// after Batch.Commit, DB.Apply. If DB.ApplyNoSyncWait is used, should be
// called after Batch.SyncWait.
func (b *Batch) CommitStats() BatchCommitStats {
return b.commitStats
}
// BatchReader iterates over the entries contained in a batch.
type BatchReader []byte
@@ -1336,6 +1466,19 @@ func (i *batchIter) Next() (*InternalKey, base.LazyValue) {
return ikey, base.MakeInPlaceValue(i.value())
}
func (i *batchIter) NextPrefix(succKey []byte) (*InternalKey, LazyValue) {
// Because NextPrefix was invoked `succKey` must be ≥ the key at i's current
// position. Seek the arena iterator using TrySeekUsingNext.
ikey := i.iter.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext())
for ikey != nil && ikey.SeqNum() >= i.snapshot {
ikey = i.iter.Next()
}
if ikey == nil {
return nil, base.LazyValue{}
}
return ikey, base.MakeInPlaceValue(i.value())
}
func (i *batchIter) Prev() (*InternalKey, base.LazyValue) {
ikey := i.iter.Prev()
for ikey != nil && ikey.SeqNum() >= i.snapshot {
@@ -1773,6 +1916,12 @@ func (i *flushableBatchIter) Prev() (*InternalKey, base.LazyValue) {
return &i.key, i.value()
}
// Note: flushFlushableBatchIter.NextPrefix mirrors the implementation of
// flushableBatchIter.NextPrefix due to performance. Keep the two in sync.
func (i *flushableBatchIter) NextPrefix(succKey []byte) (*InternalKey, LazyValue) {
return i.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext())
}
func (i *flushableBatchIter) getKey(index int) InternalKey {
e := &i.offsets[index]
kind := InternalKeyKind(i.data[e.offset])
@@ -1866,6 +2015,10 @@ func (i *flushFlushableBatchIter) First() (*InternalKey, base.LazyValue) {
return key, val
}
func (i *flushFlushableBatchIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
panic("pebble: Prev unimplemented")
}
// Note: flushFlushableBatchIter.Next mirrors the implementation of
// flushableBatchIter.Next due to performance. Keep the two in sync.
func (i *flushFlushableBatchIter) Next() (*InternalKey, base.LazyValue) {

6
vendor/github.com/cockroachdb/pebble/cache.go generated vendored
View File

@@ -15,9 +15,9 @@ type Cache = cache.Cache
// creator of the cache should usually release their reference after the DB is
// created.
//
// c := pebble.NewCache(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
// c := pebble.NewCache(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func NewCache(size int64) *cache.Cache {
return cache.New(size)
}

235
vendor/github.com/cockroachdb/pebble/checkpoint.go generated vendored
View File

@@ -5,10 +5,12 @@
package pebble
import (
"io"
"os"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
@@ -19,6 +21,9 @@ type checkpointOptions struct {
// flushWAL set to true will force a flush and sync of the WAL prior to
// checkpointing.
flushWAL bool
// If set, any SSTs that don't overlap with these spans are excluded from a checkpoint.
restrictToSpans []CheckpointSpan
}
// CheckpointOption set optional parameters used by `DB.Checkpoint`.
@@ -39,6 +44,43 @@ func WithFlushedWAL() CheckpointOption {
}
}
// WithRestrictToSpans specifies spans of interest for the checkpoint. Any SSTs
// that don't overlap with any of these spans are excluded from the checkpoint.
//
// Note that the checkpoint can still surface keys outside of these spans (from
// the WAL and from SSTs that partially overlap with these spans). Moreover,
// these surface keys aren't necessarily "valid" in that they could have been
// modified but the SST containing the modification is excluded.
func WithRestrictToSpans(spans []CheckpointSpan) CheckpointOption {
return func(opt *checkpointOptions) {
opt.restrictToSpans = spans
}
}
// CheckpointSpan is a key range [Start, End) (inclusive on Start, exclusive on
// End) of interest for a checkpoint.
type CheckpointSpan struct {
Start []byte
End []byte
}
// excludeFromCheckpoint returns true if an SST file should be excluded from the
// checkpoint because it does not overlap with the spans of interest
// (opt.restrictToSpans).
func excludeFromCheckpoint(f *fileMetadata, opt *checkpointOptions, cmp Compare) bool {
if len(opt.restrictToSpans) == 0 {
// Option not set; don't exclude anything.
return false
}
for _, s := range opt.restrictToSpans {
if f.Overlaps(cmp, s.Start, s.End, true /* exclusiveEnd */) {
return false
}
}
// None of the restrictToSpans overlapped; we can exclude this file.
return true
}
// mkdirAllAndSyncParents creates destDir and any of its missing parents.
// Those missing parents, as well as the closest existing ancestor, are synced.
// Returns a handle to the directory created at destDir.
@@ -93,6 +135,9 @@ func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
// space overhead for a checkpoint if hard links are disabled. Also beware that
// even if hard links are used, the space overhead for the checkpoint will
// increase over time as the DB performs compactions.
//
// TODO(bananabrick): Test checkpointing of virtual sstables once virtual
// sstables is running e2e.
func (d *DB) Checkpoint(
destDir string, opts ...CheckpointOption,
) (
@@ -146,7 +191,10 @@ func (d *DB) Checkpoint(
manifestFileNum := d.mu.versions.manifestFileNum
manifestSize := d.mu.versions.manifest.Size()
optionsFileNum := d.optionsFileNum
virtualBackingFiles := make(map[base.DiskFileNum]struct{})
for diskFileNum := range d.mu.versions.fileBackingMap {
virtualBackingFiles[diskFileNum] = struct{}{}
}
// Release the manifest and DB.mu so we don't block other operations on
// the database.
d.mu.versions.logUnlock()
@@ -154,13 +202,10 @@ func (d *DB) Checkpoint(
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := syncingFS{
FS: d.opts.FS,
syncOpts: vfs.SyncingFileOptions{
NoSyncOnClose: d.opts.NoSyncOnClose,
BytesPerSync: d.opts.BytesPerSync,
},
}
fs := vfs.NewSyncingFS(d.opts.FS, vfs.SyncingFileOptions{
NoSyncOnClose: d.opts.NoSyncOnClose,
BytesPerSync: d.opts.BytesPerSync,
})
// Create the dir and its parents (if necessary), and sync them.
var dir vfs.File
@@ -214,45 +259,34 @@ func (d *DB) Checkpoint(
}
}
{
// Copy the MANIFEST, and create a pointer to it. We copy rather
// than link because additional version edits added to the
// MANIFEST after we took our snapshot of the sstables will
// reference sstables that aren't in our checkpoint. For a
// similar reason, we need to limit how much of the MANIFEST we
// copy.
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LimitedCopy(fs, srcPath, destPath, manifestSize)
if ckErr != nil {
return ckErr
}
// Recent format versions use an atomic marker for setting the
// active manifest. Older versions use the CURRENT file. The
// setCurrentFunc function will return a closure that will
// take the appropriate action for the database's format
// version.
var manifestMarker *atomicfs.Marker
manifestMarker, _, ckErr = atomicfs.LocateMarker(fs, destDir, manifestMarkerName)
if ckErr != nil {
return ckErr
}
ckErr = setCurrentFunc(formatVers, manifestMarker, fs, destDir, dir)(manifestFileNum)
if ckErr != nil {
return ckErr
}
ckErr = manifestMarker.Close()
if ckErr != nil {
return ckErr
}
}
var excludedFiles map[deletedFileEntry]*fileMetadata
// Set of FileBacking.DiskFileNum which will be required by virtual sstables
// in the checkpoint.
requiredVirtualBackingFiles := make(map[base.DiskFileNum]struct{})
// Link or copy the sstables.
for l := range current.Levels {
iter := current.Levels[l].Iter()
for f := iter.First(); f != nil; f = iter.Next() {
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, f.FileNum)
if excludeFromCheckpoint(f, opt, d.cmp) {
if excludedFiles == nil {
excludedFiles = make(map[deletedFileEntry]*fileMetadata)
}
excludedFiles[deletedFileEntry{
Level: l,
FileNum: f.FileNum,
}] = f
continue
}
fileBacking := f.FileBacking
if f.Virtual {
if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok {
continue
}
requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{}
}
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
if ckErr != nil {
@@ -261,6 +295,23 @@ func (d *DB) Checkpoint(
}
}
var removeBackingTables []base.DiskFileNum
for diskFileNum := range virtualBackingFiles {
if _, ok := requiredVirtualBackingFiles[diskFileNum]; !ok {
// The backing sstable associated with fileNum is no longer
// required.
removeBackingTables = append(removeBackingTables, diskFileNum)
}
}
ckErr = d.writeCheckpointManifest(
fs, formatVers, destDir, dir, manifestFileNum.DiskFileNum(), manifestSize,
excludedFiles, removeBackingTables,
)
if ckErr != nil {
return ckErr
}
// Copy the WAL files. We copy rather than link because WAL file recycling
// will cause the WAL files to be reused which would invalidate the
// checkpoint.
@@ -269,7 +320,7 @@ func (d *DB) Checkpoint(
if logNum == 0 {
continue
}
srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum)
srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum.DiskFileNum())
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.Copy(fs, srcPath, destPath)
if ckErr != nil {
@@ -286,3 +337,99 @@ func (d *DB) Checkpoint(
dir = nil
return ckErr
}
func (d *DB) writeCheckpointManifest(
fs vfs.FS,
formatVers FormatMajorVersion,
destDirPath string,
destDir vfs.File,
manifestFileNum base.DiskFileNum,
manifestSize int64,
excludedFiles map[deletedFileEntry]*fileMetadata,
removeBackingTables []base.DiskFileNum,
) error {
// Copy the MANIFEST, and create a pointer to it. We copy rather
// than link because additional version edits added to the
// MANIFEST after we took our snapshot of the sstables will
// reference sstables that aren't in our checkpoint. For a
// similar reason, we need to limit how much of the MANIFEST we
// copy.
// If some files are excluded from the checkpoint, also append a block that
// records those files as deleted.
if err := func() error {
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum)
destPath := fs.PathJoin(destDirPath, fs.PathBase(srcPath))
src, err := fs.Open(srcPath, vfs.SequentialReadsOption)
if err != nil {
return err
}
defer src.Close()
dst, err := fs.Create(destPath)
if err != nil {
return err
}
defer dst.Close()
// Copy all existing records. We need to copy at the record level in case we
// need to append another record with the excluded files (we cannot simply
// append a record after a raw data copy; see
// https://github.com/cockroachdb/cockroach/issues/100935).
r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum.FileNum())
w := record.NewWriter(dst)
for {
rr, err := r.Next()
if err != nil {
if err == io.EOF {
break
}
return err
}
rw, err := w.Next()
if err != nil {
return err
}
if _, err := io.Copy(rw, rr); err != nil {
return err
}
}
if len(excludedFiles) > 0 {
// Write out an additional VersionEdit that deletes the excluded SST files.
ve := versionEdit{
DeletedFiles: excludedFiles,
RemovedBackingTables: removeBackingTables,
}
rw, err := w.Next()
if err != nil {
return err
}
if err := ve.Encode(rw); err != nil {
return err
}
}
if err := w.Close(); err != nil {
return err
}
return dst.Sync()
}(); err != nil {
return err
}
// Recent format versions use an atomic marker for setting the
// active manifest. Older versions use the CURRENT file. The
// setCurrentFunc function will return a closure that will
// take the appropriate action for the database's format
// version.
var manifestMarker *atomicfs.Marker
manifestMarker, _, err := atomicfs.LocateMarker(fs, destDirPath, manifestMarkerName)
if err != nil {
return err
}
if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum.FileNum()); err != nil {
return err
}
return manifestMarker.Close()
}

209
vendor/github.com/cockroachdb/pebble/commit.go generated vendored
View File

@@ -8,6 +8,7 @@ import (
"runtime"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/pebble/record"
@@ -32,7 +33,7 @@ type commitQueue struct {
//
// The head index is stored in the most-significant bits so that we can
// atomically add to it and the overflow is harmless.
headTail uint64
headTail atomic.Uint64
// slots is a ring buffer of values stored in this queue. The size must be a
// power of 2. A slot is in use until *both* the tail index has moved beyond
@@ -57,10 +58,10 @@ func (q *commitQueue) pack(head, tail uint32) uint64 {
}
func (q *commitQueue) enqueue(b *Batch) {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
// Queue is full. This should never be reached because commitPipeline.sem
// Queue is full. This should never be reached because commitPipeline.commitQueueSem
// limits the number of concurrent operations.
panic("pebble: not reached")
}
@@ -79,12 +80,12 @@ func (q *commitQueue) enqueue(b *Batch) {
// Increment head. This passes ownership of slot to dequeue and acts as a
// store barrier for writing the slot.
atomic.AddUint64(&q.headTail, 1<<dequeueBits)
q.headTail.Add(1 << dequeueBits)
}
func (q *commitQueue) dequeue() *Batch {
for {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if tail == head {
// Queue is empty.
@@ -93,7 +94,7 @@ func (q *commitQueue) dequeue() *Batch {
slot := &q.slots[tail&uint32(len(q.slots)-1)]
b := (*Batch)(atomic.LoadPointer(slot))
if b == nil || atomic.LoadUint32(&b.applied) == 0 {
if b == nil || !b.applied.Load() {
// The batch is not ready to be dequeued, or another goroutine has
// already dequeued it.
return nil
@@ -102,7 +103,7 @@ func (q *commitQueue) dequeue() *Batch {
// Confirm head and tail (for our speculative check above) and increment
// tail. If this succeeds, then we own the slot at tail.
ptrs2 := q.pack(head, tail+1)
if atomic.CompareAndSwapUint64(&q.headTail, ptrs, ptrs2) {
if q.headTail.CompareAndSwap(ptrs, ptrs2) {
// We now own slot.
//
// Tell enqueue that we're done with this slot. Zeroing the slot is also
@@ -121,10 +122,10 @@ func (q *commitQueue) dequeue() *Batch {
type commitEnv struct {
// The next sequence number to give to a batch. Protected by
// commitPipeline.mu.
logSeqNum *uint64
logSeqNum *atomic.Uint64
// The visible sequence number at which reads should be performed. Ratcheted
// upwards atomically as batches are applied to the memtable.
visibleSeqNum *uint64
visibleSeqNum *atomic.Uint64
// Apply the batch to the specified memtable. Called concurrently.
apply func(b *Batch, mem *memTable) error
@@ -140,8 +141,8 @@ type commitEnv struct {
// (contained in a single Batch) atomically to the DB. The steps are
// conceptually:
//
// 1. Write the batch to the WAL and optionally sync the WAL
// 2. Apply the mutations in the batch to the memtable
// 1. Write the batch to the WAL and optionally sync the WAL
// 2. Apply the mutations in the batch to the memtable
//
// These two simple steps are made complicated by the desire for high
// performance. In the absence of concurrency, performance is limited by how
@@ -150,10 +151,10 @@ type commitEnv struct {
// pipeline. Performance under concurrency is the primary concern of the commit
// pipeline, though it also needs to maintain two invariants:
//
// 1. Batches need to be written to the WAL in sequence number order.
// 2. Batches need to be made visible for reads in sequence number order. This
// invariant arises from the use of a single sequence number which
// indicates which mutations are visible.
// 1. Batches need to be written to the WAL in sequence number order.
// 2. Batches need to be made visible for reads in sequence number order. This
// invariant arises from the use of a single sequence number which
// indicates which mutations are visible.
//
// Taking these invariants into account, let's revisit the work the commit
// pipeline needs to perform. Writing the batch to the WAL is necessarily
@@ -173,14 +174,14 @@ type commitEnv struct {
//
// The full outline of the commit pipeline operation is as follows:
//
// with commitPipeline mutex locked:
// assign batch sequence number
// write batch to WAL
// (optionally) add batch to WAL sync list
// apply batch to memtable (concurrently)
// wait for earlier batches to apply
// ratchet read sequence number
// (optionally) wait for the WAL to sync
// with commitPipeline mutex locked:
// assign batch sequence number
// write batch to WAL
// (optionally) add batch to WAL sync list
// apply batch to memtable (concurrently)
// wait for earlier batches to apply
// ratchet read sequence number
// (optionally) wait for the WAL to sync
//
// As soon as a batch has been written to the WAL, the commitPipeline mutex is
// released allowing another batch to write to the WAL. Each commit operation
@@ -217,7 +218,27 @@ type commitPipeline struct {
// Queue of pending batches to commit.
pending commitQueue
env commitEnv
sem chan struct{}
// The commit path has two queues:
// - commitPipeline.pending contains batches whose seqnums have not yet been
// published. It is a lock-free single producer multi consumer queue.
// - LogWriter.flusher.syncQ contains state for batches that have asked for
// a sync. It is a lock-free single producer single consumer queue.
// These lock-free queues have a fixed capacity. And since they are
// lock-free, we cannot do blocking waits when pushing onto these queues, in
// case they are full. Additionally, adding to these queues happens while
// holding commitPipeline.mu, and we don't want to block while holding that
// mutex since it is also needed by other code.
//
// Popping from these queues is independent and for a particular batch can
// occur in either order, though it is more common that popping from the
// commitPipeline.pending will happen first.
//
// Due to these constraints, we reserve a unit of space in each queue before
// acquiring commitPipeline.mu, which also ensures that the push operation
// is guaranteed to have space in the queue. The commitQueueSem and
// logSyncQSem are used for this reservation.
commitQueueSem chan struct{}
logSyncQSem chan struct{}
// The mutex to use for synchronizing access to logSeqNum and serializing
// calls to commitEnv.write().
mu sync.Mutex
@@ -226,23 +247,59 @@ type commitPipeline struct {
func newCommitPipeline(env commitEnv) *commitPipeline {
p := &commitPipeline{
env: env,
// The capacity of both commitQueue.slots and syncQueue.slots is set to
// record.SyncConcurrency, which also determines the value of these
// semaphores. We used to have a single semaphore, which required that the
// capacity of these queues be the same. Now that we have two semaphores,
// the capacity of these queues could be changed to be different. Say half
// of the batches asked to be synced, but syncing took 5x the latency of
// adding to the memtable and publishing. Then syncQueue.slots could be
// sized as 0.5*5 of the commitQueue.slots. We can explore this if we find
// that LogWriterMetrics.SyncQueueLen has high utilization under some
// workloads.
//
// NB: the commit concurrency is one less than SyncConcurrency because we
// have to allow one "slot" for a concurrent WAL rotation which will close
// and sync the WAL.
sem: make(chan struct{}, record.SyncConcurrency-1),
commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
logSyncQSem: make(chan struct{}, record.SyncConcurrency-1),
}
return p
}
// directWrite is used to directly write to the WAL. commitPipeline.mu must be
// held while this is called. DB.mu must not be held. directWrite will only
// return once the WAL sync is complete. Note that DirectWrite is a special case
// function which is currently only used when ingesting sstables as a flushable.
// Reason carefully about the correctness argument when calling this function
// from any context.
func (p *commitPipeline) directWrite(b *Batch) error {
var syncWG sync.WaitGroup
var syncErr error
syncWG.Add(1)
p.logSyncQSem <- struct{}{}
_, err := p.env.write(b, &syncWG, &syncErr)
syncWG.Wait()
err = firstError(err, syncErr)
return err
}
// Commit the specified batch, writing it to the WAL, optionally syncing the
// WAL, and applying the batch to the memtable. Upon successful return the
// batch's mutations will be visible for reading.
func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error {
// REQUIRES: noSyncWait => syncWAL
func (p *commitPipeline) Commit(b *Batch, syncWAL bool, noSyncWait bool) error {
if b.Empty() {
return nil
}
p.sem <- struct{}{}
commitStartTime := time.Now()
// Acquire semaphores.
p.commitQueueSem <- struct{}{}
if syncWAL {
p.logSyncQSem <- struct{}{}
}
b.commitStats.SemaphoreWaitDuration = time.Since(commitStartTime)
// Prepare the batch for committing: enqueuing the batch in the pending
// queue, determining the batch sequence number and writing the data to the
@@ -250,27 +307,43 @@ func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error {
//
// NB: We set Batch.commitErr on error so that the batch won't be a candidate
// for reuse. See Batch.release().
mem, err := p.prepare(b, syncWAL)
mem, err := p.prepare(b, syncWAL, noSyncWait)
if err != nil {
b.db = nil // prevent batch reuse on error
// NB: we are not doing <-p.commitQueueSem since the batch is still
// sitting in the pending queue. We should consider fixing this by also
// removing the batch from the pending queue.
return err
}
// Apply the batch to the memtable.
if err := p.env.apply(b, mem); err != nil {
b.db = nil // prevent batch reuse on error
// NB: we are not doing <-p.commitQueueSem since the batch is still
// sitting in the pending queue. We should consider fixing this by also
// removing the batch from the pending queue.
return err
}
// Publish the batch sequence number.
p.publish(b)
<-p.sem
<-p.commitQueueSem
if b.commitErr != nil {
b.db = nil // prevent batch reuse on error
if !noSyncWait {
// Already waited for commit, so look at the error.
if b.commitErr != nil {
b.db = nil // prevent batch reuse on error
err = b.commitErr
}
}
return b.commitErr
// Else noSyncWait. The LogWriter can be concurrently writing to
// b.commitErr. We will read b.commitErr in Batch.SyncWait after the
// LogWriter is done writing.
b.commitStats.TotalDuration = time.Since(commitStartTime)
return err
}
// AllocateSeqNum allocates count sequence numbers, invokes the prepare
@@ -280,7 +353,9 @@ func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error {
// sstable ingestion within the commit pipeline. The prepare callback is
// invoked with commitPipeline.mu held, but note that DB.mu is not held and
// must be locked if necessary.
func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(seqNum uint64)) {
func (p *commitPipeline) AllocateSeqNum(
count int, prepare func(seqNum uint64), apply func(seqNum uint64),
) {
// This method is similar to Commit and prepare. Be careful about trying to
// share additional code with those methods because Commit and prepare are
// performance critical code paths.
@@ -294,7 +369,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
b.setCount(uint32(count))
b.commit.Add(1)
p.sem <- struct{}{}
p.commitQueueSem <- struct{}{}
p.mu.Lock()
@@ -306,12 +381,12 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
logSeqNum := atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
logSeqNum := p.env.logSeqNum.Add(uint64(count)) - uint64(count)
seqNum := logSeqNum
if seqNum == 0 {
// We can't use the value 0 for the global seqnum during ingestion, because
// 0 indicates no global seqnum. So allocate one more seqnum.
atomic.AddUint64(p.env.logSeqNum, 1)
p.env.logSeqNum.Add(1)
seqNum++
}
b.setSeqNum(seqNum)
@@ -321,7 +396,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// writes that were sequenced before the ingestion. The spin loop is
// unfortunate, but obviates the need for additional synchronization.
for {
visibleSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
visibleSeqNum := p.env.visibleSeqNum.Load()
if visibleSeqNum == logSeqNum {
break
}
@@ -331,7 +406,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// Invoke the prepare callback. Note the lack of error reporting. Even if the
// callback internally fails, the sequence number needs to be published in
// order to allow the commit pipeline to proceed.
prepare()
prepare(b.SeqNum())
p.mu.Unlock()
@@ -341,26 +416,33 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// Publish the sequence number.
p.publish(b)
<-p.sem
<-p.commitQueueSem
}
func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) {
func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memTable, error) {
n := uint64(b.Count())
if n == invalidBatchCount {
return nil, ErrInvalidBatch
}
count := 1
if syncWAL {
count++
}
// count represents the waiting needed for publish, and optionally the
// waiting needed for the WAL sync.
b.commit.Add(count)
var syncWG *sync.WaitGroup
var syncErr *error
if syncWAL {
syncWG, syncErr = &b.commit, &b.commitErr
switch {
case !syncWAL:
// Only need to wait for the publish.
b.commit.Add(1)
// Remaining cases represent syncWAL=true.
case noSyncWait:
syncErr = &b.commitErr
syncWG = &b.fsyncWait
// Only need to wait synchronously for the publish. The user will
// (asynchronously) wait on the batch's fsyncWait.
b.commit.Add(1)
b.fsyncWait.Add(1)
case !noSyncWait:
syncErr = &b.commitErr
syncWG = &b.commit
// Must wait for both the publish and the WAL fsync.
b.commit.Add(2)
}
p.mu.Lock()
@@ -373,7 +455,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) {
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
b.setSeqNum(atomic.AddUint64(p.env.logSeqNum, n) - n)
b.setSeqNum(p.env.logSeqNum.Add(n) - n)
// Write the data to the WAL.
mem, err := p.env.write(b, syncWG, syncErr)
@@ -385,7 +467,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) {
func (p *commitPipeline) publish(b *Batch) {
// Mark the batch as applied.
atomic.StoreUint32(&b.applied, 1)
b.applied.Store(true)
// Loop dequeuing applied batches from the pending queue. If our batch was
// the head of the pending queue we are guaranteed that either we'll publish
@@ -399,10 +481,12 @@ func (p *commitPipeline) publish(b *Batch) {
if t == nil {
// Wait for another goroutine to publish us. We might also be waiting for
// the WAL sync to finish.
now := time.Now()
b.commit.Wait()
b.commitStats.CommitWaitDuration += time.Since(now)
break
}
if atomic.LoadUint32(&t.applied) != 1 {
if !t.applied.Load() {
panic("not reached")
}
@@ -411,13 +495,13 @@ func (p *commitPipeline) publish(b *Batch) {
// number for a subsequent batch. That's ok as all we're guaranteeing is
// that the sequence number ratchets up.
for {
curSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
curSeqNum := p.env.visibleSeqNum.Load()
newSeqNum := t.SeqNum() + uint64(t.Count())
if newSeqNum <= curSeqNum {
// t's sequence number has already been published.
break
}
if atomic.CompareAndSwapUint64(p.env.visibleSeqNum, curSeqNum, newSeqNum) {
if p.env.visibleSeqNum.CompareAndSwap(curSeqNum, newSeqNum) {
// We successfully published t's sequence number.
break
}
@@ -426,18 +510,3 @@ func (p *commitPipeline) publish(b *Batch) {
t.commit.Done()
}
}
// ratchetSeqNum allocates and marks visible all sequence numbers less than
// but excluding `nextSeqNum`.
func (p *commitPipeline) ratchetSeqNum(nextSeqNum uint64) {
p.mu.Lock()
defer p.mu.Unlock()
logSeqNum := atomic.LoadUint64(p.env.logSeqNum)
if logSeqNum >= nextSeqNum {
return
}
count := nextSeqNum - logSeqNum
_ = atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
atomic.StoreUint64(p.env.visibleSeqNum, nextSeqNum)
}

1001
vendor/github.com/cockroachdb/pebble/compaction.go generated vendored
View File

File diff suppressed because it is too large Load Diff

442
vendor/github.com/cockroachdb/pebble/compaction_iter.go generated vendored
View File

@@ -5,6 +5,7 @@
package pebble
import (
"bytes"
"fmt"
"io"
"sort"
@@ -77,22 +78,22 @@ import (
// is that snapshots define stripes and entries are collapsed within stripes,
// but not across stripes. Consider the following scenario:
//
// a.PUT.9
// a.DEL.8
// a.PUT.7
// a.DEL.6
// a.PUT.5
// a.PUT.9
// a.DEL.8
// a.PUT.7
// a.DEL.6
// a.PUT.5
//
// In the absence of snapshots these entries would be collapsed to
// a.PUT.9. What if there is a snapshot at sequence number 7? The entries can
// be divided into two stripes and collapsed within the stripes:
//
// a.PUT.9 a.PUT.9
// a.DEL.8 --->
// a.PUT.7
// -- --
// a.DEL.6 ---> a.DEL.6
// a.PUT.5
// a.PUT.9 a.PUT.9
// a.DEL.8 --->
// a.PUT.7
// -- --
// a.DEL.6 ---> a.DEL.6
// a.PUT.5
//
// All of the rules described earlier still apply, but they are confined to
// operate within a snapshot stripe. Snapshots only affect compaction when the
@@ -111,13 +112,13 @@ import (
// subject to the rules for snapshots. For example, consider the two range
// tombstones [a,e)#1 and [c,g)#2:
//
// 2: c-------g
// 1: a-------e
// 2: c-------g
// 1: a-------e
//
// These tombstones will be fragmented into:
//
// 2: c---e---g
// 1: a---c---e
// 2: c---e---g
// 1: a---c---e
//
// Do we output the fragment [c,e)#1? Since it is covered by [c-e]#2 the answer
// depends on whether it is in a new snapshot stripe.
@@ -174,14 +175,20 @@ type compactionIter struct {
// advanced.
valueBuf []byte
// Is the current entry valid?
valid bool
iterKey *InternalKey
iterValue []byte
valid bool
iterKey *InternalKey
iterValue []byte
iterStripeChange stripeChangeType
// `skip` indicates whether the remaining skippable entries in the current
// snapshot stripe should be skipped or processed. An example of a non-
// skippable entry is a range tombstone as we need to return it from the
// `compactionIter`, even if a key covering its start key has already been
// seen in the same stripe. `skip` has no effect when `pos == iterPosNext`.
//
// TODO(jackson): If we use keyspan.InterleavingIter for range deletions,
// like we do for range keys, the only remaining 'non-skippable' key is
// the invalid key. We should be able to simplify this logic and remove this
// field.
skip bool
// `pos` indicates the iterator position at the top of `Next()`. Its type's
// (`iterPos`) values take on the following meanings in the context of
@@ -194,6 +201,11 @@ type compactionIter struct {
// user key to ensure we've seen all mergeable operands.
// - `iterPosPrev`: this is invalid as compactionIter is forward-only.
pos iterPos
// `snapshotPinned` indicates whether the last point key returned by the
// compaction iterator was only returned because an open snapshot prevents
// its elision. This field only applies to point keys, and not to range
// deletions or range keys.
snapshotPinned bool
// The index of the snapshot for the current key within the snapshots slice.
curSnapshotIdx int
curSnapshotSeqNum uint64
@@ -201,6 +213,17 @@ type compactionIter struct {
// numbers define the snapshot stripes (see the Snapshots description
// above). The sequence numbers are in ascending order.
snapshots []uint64
// frontiers holds a heap of user keys that affect compaction behavior when
// they're exceeded. Before a new key is returned, the compaction iterator
// advances the frontier, notifying any code that subscribed to be notified
// when a key was reached. The primary use today is within the
// implementation of compactionOutputSplitters in compaction.go. Many of
// these splitters wait for the compaction iterator to call Advance(k) when
// it's returning a new key. If the key that they're waiting for is
// surpassed, these splitters update internal state recording that they
// should request a compaction split next time they're asked in
// [shouldSplitBefore].
frontiers frontiers
// Reference to the range deletion tombstone fragmenter (e.g.,
// `compaction.rangeDelFrag`).
rangeDelFrag *keyspan.Fragmenter
@@ -238,6 +261,7 @@ func newCompactionIter(
merge: merge,
iter: iter,
snapshots: snapshots,
frontiers: frontiers{cmp: cmp},
rangeDelFrag: rangeDelFrag,
rangeKeyFrag: rangeKeyFrag,
allowZeroSeqNum: allowZeroSeqNum,
@@ -268,6 +292,7 @@ func (i *compactionIter) First() (*InternalKey, []byte) {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(i.iterKey.SeqNum(), i.snapshots)
}
i.pos = iterPosNext
i.iterStripeChange = newStripeNewKey
return i.Next()
}
@@ -300,7 +325,19 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
i.pos = iterPosCurForward
i.valid = false
for i.iterKey != nil {
// If we entered a new snapshot stripe with the same key, any key we
// return on this iteration is only returned because the open snapshot
// prevented it from being elided or merged with the key returned for
// the previous stripe. Mark it as pinned so that the compaction loop
// can correctly populate output tables' pinned statistics. We might
// also set snapshotPinned=true down below if we observe that the key is
// deleted by a range deletion in a higher stripe or that this key is a
// tombstone that could be elided if only it were in the last snapshot
// stripe.
i.snapshotPinned = i.iterStripeChange == newStripeSameKey
if i.iterKey.Kind() == InternalKeyKindRangeDelete || rangekey.IsRangeKey(i.iterKey.Kind()) {
// Return the span so the compaction can use it for file truncation and add
// it to the relevant fragmenter. We do not set `skip` to true before
@@ -327,25 +364,42 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
// This goes against the comment on i.key in the struct, and
// therefore warrants some investigation.
i.saveKey()
// TODO(jackson): Handle tracking pinned statistics for range keys
// and range deletions. This would require updating
// emitRangeDelChunk and rangeKeyCompactionTransform to update
// statistics when they apply their own snapshot striping logic.
i.snapshotPinned = false
i.value = i.iterValue
i.valid = true
return &i.key, i.value
}
if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) {
if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly {
// A pending range deletion deletes this key. Skip it.
i.saveKey()
i.skipInStripe()
continue
} else if cover == keyspan.CoversInvisibly {
// i.iterKey would be deleted by a range deletion if there weren't
// any open snapshots. Mark it as pinned.
i.snapshotPinned = true
}
switch i.iterKey.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
// If we're at the last snapshot stripe and the tombstone can be elided
// skip skippable keys in the same stripe.
if i.curSnapshotIdx == 0 && i.elideTombstone(i.iterKey.UserKey) {
i.saveKey()
i.skipInStripe()
continue
if i.elideTombstone(i.iterKey.UserKey) {
if i.curSnapshotIdx == 0 {
// If we're at the last snapshot stripe and the tombstone
// can be elided skip skippable keys in the same stripe.
i.saveKey()
i.skipInStripe()
continue
} else {
// We're not at the last snapshot stripe, so the tombstone
// can NOT yet be elided. Mark it as pinned, so that it's
// included in table statistics appropriately.
i.snapshotPinned = true
}
}
switch i.iterKey.Kind() {
@@ -360,7 +414,6 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
if i.singleDeleteNext() {
return &i.key, i.value
}
continue
}
@@ -458,18 +511,13 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) {
// skipInStripe skips over skippable keys in the same stripe and user key.
func (i *compactionIter) skipInStripe() {
i.skip = true
var change stripeChangeType
for {
change = i.nextInStripe()
if change == sameStripeNonSkippable || change == newStripe {
break
}
for i.nextInStripe() == sameStripeSkippable {
}
// Reset skip if we landed outside the original stripe. Otherwise, we landed
// in the same stripe on a non-skippable key. In that case we should preserve
// `i.skip == true` such that later keys in the stripe will continue to be
// skipped.
if change == newStripe {
if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey {
i.skip = false
}
}
@@ -484,12 +532,16 @@ func (i *compactionIter) iterNext() bool {
return i.iterKey != nil
}
// stripeChangeType indicates how the snapshot stripe changed relative to the previous
// key. If no change, it also indicates whether the current entry is skippable.
// stripeChangeType indicates how the snapshot stripe changed relative to the
// previous key. If no change, it also indicates whether the current entry is
// skippable. If the snapshot stripe changed, it also indicates whether the new
// stripe was entered because the iterator progressed onto an entirely new key
// or entered a new stripe within the same key.
type stripeChangeType int
const (
newStripe stripeChangeType = iota
newStripeNewKey stripeChangeType = iota
newStripeSameKey
sameStripeSkippable
sameStripeNonSkippable
)
@@ -503,8 +555,15 @@ const (
// overwriting or mutating the saved key or value before they have been returned
// to the caller of the exported function (i.e. the caller of Next, First, etc.)
func (i *compactionIter) nextInStripe() stripeChangeType {
i.iterStripeChange = i.nextInStripeHelper()
return i.iterStripeChange
}
// nextInStripeHelper is an internal helper for nextInStripe; callers should use
// nextInStripe and not call nextInStripeHelper.
func (i *compactionIter) nextInStripeHelper() stripeChangeType {
if !i.iterNext() {
return newStripe
return newStripeNewKey
}
key := i.iterKey
@@ -525,10 +584,10 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey))
}
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripe
return newStripeNewKey
} else if !i.equal(i.key.UserKey, key.UserKey) {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripe
return newStripeNewKey
}
origSnapshotIdx := i.curSnapshotIdx
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
@@ -540,7 +599,7 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripe
return newStripeSameKey
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
// Range keys are interleaved at the max sequence number for a given user
// key, so we should not see any more range keys in this stripe.
@@ -549,12 +608,12 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripe
return newStripeSameKey
}
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeSkippable
}
return newStripe
return newStripeSameKey
}
func (i *compactionIter) setNext() {
@@ -581,49 +640,50 @@ func (i *compactionIter) setNext() {
// Else, we continue to loop through entries in the stripe looking for a
// DEL. Note that we may stop *before* encountering a DEL, if one exists.
for {
switch t := i.nextInStripe(); t {
case newStripe, sameStripeNonSkippable:
switch i.nextInStripe() {
case newStripeNewKey, newStripeSameKey:
i.pos = iterPosNext
if t == sameStripeNonSkippable {
// We iterated onto a key that we cannot skip. We can
// conservatively transform the original SET into a SETWITHDEL
// as an indication that there *may* still be a DEL/SINGLEDEL
// under this SET, even if we did not actually encounter one.
//
// This is safe to do, as:
//
// - in the case that there *is not* actually a DEL/SINGLEDEL
// under this entry, any SINGLEDEL above this now-transformed
// SETWITHDEL will become a DEL when the two encounter in a
// compaction. The DEL will eventually be elided in a
// subsequent compaction. The cost for ensuring correctness is
// that this entry is kept around for an additional compaction
// cycle(s).
//
// - in the case there *is* indeed a DEL/SINGLEDEL under us
// (but in a different stripe or sstable), then we will have
// already done the work to transform the SET into a
// SETWITHDEL, and we will skip any additional iteration when
// this entry is encountered again in a subsequent compaction.
//
// Ideally, this codepath would be smart enough to handle the
// case of SET <- RANGEDEL <- ... <- DEL/SINGLEDEL <- ....
// This requires preserving any RANGEDEL entries we encounter
// along the way, then emitting the original (possibly
// transformed) key, followed by the RANGEDELs. This requires
// a sizable refactoring of the existing code, as nextInStripe
// currently returns a sameStripeNonSkippable when it
// encounters a RANGEDEL.
// TODO(travers): optimize to handle the RANGEDEL case if it
// turns out to be a performance problem.
i.key.SetKind(InternalKeyKindSetWithDelete)
return
case sameStripeNonSkippable:
i.pos = iterPosNext
// We iterated onto a key that we cannot skip. We can
// conservatively transform the original SET into a SETWITHDEL
// as an indication that there *may* still be a DEL/SINGLEDEL
// under this SET, even if we did not actually encounter one.
//
// This is safe to do, as:
//
// - in the case that there *is not* actually a DEL/SINGLEDEL
// under this entry, any SINGLEDEL above this now-transformed
// SETWITHDEL will become a DEL when the two encounter in a
// compaction. The DEL will eventually be elided in a
// subsequent compaction. The cost for ensuring correctness is
// that this entry is kept around for an additional compaction
// cycle(s).
//
// - in the case there *is* indeed a DEL/SINGLEDEL under us
// (but in a different stripe or sstable), then we will have
// already done the work to transform the SET into a
// SETWITHDEL, and we will skip any additional iteration when
// this entry is encountered again in a subsequent compaction.
//
// Ideally, this codepath would be smart enough to handle the
// case of SET <- RANGEDEL <- ... <- DEL/SINGLEDEL <- ....
// This requires preserving any RANGEDEL entries we encounter
// along the way, then emitting the original (possibly
// transformed) key, followed by the RANGEDELs. This requires
// a sizable refactoring of the existing code, as nextInStripe
// currently returns a sameStripeNonSkippable when it
// encounters a RANGEDEL.
// TODO(travers): optimize to handle the RANGEDEL case if it
// turns out to be a performance problem.
i.key.SetKind(InternalKeyKindSetWithDelete)
// By setting i.skip=true, we are saying that after the
// non-skippable key is emitted (which is likely a RANGEDEL),
// the remaining point keys that share the same user key as this
// saved key should be skipped.
i.skip = true
}
// By setting i.skip=true, we are saying that after the
// non-skippable key is emitted (which is likely a RANGEDEL),
// the remaining point keys that share the same user key as this
// saved key should be skipped.
i.skip = true
return
case sameStripeSkippable:
// We're still in the same stripe. If this is a DEL/SINGLEDEL, we
@@ -636,7 +696,7 @@ func (i *compactionIter) setNext() {
return
}
default:
panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(t)))
panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange)))
}
}
}
@@ -649,9 +709,9 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
// Loop looking for older values in the current snapshot stripe and merge
// them.
for {
if change := i.nextInStripe(); change == sameStripeNonSkippable || change == newStripe {
if i.nextInStripe() != sameStripeSkippable {
i.pos = iterPosNext
return change
return i.iterStripeChange
}
key := i.iterKey
switch key.Kind() {
@@ -677,7 +737,7 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
return sameStripeSkippable
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) {
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
@@ -701,7 +761,7 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
return sameStripeSkippable
case InternalKeyKindMerge:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) {
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
@@ -735,7 +795,7 @@ func (i *compactionIter) singleDeleteNext() bool {
// Loop until finds a key to be passed to the next level.
for {
if change := i.nextInStripe(); change == sameStripeNonSkippable || change == newStripe {
if i.nextInStripe() != sameStripeSkippable {
i.pos = iterPosNext
return true
}
@@ -770,6 +830,7 @@ func (i *compactionIter) saveKey() {
i.key.UserKey = i.keyBuf
i.key.Trailer = i.iterKey.Trailer
i.keyTrailer = i.iterKey.Trailer
i.frontiers.Advance(i.key.UserKey)
}
func (i *compactionIter) cloneKey(key []byte) []byte {
@@ -896,5 +957,206 @@ func (i *compactionIter) maybeZeroSeqnum(snapshotIdx int) {
// This is not the last snapshot
return
}
i.key.SetSeqNum(0)
i.key.SetSeqNum(base.SeqNumZero)
}
// A frontier is used to monitor a compaction's progression across the user
// keyspace.
//
// A frontier hold a user key boundary that it's concerned with in its `key`
// field. If/when the compaction iterator returns an InternalKey with a user key
// _k_ such that k ≥ frontier.key, the compaction iterator invokes the
// frontier's `reached` function, passing _k_ as its argument.
//
// The `reached` function returns a new value to use as the key. If `reached`
// returns nil, the frontier is forgotten and its `reached` method will not be
// invoked again, unless the user calls [Update] to set a new key.
//
// A frontier's key may be updated outside the context of a `reached`
// invocation at any time, through its Update method.
type frontier struct {
// container points to the containing *frontiers that was passed to Init
// when the frontier was initialized.
container *frontiers
// key holds the frontier's current key. If nil, this frontier is inactive
// and its reached func will not be invoked. The value of this key may only
// be updated by the `frontiers` type, or the Update method.
key []byte
// reached is invoked to inform a frontier that its key has been reached.
// It's invoked with the user key that reached the limit. The `key` argument
// is guaranteed to be ≥ the frontier's key.
//
// After reached is invoked, the frontier's key is updated to the return
// value of `reached`. Note bene, the frontier is permitted to update its
// key to a user key ≤ the argument `key`.
//
// If a frontier is set to key k1, and reached(k2) is invoked (k2 ≥ k1), the
// frontier will receive reached(k2) calls until it returns nil or a key
// `k3` such that k2 < k3. This property is useful for frontiers that use
// `reached` invocations to drive iteration through collections of keys that
// may contain multiple keys that are both < k2 and ≥ k1.
reached func(key []byte) (next []byte)
}
// Init initializes the frontier with the provided key and reached callback.
// The frontier is attached to the provided *frontiers and the provided reached
// func will be invoked when the *frontiers is advanced to a key ≥ this
// frontier's key.
func (f *frontier) Init(
frontiers *frontiers, initialKey []byte, reached func(key []byte) (next []byte),
) {
*f = frontier{
container: frontiers,
key: initialKey,
reached: reached,
}
if initialKey != nil {
f.container.push(f)
}
}
// String implements fmt.Stringer.
func (f *frontier) String() string {
return string(f.key)
}
// Update replaces the existing frontier's key with the provided key. The
// frontier's reached func will be invoked when the new key is reached.
func (f *frontier) Update(key []byte) {
c := f.container
prevKeyIsNil := f.key == nil
f.key = key
if prevKeyIsNil {
if key != nil {
c.push(f)
}
return
}
// Find the frontier within the heap (it must exist within the heap because
// f.key was != nil). If the frontier key is now nil, remove it from the
// heap. Otherwise, fix up its position.
for i := 0; i < len(c.items); i++ {
if c.items[i] == f {
if key != nil {
c.fix(i)
} else {
n := c.len() - 1
c.swap(i, n)
c.down(i, n)
c.items = c.items[:n]
}
return
}
}
panic("unreachable")
}
// frontiers is used to track progression of a task (eg, compaction) across the
// keyspace. Clients that want to be informed when the task advances to a key ≥
// some frontier may register a frontier, providing a callback. The task calls
// `Advance(k)` with each user key encountered, which invokes the `reached` func
// on all tracked frontiers with `key`s ≤ k.
//
// Internally, frontiers is implemented as a simple heap.
type frontiers struct {
cmp Compare
items []*frontier
}
// String implements fmt.Stringer.
func (f *frontiers) String() string {
var buf bytes.Buffer
for i := 0; i < len(f.items); i++ {
if i > 0 {
fmt.Fprint(&buf, ", ")
}
fmt.Fprintf(&buf, "%s: %q", f.items[i], f.items[i].key)
}
return buf.String()
}
// Advance notifies all member frontiers with keys ≤ k.
func (f *frontiers) Advance(k []byte) {
for len(f.items) > 0 && f.cmp(k, f.items[0].key) >= 0 {
// This frontier has been reached. Invoke the closure and update with
// the next frontier.
f.items[0].key = f.items[0].reached(k)
if f.items[0].key == nil {
// This was the final frontier that this user was concerned with.
// Remove it from the heap.
f.pop()
} else {
// Fix up the heap root.
f.fix(0)
}
}
}
func (f *frontiers) len() int {
return len(f.items)
}
func (f *frontiers) less(i, j int) bool {
return f.cmp(f.items[i].key, f.items[j].key) < 0
}
func (f *frontiers) swap(i, j int) {
f.items[i], f.items[j] = f.items[j], f.items[i]
}
// fix, up and down are copied from the go stdlib.
func (f *frontiers) fix(i int) {
if !f.down(i, f.len()) {
f.up(i)
}
}
func (f *frontiers) push(ff *frontier) {
n := len(f.items)
f.items = append(f.items, ff)
f.up(n)
}
func (f *frontiers) pop() *frontier {
n := f.len() - 1
f.swap(0, n)
f.down(0, n)
item := f.items[n]
f.items = f.items[:n]
return item
}
func (f *frontiers) up(j int) {
for {
i := (j - 1) / 2 // parent
if i == j || !f.less(j, i) {
break
}
f.swap(i, j)
j = i
}
}
func (f *frontiers) down(i0, n int) bool {
i := i0
for {
j1 := 2*i + 1
if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
break
}
j := j1 // left child
if j2 := j1 + 1; j2 < n && f.less(j2, j1) {
j = j2 // = 2*i + 2 // right child
}
if !f.less(j, i) {
break
}
f.swap(i, j)
i = j
}
return i > i0
}

191
vendor/github.com/cockroachdb/pebble/compaction_picker.go generated vendored
View File

@@ -9,6 +9,7 @@ import (
"fmt"
"math"
"sort"
"strings"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/humanize"
@@ -46,13 +47,17 @@ type readCompactionEnv struct {
flushing bool
}
// Information about in-progress compactions provided to the compaction picker. These are used to
// constrain the new compactions that will be picked.
// Information about in-progress compactions provided to the compaction picker.
// These are used to constrain the new compactions that will be picked.
type compactionInfo struct {
inputs []compactionLevel
outputLevel int
smallest InternalKey
largest InternalKey
// versionEditApplied is true if this compaction's version edit has already
// been committed. The compaction may still be in-progress deleting newly
// obsolete files.
versionEditApplied bool
inputs []compactionLevel
outputLevel int
smallest InternalKey
largest InternalKey
}
func (info compactionInfo) String() string {
@@ -98,6 +103,16 @@ type sublevelInfo struct {
sublevel manifest.Level
}
func (cl sublevelInfo) Clone() sublevelInfo {
return sublevelInfo{
sublevel: cl.sublevel,
LevelSlice: cl.LevelSlice.Reslice(func(start, end *manifest.LevelIterator) {}),
}
}
func (cl sublevelInfo) String() string {
return fmt.Sprintf(`Sublevel %s; Levels %s`, cl.sublevel, cl.LevelSlice)
}
// generateSublevelInfo will generate the level slices for each of the sublevels
// from the level slice for all of L0.
func generateSublevelInfo(cmp base.Compare, levelFiles manifest.LevelSlice) []sublevelInfo {
@@ -257,6 +272,69 @@ func newPickedCompactionFromL0(
return pc
}
func (pc *pickedCompaction) String() string {
var builder strings.Builder
builder.WriteString(fmt.Sprintf(`Score=%f, `, pc.score))
builder.WriteString(fmt.Sprintf(`Kind=%s, `, pc.kind))
builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, pc.adjustedOutputLevel))
builder.WriteString(fmt.Sprintf(`maxOutputFileSize=%d, `, pc.maxOutputFileSize))
builder.WriteString(fmt.Sprintf(`maxReadCompactionBytes=%d, `, pc.maxReadCompactionBytes))
builder.WriteString(fmt.Sprintf(`smallest=%s, `, pc.smallest))
builder.WriteString(fmt.Sprintf(`largest=%s, `, pc.largest))
builder.WriteString(fmt.Sprintf(`version=%s, `, pc.version))
builder.WriteString(fmt.Sprintf(`inputs=%s, `, pc.inputs))
builder.WriteString(fmt.Sprintf(`startlevel=%s, `, pc.startLevel))
builder.WriteString(fmt.Sprintf(`outputLevel=%s, `, pc.outputLevel))
builder.WriteString(fmt.Sprintf(`extraLevels=%s, `, pc.extraLevels))
builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.l0SublevelInfo))
builder.WriteString(fmt.Sprintf(`lcf=%s`, pc.lcf))
return builder.String()
}
// Clone creates a deep copy of the pickedCompaction
func (pc *pickedCompaction) clone() *pickedCompaction {
// Quickly copy over fields that do not require special deep copy care, and
// set all fields that will require a deep copy to nil.
newPC := &pickedCompaction{
cmp: pc.cmp,
score: pc.score,
kind: pc.kind,
adjustedOutputLevel: pc.adjustedOutputLevel,
maxOutputFileSize: pc.maxOutputFileSize,
maxOverlapBytes: pc.maxOverlapBytes,
maxReadCompactionBytes: pc.maxReadCompactionBytes,
smallest: pc.smallest.Clone(),
largest: pc.largest.Clone(),
// Both copies see the same manifest, therefore, it's ok for them to se
// share the same pc. version.
version: pc.version,
}
newPC.inputs = make([]compactionLevel, len(pc.inputs))
newPC.extraLevels = make([]*compactionLevel, 0, len(pc.extraLevels))
for i := range pc.inputs {
newPC.inputs[i] = pc.inputs[i].Clone()
if i == 0 {
newPC.startLevel = &newPC.inputs[i]
} else if i == len(pc.inputs)-1 {
newPC.outputLevel = &newPC.inputs[i]
} else {
newPC.extraLevels = append(newPC.extraLevels, &newPC.inputs[i])
}
}
newPC.l0SublevelInfo = make([]sublevelInfo, len(pc.l0SublevelInfo))
for i := range pc.l0SublevelInfo {
newPC.l0SublevelInfo[i] = pc.l0SublevelInfo[i].Clone()
}
if pc.lcf != nil {
newPC.lcf = pc.lcf.Clone()
}
return newPC
}
// maybeExpandedBounds is a helper function for setupInputs which ensures the
// pickedCompaction's smallest and largest internal keys are updated iff
// the candidate keys expand the key span. This avoids a bug for multi-level
@@ -286,6 +364,8 @@ func (pc *pickedCompaction) maybeExpandBounds(smallest InternalKey, largest Inte
}
}
// setupInputs returns true if a compaction has been set up. It returns false if
// a concurrent compaction is occurring on the start or output level files.
func (pc *pickedCompaction) setupInputs(
opts *Options, diskAvailBytes uint64, startLevel *compactionLevel,
) bool {
@@ -362,8 +442,7 @@ func (pc *pickedCompaction) setupInputs(
}
})
}
oldLcf := *pc.lcf
oldLcf := pc.lcf.Clone()
if pc.version.L0Sublevels.ExtendL0ForBaseCompactionTo(smallestBaseKey, largestBaseKey, pc.lcf) {
var newStartLevelFiles []*fileMetadata
iter := pc.version.Levels[0].Iter()
@@ -379,7 +458,7 @@ func (pc *pickedCompaction) setupInputs(
pc.smallest, pc.largest = manifest.KeyRange(pc.cmp,
startLevel.files.Iter(), pc.outputLevel.files.Iter())
} else {
*pc.lcf = oldLcf
*pc.lcf = *oldLcf
}
}
} else if pc.grow(pc.smallest, pc.largest, maxExpandedBytes, startLevel) {
@@ -433,12 +512,28 @@ func (pc *pickedCompaction) grow(
return true
}
// initMultiLevelCompaction returns true if it initiated a multilevel input
// compaction. This currently never inits a multiLevel compaction.
func (pc *pickedCompaction) initMultiLevelCompaction(
opts *Options, vers *version, levelMaxBytes [7]int64, diskAvailBytes uint64,
) bool {
return false
func (pc *pickedCompaction) compactionSize() uint64 {
var bytesToCompact uint64
for i := range pc.inputs {
bytesToCompact += pc.inputs[i].files.SizeSum()
}
return bytesToCompact
}
// setupMultiLevelCandidated returns true if it successfully added another level
// to the compaction.
func (pc *pickedCompaction) setupMultiLevelCandidate(opts *Options, diskAvailBytes uint64) bool {
pc.inputs = append(pc.inputs, compactionLevel{level: pc.outputLevel.level + 1})
// Recalibrate startLevel and outputLevel:
// - startLevel and outputLevel pointers may be obsolete after appending to pc.inputs.
// - push outputLevel to extraLevels and move the new level to outputLevel
pc.startLevel = &pc.inputs[0]
pc.extraLevels = []*compactionLevel{&pc.inputs[1]}
pc.outputLevel = &pc.inputs[2]
pc.adjustedOutputLevel++
return pc.setupInputs(opts, diskAvailBytes, pc.extraLevels[len(pc.extraLevels)-1])
}
// expandToAtomicUnit expands the provided level slice within its level both
@@ -461,18 +556,18 @@ func (pc *pickedCompaction) initMultiLevelCompaction(
// truncation of range tombstones to atomic compaction unit boundaries.
// Consider the scenario:
//
// L3:
// 12:[a#2,15-b#1,1]
// 13:[b#0,15-d#72057594037927935,15]
// L3:
// 12:[a#2,15-b#1,1]
// 13:[b#0,15-d#72057594037927935,15]
//
// These sstables contain a range tombstone [a-d)#2 which spans the two
// sstables. The two sstables need to always be kept together. Compacting
// sstable 13 independently of sstable 12 would result in:
//
// L3:
// 12:[a#2,15-b#1,1]
// L4:
// 14:[b#0,15-d#72057594037927935,15]
// L3:
// 12:[a#2,15-b#1,1]
// L4:
// 14:[b#0,15-d#72057594037927935,15]
//
// This state is still ok, but when sstable 12 is next compacted, its range
// tombstones will be truncated at "b" (the largest key in its atomic
@@ -500,6 +595,10 @@ func expandToAtomicUnit(
return inputs, false
}
// TODO(jackson): Update to avoid use of LevelIterator.Current(). The
// Reslice interface will require some tweaking, because we currently rely
// on Reslice having already positioned the LevelIterator appropriately.
inputs = inputs.Reslice(func(start, end *manifest.LevelIterator) {
iter := start.Clone()
iter.Prev()
@@ -611,7 +710,7 @@ func (a compensatedSizeAnnotator) Accumulate(
) (v interface{}, cacheOK bool) {
vptr := dst.(*uint64)
*vptr = *vptr + compensatedSize(f, a.pointTombstoneWeight)
return vptr, f.StatsValidLocked()
return vptr, f.StatsValid()
}
func (a compensatedSizeAnnotator) Merge(src interface{}, dst interface{}) interface{} {
@@ -1237,7 +1336,7 @@ func (a elisionOnlyAnnotator) Accumulate(f *fileMetadata, dst interface{}) (inte
if f.IsCompacting() {
return dst, true
}
if !f.StatsValidLocked() {
if !f.StatsValid() {
return dst, false
}
// Bottommost files are large and not worthwhile to compact just
@@ -1328,6 +1427,9 @@ func markedMergeHelper(f *fileMetadata, dst interface{}) (interface{}, bool) {
func (p *compactionPickerByScore) pickElisionOnlyCompaction(
env compactionEnv,
) (pc *pickedCompaction) {
if p.opts.private.disableElisionOnlyCompactions {
return nil
}
v := p.vers.Levels[numLevels-1].Annotation(elisionOnlyAnnotator{})
if v == nil {
return nil
@@ -1456,12 +1558,35 @@ func pickAutoLPositive(
if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) {
return nil
}
if opts.Experimental.MultiLevelCompaction &&
pc.initMultiLevelCompaction(opts, vers, levelMaxBytes, diskAvailBytes()) {
if !pc.setupInputs(opts, diskAvailBytes(), pc.extraLevels[len(pc.extraLevels)-1]) {
return nil
}
return pc.maybeAddLevel(opts, diskAvailBytes())
}
// maybeAddLevel maybe adds a level to the picked compaction.
func (pc *pickedCompaction) maybeAddLevel(opts *Options, diskAvailBytes uint64) *pickedCompaction {
if pc.outputLevel.level == numLevels-1 {
// Don't add a level if the current output level is in L6
return pc
}
if pc.compactionSize() > expandedCompactionByteSizeLimit(
opts, pc.adjustedOutputLevel, diskAvailBytes) {
// Don't add a level if the current compaction exceeds the compaction size limit
return pc
}
return opts.Experimental.MultiLevelCompactionHueristic.pick(pc, opts, diskAvailBytes)
}
// MultiLevelHeuristic evaluates whether to add files from the next level into the compaction.
type MultiLevelHeuristic interface {
// evaluate returns the preferred compaction.
pick(pc *pickedCompaction, opts *Options, diskAvailBytes uint64) *pickedCompaction
}
// NoMultiLevel will never add an additional level to the compaction.
type NoMultiLevel struct{}
func (nml NoMultiLevel) pick(
pc *pickedCompaction, opts *Options, diskAvailBytes uint64,
) *pickedCompaction {
return pc
}
@@ -1585,13 +1710,7 @@ func pickManualHelper(
if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) {
return nil
}
if opts.Experimental.MultiLevelCompaction && pc.startLevel.level > 0 &&
pc.initMultiLevelCompaction(opts, vers, levelMaxBytes, diskAvailBytes()) {
if !pc.setupInputs(opts, diskAvailBytes(), pc.extraLevels[len(pc.extraLevels)-1]) {
return nil
}
}
return pc
return pc.maybeAddLevel(opts, diskAvailBytes())
}
func (p *compactionPickerByScore) pickReadTriggeredCompaction(

759
vendor/github.com/cockroachdb/pebble/db.go generated vendored
View File

File diff suppressed because it is too large Load Diff

4
vendor/github.com/cockroachdb/pebble/error_iter.go generated vendored
View File

@@ -50,6 +50,10 @@ func (c *errorIter) Prev() (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) Error() error {
return c.err
}

107
vendor/github.com/cockroachdb/pebble/event.go generated vendored
View File

@@ -5,11 +5,15 @@
package pebble
import (
"fmt"
"strings"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/redact"
)
@@ -115,22 +119,7 @@ func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) {
// DiskSlowInfo contains the info for a disk slowness event when writing to a
// file.
type DiskSlowInfo struct {
// Path of file being written to.
Path string
// Duration that has elapsed since this disk operation started.
Duration time.Duration
}
func (i DiskSlowInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("disk slowness detected: write to file %s has been ongoing for %0.1fs",
i.Path, redact.Safe(i.Duration.Seconds()))
}
type DiskSlowInfo = vfs.DiskSlowInfo
// FlushInfo contains the info for a flush event.
type FlushInfo struct {
@@ -149,8 +138,14 @@ type FlushInfo struct {
// TotalDuration is the total wall-time duration of the flush, including
// applying the flush to the database. TotalDuration is always ≥ Duration.
TotalDuration time.Duration
Done bool
Err error
// Ingest is set to true if the flush is handling tables that were added to
// the flushable queue via an ingestion operation.
Ingest bool
// IngestLevels are the output levels for each ingested table in the flush.
// This field is only populated when Ingest is true.
IngestLevels []int
Done bool
Err error
}
func (i FlushInfo) String() string {
@@ -169,20 +164,47 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
plural = ""
}
if !i.Done {
w.Printf("[JOB %d] flushing %d memtable", redact.Safe(i.JobID), redact.Safe(i.Input))
w.SafeString(plural)
w.Printf(" to L0")
w.Printf("[JOB %d] ", redact.Safe(i.JobID))
if !i.Ingest {
w.Printf("flushing %d memtable", redact.Safe(i.Input))
w.SafeString(plural)
w.Printf(" to L0")
} else {
w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural)
}
return
}
outputSize := tablesTotalSize(i.Output)
w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.JobID), redact.Safe(i.Input), plural,
redact.Safe(formatFileNums(i.Output)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
if !i.Ingest {
if invariants.Enabled && len(i.IngestLevels) > 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0"))
}
w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.JobID), redact.Safe(i.Input), plural,
redact.Safe(formatFileNums(i.Output)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
} else {
if invariants.Enabled && len(i.IngestLevels) == 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0"))
}
w.Printf("[JOB %d] flushed %d ingested flushable%s",
redact.Safe(i.JobID), redact.Safe(len(i.Output)), plural)
for j, level := range i.IngestLevels {
file := i.Output[j]
if j > 0 {
w.Printf(" +")
}
w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.IEC.Uint64(file.Size))
}
w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
}
// ManifestCreateInfo contains info about a manifest creation event.
@@ -283,7 +305,10 @@ type TableIngestInfo struct {
// GlobalSeqNum is the sequence number that was assigned to all entries in
// the ingested table.
GlobalSeqNum uint64
Err error
// flushable indicates whether the ingested sstable was treated as a
// flushable.
flushable bool
Err error
}
func (i TableIngestInfo) String() string {
@@ -297,13 +322,22 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
return
}
w.Printf("[JOB %d] ingested", redact.Safe(i.JobID))
if i.flushable {
w.Printf("[JOB %d] ingested as flushable", redact.Safe(i.JobID))
} else {
w.Printf("[JOB %d] ingested", redact.Safe(i.JobID))
}
for j := range i.Tables {
t := &i.Tables[j]
if j > 0 {
w.Printf(",")
}
w.Printf(" L%d:%s (%s)", redact.Safe(t.Level), redact.Safe(t.FileNum),
levelStr := ""
if !i.flushable {
levelStr = fmt.Sprintf("L%d:", t.Level)
}
w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum),
redact.Safe(humanize.Uint64(t.Size)))
}
}
@@ -427,9 +461,14 @@ type EventListener struct {
// has been installed.
CompactionEnd func(CompactionInfo)
// DiskSlow is invoked after a disk write operation on a file created
// with a disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks)
// is observed to exceed the specified disk slowness threshold duration.
// DiskSlow is invoked after a disk write operation on a file created with a
// disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks) is
// observed to exceed the specified disk slowness threshold duration. DiskSlow
// is called on a goroutine that is monitoring slowness/stuckness. The callee
// MUST return without doing any IO, or blocking on anything (like a mutex)
// that is waiting on IO. This is imperative in order to reliably monitor for
// slowness, since if this goroutine gets stuck, the monitoring will stop
// working.
DiskSlow func(DiskSlowInfo)
// FlushBegin is invoked after the inputs to a flush have been determined,

57
vendor/github.com/cockroachdb/pebble/external_iterator.go generated vendored
View File

@@ -5,6 +5,7 @@
package pebble
import (
"context"
"fmt"
"sort"
@@ -79,6 +80,18 @@ func NewExternalIter(
iterOpts *IterOptions,
files [][]sstable.ReadableFile,
extraOpts ...ExternalIterOption,
) (it *Iterator, err error) {
return NewExternalIterWithContext(context.Background(), o, iterOpts, files, extraOpts...)
}
// NewExternalIterWithContext is like NewExternalIter, and additionally
// accepts a context for tracing.
func NewExternalIterWithContext(
ctx context.Context,
o *Options,
iterOpts *IterOptions,
files [][]sstable.ReadableFile,
extraOpts ...ExternalIterOption,
) (it *Iterator, err error) {
if iterOpts != nil {
if err := validateExternalIterOpts(iterOpts); err != nil {
@@ -119,6 +132,7 @@ func NewExternalIter(
buf := iterAllocPool.Get().(*iterAlloc)
dbi := &buf.dbi
*dbi = Iterator{
ctx: ctx,
alloc: buf,
merge: o.Merger.Merge,
comparer: *o.Comparer,
@@ -130,7 +144,9 @@ func NewExternalIter(
// Add the readers to the Iterator so that Close closes them, and
// SetOptions can re-construct iterators from them.
externalReaders: readers,
newIters: func(f *manifest.FileMetadata, opts *IterOptions, internalOpts internalIterOpts) (internalIterator, keyspan.FragmentIterator, error) {
newIters: func(
ctx context.Context, f *manifest.FileMetadata, opts *IterOptions,
internalOpts internalIterOpts) (internalIterator, keyspan.FragmentIterator, error) {
// NB: External iterators are currently constructed without any
// `levelIters`. newIters should never be called. When we support
// organizing multiple non-overlapping files into a single level
@@ -143,12 +159,12 @@ func NewExternalIter(
}
if iterOpts != nil {
dbi.opts = *iterOpts
dbi.saveBounds(iterOpts.LowerBound, iterOpts.UpperBound)
dbi.processBounds(iterOpts.LowerBound, iterOpts.UpperBound)
}
for i := range extraOpts {
extraOpts[i].iterApply(dbi)
}
finishInitializingExternal(dbi)
finishInitializingExternal(ctx, dbi)
return dbi, nil
}
@@ -168,7 +184,7 @@ func validateExternalIterOpts(iterOpts *IterOptions) error {
return nil
}
func createExternalPointIter(it *Iterator) (internalIterator, error) {
func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterator, error) {
// TODO(jackson): In some instances we could generate fewer levels by using
// L0Sublevels code to organize nonoverlapping files into the same level.
// This would allow us to use levelIters and keep a smaller set of data and
@@ -193,12 +209,14 @@ func createExternalPointIter(it *Iterator) (internalIterator, error) {
pointIter internalIterator
err error
)
pointIter, err = r.NewIterWithBlockPropertyFilters(
pointIter, err = r.NewIterWithBlockPropertyFiltersAndContext(
ctx,
it.opts.LowerBound,
it.opts.UpperBound,
nil, /* BlockPropertiesFilterer */
false, /* useFilterBlock */
&it.stats.InternalStats,
sstable.TrivialReaderProvider{Reader: r},
)
if err != nil {
return nil, err
@@ -248,12 +266,14 @@ func createExternalPointIter(it *Iterator) (internalIterator, error) {
it.alloc.merging.init(&it.opts, &it.stats.InternalStats, it.comparer.Compare, it.comparer.Split, mlevels...)
it.alloc.merging.snapshot = base.InternalKeySeqNumMax
it.alloc.merging.elideRangeTombstones = true
if len(mlevels) <= cap(it.alloc.levelsPositioned) {
it.alloc.merging.levelsPositioned = it.alloc.levelsPositioned[:len(mlevels)]
}
return &it.alloc.merging, nil
}
func finishInitializingExternal(it *Iterator) {
pointIter, err := createExternalPointIter(it)
func finishInitializingExternal(ctx context.Context, it *Iterator) {
pointIter, err := createExternalPointIter(ctx, it)
if err != nil {
it.pointIter = &errorIter{err: err}
} else {
@@ -291,6 +311,7 @@ func finishInitializingExternal(it *Iterator) {
base.InternalKeySeqNumMax,
it.opts.LowerBound, it.opts.UpperBound,
&it.hasPrefix, &it.prefixOrFullSeekKey,
true /* onlySets */, &it.rangeKey.internal,
)
for i := range rangeKeyIters {
it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])
@@ -314,7 +335,11 @@ func openExternalTables(
) (readers []*sstable.Reader, err error) {
readers = make([]*sstable.Reader, 0, len(files))
for i := range files {
r, err := sstable.NewReader(files[i], readerOpts, extraReaderOpts...)
readable, err := sstable.NewSimpleReadable(files[i])
if err != nil {
return readers, err
}
r, err := sstable.NewReader(readable, readerOpts, extraReaderOpts...)
if err != nil {
return readers, err
}
@@ -474,6 +499,20 @@ func (s *simpleLevelIter) Next() (*base.InternalKey, base.LazyValue) {
return s.skipEmptyFileForward(nil /* seekKey */, base.SeekGEFlagsNone)
}
func (s *simpleLevelIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
if s.err != nil {
return nil, base.LazyValue{}
}
if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) {
return nil, base.LazyValue{}
}
if iterKey, val := s.filtered[s.currentIdx].NextPrefix(succKey); iterKey != nil {
return iterKey, val
}
s.currentIdx++
return s.skipEmptyFileForward(succKey /* seekKey */, base.SeekGEFlagsNone)
}
func (s *simpleLevelIter) Prev() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}

2
vendor/github.com/cockroachdb/pebble/filenames.go generated vendored
View File

@@ -33,7 +33,7 @@ const (
// NB: This is a low-level routine and typically not what you want to
// use. Newer versions of Pebble running newer format major versions do
// not use the CURRENT file. See setCurrentFunc in version_set.go.
func setCurrentFile(dirname string, fs vfs.FS, fileNum FileNum) error {
func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error {
newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum)
oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum)
fs.Remove(oldFilename)

111
vendor/github.com/cockroachdb/pebble/flush_external.go generated vendored
View File

@@ -1,111 +0,0 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"time"
"github.com/cockroachdb/pebble/internal/private"
)
// flushExternalTable is installed as the private.FlushExternalTable hook.
// It's used by the internal/replay package to flush external tables into L0
// for supporting the pebble bench compact commands. Clients should use the
// replay package rather than calling this private hook directly.
func flushExternalTable(untypedDB interface{}, path string, originalMeta *fileMetadata) error {
d := untypedDB.(*DB)
if err := d.closed.Load(); err != nil {
panic(err)
}
if d.opts.ReadOnly {
return ErrReadOnly
}
d.mu.Lock()
fileNum := d.mu.versions.getNextFileNum()
jobID := d.mu.nextJobID
d.mu.nextJobID++
d.mu.Unlock()
m := &fileMetadata{
FileNum: fileNum,
Size: originalMeta.Size,
CreationTime: time.Now().Unix(),
SmallestSeqNum: originalMeta.SmallestSeqNum,
LargestSeqNum: originalMeta.LargestSeqNum,
}
if originalMeta.HasPointKeys {
m.ExtendPointKeyBounds(d.cmp, originalMeta.SmallestPointKey, originalMeta.LargestPointKey)
}
if originalMeta.HasRangeKeys {
m.ExtendRangeKeyBounds(d.cmp, originalMeta.SmallestRangeKey, originalMeta.LargestRangeKey)
}
// Hard link the sstable into the DB directory.
if err := ingestLink(jobID, d.opts, d.dirname, []string{path}, []*fileMetadata{m}); err != nil {
return err
}
if err := d.dataDir.Sync(); err != nil {
return err
}
// Assign sequence numbers from its current value through to the
// external file's largest sequence number. It's possible that the current
// sequence number has already exceeded m.LargestSeqNum if there were many
// nonoverlapping ingestions, so skip if this doesn't bump the sequence
// number.
d.commit.ratchetSeqNum(m.LargestSeqNum + 1)
d.commit.mu.Lock()
defer d.commit.mu.Unlock()
// Apply the version edit.
d.mu.Lock()
d.mu.versions.logLock()
ve := &versionEdit{
NewFiles: []newFileEntry{{Level: 0, Meta: m}},
}
metrics := map[int]*LevelMetrics{
0: {
NumFiles: 1,
Size: int64(m.Size),
BytesIngested: m.Size,
TablesIngested: 1,
},
}
err := d.mu.versions.logAndApply(jobID, ve, metrics, false /* forceRotation */, func() []compactionInfo {
return d.getInProgressCompactionInfoLocked(nil)
})
if err != nil {
// NB: logAndApply will release d.mu.versions.logLock unconditionally.
d.mu.Unlock()
if err2 := ingestCleanup(d.opts.FS, d.dirname, []*fileMetadata{m}); err2 != nil {
d.opts.Logger.Infof("flush external cleanup failed: %v", err2)
}
return err
}
d.updateReadStateLocked(d.opts.DebugCheck)
d.updateTableStatsLocked(ve.NewFiles)
d.deleteObsoleteFiles(jobID, true /* waitForOngoing */)
d.maybeScheduleCompaction()
d.mu.Unlock()
return nil
}
// ratchetSeqNum is a hook for allocating and publishing sequence numbers up
// to a specific absolute value. Its first parameter is a *pebble.DB and its
// second is the new next sequence number. RatchetSeqNum does nothing if the
// next sequence is already greater than or equal to nextSeqNum.
//
// This function is used by the internal/replay package to ensure replayed
// operations receive the same absolute sequence number.
func ratchetSeqNum(untypedDB interface{}, nextSeqNum uint64) {
d := untypedDB.(*DB)
d.commit.ratchetSeqNum(nextSeqNum)
}
func init() {
private.FlushExternalTable = flushExternalTable
private.RatchetSeqNum = ratchetSeqNum
}

168
vendor/github.com/cockroachdb/pebble/flushable.go generated vendored
View File

@@ -5,11 +5,13 @@
package pebble
import (
"context"
"fmt"
"sync/atomic"
"time"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
)
// flushable defines the interface for immutable memtables.
@@ -53,12 +55,21 @@ type flushableEntry struct {
// readerRefs tracks the read references on the flushable. The two sources of
// reader references are DB.mu.mem.queue and readState.memtables. The memory
// reserved by the flushable in the cache is released when the reader refs
// drop to zero. If the flushable is a memTable, when the reader refs drops
// to zero, the writer refs will already be zero because the memtable will
// have been flushed and that only occurs once the writer refs drops to zero.
// drop to zero. If the flushable is referencing sstables, then the file
// refount is also decreased once the reader refs drops to 0. If the
// flushable is a memTable, when the reader refs drops to zero, the writer
// refs will already be zero because the memtable will have been flushed and
// that only occurs once the writer refs drops to zero.
readerRefs int32
// Closure to invoke to release memory accounting.
releaseMemAccounting func()
// unrefFiles, if not nil, should be invoked to decrease the ref count of
// files which are backing the flushable.
unrefFiles func() []*fileBacking
// deleteFnLocked should be called if the caller is holding DB.mu.
deleteFnLocked func(obsolete []*fileBacking)
// deleteFn should be called if the caller is not holding DB.mu.
deleteFn func(obsolete []*fileBacking)
}
func (e *flushableEntry) readerRef() {
@@ -68,7 +79,19 @@ func (e *flushableEntry) readerRef() {
}
}
func (e *flushableEntry) readerUnref() {
// db.mu must not be held when this is called.
func (e *flushableEntry) readerUnref(deleteFiles bool) {
e.readerUnrefHelper(deleteFiles, e.deleteFn)
}
// db.mu must be held when this is called.
func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) {
e.readerUnrefHelper(deleteFiles, e.deleteFnLocked)
}
func (e *flushableEntry) readerUnrefHelper(
deleteFiles bool, deleteFn func(obsolete []*fileBacking),
) {
switch v := atomic.AddInt32(&e.readerRefs, -1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
@@ -78,7 +101,144 @@ func (e *flushableEntry) readerUnref() {
}
e.releaseMemAccounting()
e.releaseMemAccounting = nil
if e.unrefFiles != nil {
obsolete := e.unrefFiles()
e.unrefFiles = nil
if deleteFiles {
deleteFn(obsolete)
}
}
}
}
type flushableList []*flushableEntry
// ingestedFlushable is the implementation of the flushable interface for the
// ingesting sstables which are added to the flushable list.
type ingestedFlushable struct {
files []physicalMeta
cmp Compare
split Split
newIters tableNewIters
newRangeKeyIters keyspan.TableNewSpanIter
// Since the level slice is immutable, we construct and set it once. It
// should be safe to read from slice in future reads.
slice manifest.LevelSlice
// hasRangeKeys is set on ingestedFlushable construction.
hasRangeKeys bool
}
func newIngestedFlushable(
files []*fileMetadata,
cmp Compare,
split Split,
newIters tableNewIters,
newRangeKeyIters keyspan.TableNewSpanIter,
) *ingestedFlushable {
var physicalFiles []physicalMeta
var hasRangeKeys bool
for _, f := range files {
if f.HasRangeKeys {
hasRangeKeys = true
}
physicalFiles = append(physicalFiles, f.PhysicalMeta())
}
ret := &ingestedFlushable{
files: physicalFiles,
cmp: cmp,
split: split,
newIters: newIters,
newRangeKeyIters: newRangeKeyIters,
// slice is immutable and can be set once and used many times.
slice: manifest.NewLevelSliceKeySorted(cmp, files),
hasRangeKeys: hasRangeKeys,
}
return ret
}
// TODO(sumeer): ingestedFlushable iters also need to plumb context for
// tracing.
func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
var opts IterOptions
if o != nil {
opts = *o
}
// TODO(bananabrick): The manifest.Level in newLevelIter is only used for
// logging. Update the manifest.Level encoding to account for levels which
// aren't truly levels in the lsm. Right now, the encoding only supports
// L0 sublevels, and the rest of the levels in the lsm.
return newLevelIter(
opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), nil,
)
}
func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
// newFlushIter is only used for writing memtables to disk as sstables.
// Since ingested sstables are already present on disk, they don't need to
// make use of a flush iter.
panic("pebble: not implemented")
}
func (s *ingestedFlushable) constructRangeDelIter(
file *manifest.FileMetadata, _ *keyspan.SpanIterOptions,
) (keyspan.FragmentIterator, error) {
// Note that the keyspan level iter expects a non-nil iterator to be
// returned even if there is an error. So, we return the emptyKeyspanIter.
iter, rangeDelIter, err := s.newIters(context.Background(), file, nil, internalIterOpts{})
if err != nil {
return emptyKeyspanIter, err
}
iter.Close()
if rangeDelIter == nil {
return emptyKeyspanIter, nil
}
return rangeDelIter, nil
}
// TODO(bananabrick): Using a level iter instead of a keyspan level iter to
// surface range deletes is more efficient.
func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp,
s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
manifest.KeyTypePoint,
)
}
func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
if !s.containsRangeKeys() {
return nil
}
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters,
s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
)
}
func (s *ingestedFlushable) containsRangeKeys() bool {
return s.hasRangeKeys
}
func (s *ingestedFlushable) inuseBytes() uint64 {
// inuseBytes is only used when memtables are flushed to disk as sstables.
panic("pebble: not implemented")
}
func (s *ingestedFlushable) totalBytes() uint64 {
// We don't allocate additional bytes for the ingestedFlushable.
return 0
}
func (s *ingestedFlushable) readyForFlush() bool {
// ingestedFlushable should always be ready to flush. However, note that
// memtables before the ingested sstables in the memtable queue must be
// flushed before an ingestedFlushable can be flushed. This is because the
// ingested sstables need an updated view of the Version to
// determine where to place the files in the lsm.
return true
}

122
vendor/github.com/cockroachdb/pebble/format_major_version.go generated vendored
View File

@@ -110,18 +110,49 @@ const (
// 23.1 versions.
// FormatPrePebblev1MarkedCompacted is a format major version that
// guarantees that all sstables explicitly marked for compaction in the
// manifest have been compacted. Ratcheting to this format version will block
// (without holding mutexes) until all necessary compactions for files marked
// for compaction are complete.
// FormatUnusedPrePebblev1MarkedCompacted is an unused format major version.
// This format major version was originally intended to ship in the 23.1
// release. It was later decided that this should be deferred until a
// subsequent release. The original ordering is preserved so as not to
// introduce breaking changes in Cockroach.
FormatUnusedPrePebblev1MarkedCompacted
// FormatSSTableValueBlocks is a format major version that adds support for
// storing values in value blocks in the sstable. Value block support is not
// necessarily enabled when writing sstables, when running with this format
// major version.
//
// WARNING: In development, so no production code should upgrade to this
// format, since a DB with this format major version will not actually
// interoperate correctly with another DB with the same format major
// version. This format major version is introduced so that tests can start
// being executed up to this version. Note that these tests succeed despite
// the incomplete support since they do not enable value blocks and use
// TableFormatPebblev2.
FormatSSTableValueBlocks
// FormatFlushableIngest is a format major version that enables lazy
// addition of ingested sstables into the LSM structure. When an ingest
// overlaps with a memtable, a record of the ingest is written to the WAL
// without waiting for a flush. Subsequent reads treat the ingested files as
// a level above the overlapping memtable. Once the memtable is flushed, the
// ingested files are moved into the lowest possible levels.
//
// This feature is behind a format major version because it required
// breaking changes to the WAL format.
FormatFlushableIngest
// 23.2 versions.
// FormatPrePebblev1MarkedCompacted is a format major version that guarantees
// that all sstables explicitly marked for compaction in the manifest (see
// FormatPrePebblev1Marked) have been compacted. Ratcheting to this format
// version will block (without holding mutexes) until all necessary
// compactions for files marked for compaction are complete.
FormatPrePebblev1MarkedCompacted
// FormatNewest always contains the most recent format major version.
// NB: When adding new versions, the MaxTableFormat method should also be
// updated to return the maximum allowable version for the new
// FormatMajorVersion.
FormatNewest FormatMajorVersion = FormatPrePebblev1MarkedCompacted
FormatNewest FormatMajorVersion = iota - 1
)
// MaxTableFormat returns the maximum sstable.TableFormat that can be used at
@@ -135,8 +166,10 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
FormatSplitUserKeysMarkedCompacted:
return sstable.TableFormatPebblev1
case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatPrePebblev1MarkedCompacted:
FormatUnusedPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev2
case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev3
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
}
@@ -152,7 +185,8 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
FormatRangeKeys:
return sstable.TableFormatLevelDB
case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatPrePebblev1MarkedCompacted:
FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev1
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -184,7 +218,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
// guaranteed to exist, because we unconditionally locate it
// during Open.
manifestFileNum := d.mu.versions.manifestFileNum
filename := base.MakeFilename(fileTypeManifest, manifestFileNum)
filename := base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum())
if err := d.mu.versions.manifestMarker.Move(filename); err != nil {
return errors.Wrap(err, "moving manifest marker")
}
@@ -222,7 +256,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
// version that does not know about format major versions
// attempts to open the database, it will error avoiding
// accidental corruption.
if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, 0); err != nil {
if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatVersioned)
@@ -266,6 +300,16 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
},
FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
// Intentional no-op.
return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted)
},
FormatSSTableValueBlocks: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
},
FormatFlushableIngest: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatFlushableIngest)
},
FormatPrePebblev1MarkedCompacted: func(d *DB) error {
// Before finalizing the format major version, rewrite any sstables
// still marked for compaction. Note all format major versions
@@ -476,17 +520,24 @@ var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc {
for l := numLevels - 1; l > 0; l-- {
iter := v.Levels[l].Iter()
for f := iter.First(); f != nil; f = iter.Next() {
err = tc.withReader(f, func(r *sstable.Reader) error {
tf, err := r.TableFormat()
if err != nil {
return err
}
if tf < sstable.TableFormatPebblev1 {
found = true
files[l] = append(files[l], f)
}
return nil
})
if f.Virtual {
// Any physical sstable which has been virtualized must
// have already undergone this migration, and we don't
// need to worry about the virtual sstable themselves.
panic("pebble: unexpected virtual sstable during migration")
}
err = tc.withReader(
f.PhysicalMeta(), func(r *sstable.Reader) error {
tf, err := r.TableFormat()
if err != nil {
return err
}
if tf < sstable.TableFormatPebblev1 {
found = true
files[l] = append(files[l], f)
}
return nil
})
if err != nil {
return
}
@@ -502,31 +553,24 @@ func (d *DB) markFilesLocked(findFn findFilesFunc) error {
jobID := d.mu.nextJobID
d.mu.nextJobID++
vers := d.mu.versions.currentVersion()
// Acquire a read state to have a view of the LSM and a guarantee that none
// of the referenced files will be deleted until we've unreferenced the read
// state. Some findFilesFuncs may read the files, requiring they not be
// deleted.
rs := d.loadReadState()
var (
found bool
files [numLevels][]*fileMetadata
err error
)
func() {
// Disable file deletions. Some migrations that mark files may need to
// read the files themselves. The LSM may be changing while the
// migration is determining which files to mark. That's ok, because
// after re-acquiring the mutex and acquiring the manifest lock, we'll
// discard any sstables that have been removed from the LSM. However, we
// also need to make sure that the files we try to open during the scan
// are not removed before we read them. This is preferable over
// gracefully handling nonexistent files because some environments (eg,
// Windows) error if you attempt to remove an open file.
d.disableFileDeletions()
defer d.enableFileDeletions()
defer rs.unrefLocked()
// Note the unusual locking: unlock, defer Lock(). The scan of the files in
// the version does not need to block other operations that require the
// DB.mu. Drop it for the scan, before re-acquiring it.
d.mu.Unlock()
defer d.mu.Lock()
found, files, err = findFn(vers)
found, files, err = findFn(rs.current)
}()
if err != nil {
return err
@@ -547,7 +591,7 @@ func (d *DB) markFilesLocked(findFn findFilesFunc) error {
// Lock the manifest for a coherent view of the LSM. The database lock has
// been re-acquired by the defer within the above anonymous function.
d.mu.versions.logLock()
vers = d.mu.versions.currentVersion()
vers := d.mu.versions.currentVersion()
for l, filesToMark := range files {
if len(filesToMark) == 0 {
continue

9
vendor/github.com/cockroachdb/pebble/get_iter.go generated vendored
View File

@@ -5,6 +5,7 @@
package pebble
import (
"context"
"fmt"
"github.com/cockroachdb/pebble/internal/base"
@@ -158,7 +159,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
files := g.l0[n-1].Iter()
g.l0 = g.l0[:n-1]
iterOpts := IterOptions{logger: g.logger}
g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters,
g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
files, manifest.L0Sublevel(n), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
g.iter = &g.levelIter
@@ -177,7 +178,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
}
iterOpts := IterOptions{logger: g.logger}
g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters,
g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
g.level++
@@ -190,6 +191,10 @@ func (g *getIter) Prev() (*InternalKey, base.LazyValue) {
panic("pebble: Prev unimplemented")
}
func (g *getIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) {
panic("pebble: NextPrefix unimplemented")
}
func (g *getIter) Valid() bool {
return g.iterKey != nil && g.err == nil
}

540
vendor/github.com/cockroachdb/pebble/ingest.go generated vendored
View File

@@ -5,6 +5,7 @@
package pebble
import (
"context"
"sort"
"time"
@@ -13,8 +14,8 @@ import (
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/internal/private"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
)
func sstableKeyCompare(userCmp Compare, a, b InternalKey) int {
@@ -22,12 +23,12 @@ func sstableKeyCompare(userCmp Compare, a, b InternalKey) int {
if c != 0 {
return c
}
if a.Trailer == InternalKeyRangeDeleteSentinel {
if b.Trailer != InternalKeyRangeDeleteSentinel {
if a.IsExclusiveSentinel() {
if !b.IsExclusiveSentinel() {
return -1
}
} else if b.Trailer == InternalKeyRangeDeleteSentinel {
return 1
} else if b.IsExclusiveSentinel() {
return +1
}
return 0
}
@@ -45,20 +46,20 @@ func ingestValidateKey(opts *Options, key *InternalKey) error {
}
func ingestLoad1(
opts *Options, fmv FormatMajorVersion, path string, cacheID uint64, fileNum FileNum,
opts *Options, fmv FormatMajorVersion, path string, cacheID uint64, fileNum base.DiskFileNum,
) (*fileMetadata, error) {
stat, err := opts.FS.Stat(path)
if err != nil {
return nil, err
}
f, err := opts.FS.Open(path)
if err != nil {
return nil, err
}
readable, err := sstable.NewSimpleReadable(f)
if err != nil {
return nil, err
}
cacheOpts := private.SSTableCacheOpts(cacheID, fileNum).(sstable.ReaderOption)
r, err := sstable.NewReader(f, opts.MakeReaderOptions(), cacheOpts)
r, err := sstable.NewReader(readable, opts.MakeReaderOptions(), cacheOpts)
if err != nil {
return nil, err
}
@@ -77,9 +78,10 @@ func ingestLoad1(
}
meta := &fileMetadata{}
meta.FileNum = fileNum
meta.Size = uint64(stat.Size())
meta.FileNum = fileNum.FileNum()
meta.Size = uint64(readable.Size())
meta.CreationTime = time.Now().Unix()
meta.InitPhysicalBacking()
// Avoid loading into the table cache for collecting stats if we
// don't need to. If there are no range deletions, we have all the
@@ -90,7 +92,7 @@ func ingestLoad1(
// disallowing removal of an open file. Under MemFS, if we don't populate
// meta.Stats here, the file will be loaded into the table cache for
// calculating stats before we can remove the original link.
maybeSetStatsFromProperties(meta, &r.Properties)
maybeSetStatsFromProperties(meta.PhysicalMeta(), &r.Properties)
{
iter, err := r.NewIter(nil /* lower */, nil /* upper */)
@@ -194,7 +196,7 @@ func ingestLoad1(
}
func ingestLoad(
opts *Options, fmv FormatMajorVersion, paths []string, cacheID uint64, pending []FileNum,
opts *Options, fmv FormatMajorVersion, paths []string, cacheID uint64, pending []base.DiskFileNum,
) ([]*fileMetadata, []string, error) {
meta := make([]*fileMetadata, 0, len(paths))
newPaths := make([]string, 0, len(paths))
@@ -252,54 +254,28 @@ func ingestSortAndVerify(cmp Compare, meta []*fileMetadata, paths []string) erro
return nil
}
func ingestCleanup(fs vfs.FS, dirname string, meta []*fileMetadata) error {
func ingestCleanup(objProvider objstorage.Provider, meta []*fileMetadata) error {
var firstErr error
for i := range meta {
target := base.MakeFilepath(fs, dirname, fileTypeTable, meta[i].FileNum)
if err := fs.Remove(target); err != nil {
if err := objProvider.Remove(fileTypeTable, meta[i].FileBacking.DiskFileNum); err != nil {
firstErr = firstError(firstErr, err)
}
}
return firstErr
}
// ingestLink creates new objects which are backed by either hardlinks to or
// copies of the ingested files.
func ingestLink(
jobID int, opts *Options, dirname string, paths []string, meta []*fileMetadata,
jobID int, opts *Options, objProvider objstorage.Provider, paths []string, meta []*fileMetadata,
) error {
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := syncingFS{
FS: opts.FS,
syncOpts: vfs.SyncingFileOptions{
NoSyncOnClose: opts.NoSyncOnClose,
BytesPerSync: opts.BytesPerSync,
},
}
for i := range paths {
target := base.MakeFilepath(fs, dirname, fileTypeTable, meta[i].FileNum)
var err error
if _, ok := opts.FS.(*vfs.MemFS); ok && opts.DebugCheck != nil {
// The combination of MemFS+Ingest+DebugCheck produces awkwardness around
// the subsequent deletion of files. The problem is that MemFS implements
// the Windows semantics of disallowing removal of an open file. This is
// desirable because it helps catch bugs where we violate the
// requirements of the Windows semantics. The normal practice for Ingest
// is for the caller to remove the source files after the ingest
// completes successfully. Unfortunately, Options.DebugCheck causes
// ingest to run DB.CheckLevels() before the ingest finishes, and
// DB.CheckLevels() populates the table cache with the newly ingested
// files.
//
// The combination of MemFS+Ingest+DebugCheck is primarily used in
// tests. As a workaround, disable hard linking this combination
// occurs. See https://github.com/cockroachdb/pebble/issues/495.
err = vfs.Copy(fs, paths[i], target)
} else {
err = vfs.LinkOrCopy(fs, paths[i], target)
}
objMeta, err := objProvider.LinkOrCopyFromLocal(
context.TODO(), opts.FS, paths[i], fileTypeTable, meta[i].FileBacking.DiskFileNum,
objstorage.CreateOptions{PreferSharedStorage: true},
)
if err != nil {
if err2 := ingestCleanup(fs, dirname, meta[:i]); err2 != nil {
if err2 := ingestCleanup(objProvider, meta[:i]); err2 != nil {
opts.Logger.Infof("ingest cleanup failed: %v", err2)
}
return err
@@ -308,7 +284,7 @@ func ingestLink(
opts.EventListener.TableCreated(TableCreateInfo{
JobID: jobID,
Reason: "ingesting",
Path: target,
Path: objProvider.Path(objMeta),
FileNum: meta[i].FileNum,
})
}
@@ -320,18 +296,28 @@ func ingestLink(
func ingestMemtableOverlaps(cmp Compare, mem flushable, meta []*fileMetadata) bool {
iter := mem.newIter(nil)
rangeDelIter := mem.newRangeDelIter(nil)
defer iter.Close()
rkeyIter := mem.newRangeKeyIter(nil)
if rangeDelIter != nil {
defer rangeDelIter.Close()
closeIters := func() error {
err := iter.Close()
if rangeDelIter != nil {
err = firstError(err, rangeDelIter.Close())
}
if rkeyIter != nil {
err = firstError(err, rkeyIter.Close())
}
return err
}
for _, m := range meta {
if overlapWithIterator(iter, &rangeDelIter, m, cmp) {
if overlapWithIterator(iter, &rangeDelIter, rkeyIter, m, cmp) {
closeIters()
return true
}
}
return false
// Assume overlap if any iterator errored out.
return closeIters() != nil
}
func ingestUpdateSeqNum(
@@ -377,7 +363,11 @@ func ingestUpdateSeqNum(
}
func overlapWithIterator(
iter internalIterator, rangeDelIter *keyspan.FragmentIterator, meta *fileMetadata, cmp Compare,
iter internalIterator,
rangeDelIter *keyspan.FragmentIterator,
rkeyIter keyspan.FragmentIterator,
meta *fileMetadata,
cmp Compare,
) bool {
// Check overlap with point operations.
//
@@ -405,36 +395,60 @@ func overlapWithIterator(
return true
}
}
// Assume overlap if iterator errored.
if err := iter.Error(); err != nil {
return true
}
computeOverlapWithSpans := func(rIter keyspan.FragmentIterator) bool {
// NB: The spans surfaced by the fragment iterator are non-overlapping.
span := rIter.SeekLT(meta.Smallest.UserKey)
if span == nil {
span = rIter.Next()
}
for ; span != nil; span = rIter.Next() {
if span.Empty() {
continue
}
key := span.SmallestKey()
c := sstableKeyCompare(cmp, key, meta.Largest)
if c > 0 {
// The start of the span is after the largest key in the
// ingested table.
return false
}
if cmp(span.End, meta.Smallest.UserKey) > 0 {
// The end of the span is greater than the smallest in the
// table. Note that the span end key is exclusive, thus ">0"
// instead of ">=0".
return true
}
}
// Assume overlap if iterator errored.
if err := rIter.Error(); err != nil {
return true
}
return false
}
// rkeyIter is either a range key level iter, or a range key iterator
// over a single file.
if rkeyIter != nil {
if computeOverlapWithSpans(rkeyIter) {
return true
}
}
// Check overlap with range deletions.
if rangeDelIter == nil || *rangeDelIter == nil {
return false
}
rangeDelItr := *rangeDelIter
rangeDel := rangeDelItr.SeekLT(meta.Smallest.UserKey)
if rangeDel == nil {
rangeDel = rangeDelItr.Next()
}
for ; rangeDel != nil; rangeDel = rangeDelItr.Next() {
key := rangeDel.SmallestKey()
c := sstableKeyCompare(cmp, key, meta.Largest)
if c > 0 {
// The start of the tombstone is after the largest key in the
// ingested table.
return false
}
if cmp(rangeDel.End, meta.Smallest.UserKey) > 0 {
// The end of the tombstone is greater than the smallest in the
// table. Note that the tombstone end key is exclusive, thus ">0"
// instead of ">=0".
return true
}
}
return false
return computeOverlapWithSpans(*rangeDelIter)
}
func ingestTargetLevel(
newIters tableNewIters,
newRangeKeyIter keyspan.TableNewSpanIter,
iterOps IterOptions,
cmp Compare,
v *version,
@@ -502,24 +516,33 @@ func ingestTargetLevel(
targetLevel := 0
// Do we overlap with keys in L0?
iter := v.Levels[0].Iter()
for meta0 := iter.First(); meta0 != nil; meta0 = iter.Next() {
c1 := sstableKeyCompare(cmp, meta.Smallest, meta0.Largest)
c2 := sstableKeyCompare(cmp, meta.Largest, meta0.Smallest)
if c1 > 0 || c2 < 0 {
continue
}
// This assertion implicitly checks that we have the current version of
// the metadata.
if v.L0Sublevels == nil {
return 0, errors.AssertionFailedf("could not read L0 sublevels")
}
// Check for overlap over the keys of L0 by iterating over the sublevels.
for subLevel := 0; subLevel < len(v.L0SublevelFiles); subLevel++ {
iter := newLevelIter(iterOps, cmp, nil /* split */, newIters,
v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), nil)
iter, rangeDelIter, err := newIters(iter.Current(), nil, internalIterOpts{})
var rangeDelIter keyspan.FragmentIterator
// Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE
// sets it up for the target file.
iter.initRangeDel(&rangeDelIter)
levelIter := keyspan.LevelIter{}
levelIter.Init(
keyspan.SpanIterOptions{}, cmp, newRangeKeyIter,
v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), manifest.KeyTypeRange,
)
overlap := overlapWithIterator(iter, &rangeDelIter, &levelIter, meta, cmp)
err := iter.Close() // Closes range del iter as well.
err = firstError(err, levelIter.Close())
if err != nil {
return 0, err
}
overlap := overlapWithIterator(iter, &rangeDelIter, meta, cmp)
iter.Close()
if rangeDelIter != nil {
rangeDelIter.Close()
}
if overlap {
return targetLevel, nil
}
@@ -533,8 +556,19 @@ func ingestTargetLevel(
// Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE
// sets it up for the target file.
levelIter.initRangeDel(&rangeDelIter)
overlap := overlapWithIterator(levelIter, &rangeDelIter, meta, cmp)
levelIter.Close() // Closes range del iter as well.
rkeyLevelIter := &keyspan.LevelIter{}
rkeyLevelIter.Init(
keyspan.SpanIterOptions{}, cmp, newRangeKeyIter,
v.Levels[level].Iter(), manifest.Level(level), manifest.KeyTypeRange,
)
overlap := overlapWithIterator(levelIter, &rangeDelIter, rkeyLevelIter, meta, cmp)
err := levelIter.Close() // Closes range del iter as well.
err = firstError(err, rkeyLevelIter.Close())
if err != nil {
return 0, err
}
if overlap {
return targetLevel, nil
}
@@ -593,18 +627,18 @@ func ingestTargetLevel(
//
// The steps for ingestion are:
//
// 1. Allocate file numbers for every sstable being ingested.
// 2. Load the metadata for all sstables being ingest.
// 3. Sort the sstables by smallest key, verifying non overlap.
// 4. Hard link (or copy) the sstables into the DB directory.
// 5. Allocate a sequence number to use for all of the entries in the
// sstables. This is the step where overlap with memtables is
// determined. If there is overlap, we remember the most recent memtable
// that overlaps.
// 6. Update the sequence number in the ingested sstables.
// 7. Wait for the most recent memtable that overlaps to flush (if any).
// 8. Add the ingested sstables to the version (DB.ingestApply).
// 9. Publish the ingestion sequence number.
// 1. Allocate file numbers for every sstable being ingested.
// 2. Load the metadata for all sstables being ingest.
// 3. Sort the sstables by smallest key, verifying non overlap.
// 4. Hard link (or copy) the sstables into the DB directory.
// 5. Allocate a sequence number to use for all of the entries in the
// sstables. This is the step where overlap with memtables is
// determined. If there is overlap, we remember the most recent memtable
// that overlaps.
// 6. Update the sequence number in the ingested sstables.
// 7. Wait for the most recent memtable that overlaps to flush (if any).
// 8. Add the ingested sstables to the version (DB.ingestApply).
// 9. Publish the ingestion sequence number.
//
// Note that if the mutable memtable overlaps with ingestion, a flush of the
// memtable is forced equivalent to DB.Flush. Additionally, subsequent
@@ -630,11 +664,14 @@ type IngestOperationStats struct {
// Bytes is the total bytes in the ingested sstables.
Bytes uint64
// ApproxIngestedIntoL0Bytes is the approximate number of bytes ingested
// into L0.
// Currently, this value is completely accurate, but we are allowing this to
// be approximate once https://github.com/cockroachdb/pebble/issues/25 is
// implemented.
// into L0. This value is approximate when flushable ingests are active and
// an ingest overlaps an entry in the flushable queue. Currently, this
// approximation is very rough, only including tables that overlapped the
// memtable. This estimate may be improved with #2112.
ApproxIngestedIntoL0Bytes uint64
// MemtableOverlappingFiles is the count of ingested sstables
// that overlapped keys in the memtables.
MemtableOverlappingFiles int
}
// IngestWithStats does the same as Ingest, and additionally returns
@@ -649,6 +686,113 @@ func (d *DB) IngestWithStats(paths []string) (IngestOperationStats, error) {
return d.ingest(paths, ingestTargetLevel)
}
// Both DB.mu and commitPipeline.mu must be held while this is called.
func (d *DB) newIngestedFlushableEntry(
meta []*fileMetadata, seqNum uint64, logNum FileNum,
) (*flushableEntry, error) {
// Update the sequence number for all of the sstables in the
// metadata. Writing the metadata to the manifest when the
// version edit is applied is the mechanism that persists the
// sequence number. The sstables themselves are left unmodified.
// In this case, a version edit will only be written to the manifest
// when the flushable is eventually flushed. If Pebble restarts in that
// time, then we'll lose the ingest sequence number information. But this
// information will also be reconstructed on node restart.
if err := ingestUpdateSeqNum(
d.cmp, d.opts.Comparer.FormatKey, seqNum, meta,
); err != nil {
return nil, err
}
f := newIngestedFlushable(meta, d.cmp, d.split, d.newIters, d.tableNewRangeKeyIter)
// NB: The logNum/seqNum are the WAL number which we're writing this entry
// to and the sequence number within the WAL which we'll write this entry
// to.
entry := d.newFlushableEntry(f, logNum, seqNum)
// The flushable entry starts off with a single reader ref, so increment
// the FileMetadata.Refs.
for _, file := range f.files {
file.Ref()
}
entry.unrefFiles = func() []*fileBacking {
var obsolete []*fileBacking
for _, file := range f.files {
if file.Unref() == 0 {
obsolete = append(obsolete, file.FileMetadata.FileBacking)
}
}
return obsolete
}
entry.flushForced = true
entry.releaseMemAccounting = func() {}
return entry, nil
}
// Both DB.mu and commitPipeline.mu must be held while this is called. Since
// we're holding both locks, the order in which we rotate the memtable or
// recycle the WAL in this function is irrelevant as long as the correct log
// numbers are assigned to the appropriate flushable.
func (d *DB) handleIngestAsFlushable(meta []*fileMetadata, seqNum uint64) error {
b := d.NewBatch()
for _, m := range meta {
b.ingestSST(m.FileNum)
}
b.setSeqNum(seqNum)
// If the WAL is disabled, then the logNum used to create the flushable
// entry doesn't matter. We just use the logNum assigned to the current
// mutable memtable. If the WAL is enabled, then this logNum will be
// overwritten by the logNum of the log which will contain the log entry
// for the ingestedFlushable.
logNum := d.mu.mem.queue[len(d.mu.mem.queue)-1].logNum
if !d.opts.DisableWAL {
// We create a new WAL for the flushable instead of reusing the end of
// the previous WAL. This simplifies the increment of the minimum
// unflushed log number, and also simplifies WAL replay.
logNum, _ = d.recycleWAL()
d.mu.Unlock()
err := d.commit.directWrite(b)
if err != nil {
d.opts.Logger.Fatalf("%v", err)
}
d.mu.Lock()
}
entry, err := d.newIngestedFlushableEntry(meta, seqNum, logNum)
if err != nil {
return err
}
nextSeqNum := seqNum + uint64(b.Count())
// Set newLogNum to the logNum of the previous flushable. This value is
// irrelevant if the WAL is disabled. If the WAL is enabled, then we set
// the appropriate value below.
newLogNum := d.mu.mem.queue[len(d.mu.mem.queue)-1].logNum
if !d.opts.DisableWAL {
// This is WAL num of the next mutable memtable which comes after the
// ingestedFlushable in the flushable queue. The mutable memtable
// will be created below.
newLogNum, _ = d.recycleWAL()
if err != nil {
return err
}
}
currMem := d.mu.mem.mutable
// NB: Placing ingested sstables above the current memtables
// requires rotating of the existing memtables/WAL. There is
// some concern of churning through tiny memtables due to
// ingested sstables being placed on top of them, but those
// memtables would have to be flushed anyways.
d.mu.mem.queue = append(d.mu.mem.queue, entry)
d.rotateMemtable(newLogNum, nextSeqNum, currMem)
d.updateReadStateLocked(d.opts.DebugCheck)
d.maybeScheduleFlush()
return nil
}
func (d *DB) ingest(
paths []string, targetLevelFunc ingestTargetLevelFunc,
) (IngestOperationStats, error) {
@@ -658,9 +802,9 @@ func (d *DB) ingest(
// ordering. The sorting of L0 tables by sequence number avoids relying on
// that (busted) invariant.
d.mu.Lock()
pendingOutputs := make([]FileNum, len(paths))
pendingOutputs := make([]base.DiskFileNum, len(paths))
for i := range paths {
pendingOutputs[i] = d.mu.versions.getNextFileNum()
pendingOutputs[i] = d.mu.versions.getNextFileNum().DiskFileNum()
}
jobID := d.mu.nextJobID
d.mu.nextJobID++
@@ -687,19 +831,25 @@ func (d *DB) ingest(
// (e.g. because the files reside on a different filesystem), ingestLink will
// fall back to copying, and if that fails we undo our work and return an
// error.
if err := ingestLink(jobID, d.opts, d.dirname, paths, meta); err != nil {
if err := ingestLink(jobID, d.opts, d.objProvider, paths, meta); err != nil {
return IngestOperationStats{}, err
}
// Fsync the directory we added the tables to. We need to do this at some
// point before we update the MANIFEST (via logAndApply), otherwise a crash
// can have the tables referenced in the MANIFEST, but not present in the
// directory.
if err := d.dataDir.Sync(); err != nil {
// Make the new tables durable. We need to do this at some point before we
// update the MANIFEST (via logAndApply), otherwise a crash can have the
// tables referenced in the MANIFEST, but not present in the provider.
if err := d.objProvider.Sync(); err != nil {
return IngestOperationStats{}, err
}
// metaFlushableOverlaps is a slice parallel to meta indicating which of the
// ingested sstables overlap some table in the flushable queue. It's used to
// approximate ingest-into-L0 stats when using flushable ingests.
metaFlushableOverlaps := make([]bool, len(meta))
var mem *flushableEntry
prepare := func() {
var mut *memTable
// asFlushable indicates whether the sstable was ingested as a flushable.
var asFlushable bool
prepare := func(seqNum uint64) {
// Note that d.commit.mu is held by commitPipeline when calling prepare.
d.mu.Lock()
@@ -709,24 +859,89 @@ func (d *DB) ingest(
// is ordered from oldest to newest with the mutable memtable being the
// last element in the slice. We want to wait for the newest table that
// overlaps.
for i := len(d.mu.mem.queue) - 1; i >= 0; i-- {
m := d.mu.mem.queue[i]
if ingestMemtableOverlaps(d.cmp, m, meta) {
mem = m
if mem.flushable == d.mu.mem.mutable {
err = d.makeRoomForWrite(nil)
iter := m.newIter(nil)
rangeDelIter := m.newRangeDelIter(nil)
rkeyIter := m.newRangeKeyIter(nil)
for i := range meta {
if metaFlushableOverlaps[i] {
// This table already overlapped a more recent flushable.
continue
}
mem.flushForced = true
d.maybeScheduleFlush()
return
if overlapWithIterator(iter, &rangeDelIter, rkeyIter, meta[i], d.cmp) {
// If this is the first table to overlap a flushable, save
// the flushable. This ingest must be ingested or flushed
// after it.
if mem == nil {
mem = m
}
metaFlushableOverlaps[i] = true
}
}
err := iter.Close()
if rangeDelIter != nil {
err = firstError(err, rangeDelIter.Close())
}
if rkeyIter != nil {
err = firstError(err, rkeyIter.Close())
}
if err != nil {
d.opts.Logger.Infof("ingest error reading flushable for log %s: %s", m.logNum, err)
}
}
if mem == nil {
// No overlap with any of the queued flushables, so no need to queue
// after them.
// New writes with higher sequence numbers may be concurrently
// committed. We must ensure they don't flush before this ingest
// completes. To do that, we ref the mutable memtable as a writer,
// preventing its flushing (and the flushing of all subsequent
// flushables in the queue). Once we've acquired the manifest lock
// to add the ingested sstables to the LSM, we can unref as we're
// guaranteed that the flush won't edit the LSM before this ingest.
mut = d.mu.mem.mutable
mut.writerRef()
return
}
// The ingestion overlaps with some entry in the flushable queue.
if d.mu.formatVers.vers < FormatFlushableIngest ||
d.opts.Experimental.DisableIngestAsFlushable() ||
(len(d.mu.mem.queue) > d.opts.MemTableStopWritesThreshold-1) {
// We're not able to ingest as a flushable,
// so we must synchronously flush.
if mem.flushable == d.mu.mem.mutable {
err = d.makeRoomForWrite(nil)
}
// New writes with higher sequence numbers may be concurrently
// committed. We must ensure they don't flush before this ingest
// completes. To do that, we ref the mutable memtable as a writer,
// preventing its flushing (and the flushing of all subsequent
// flushables in the queue). Once we've acquired the manifest lock
// to add the ingested sstables to the LSM, we can unref as we're
// guaranteed that the flush won't edit the LSM before this ingest.
mut = d.mu.mem.mutable
mut.writerRef()
mem.flushForced = true
d.maybeScheduleFlush()
return
}
// Since there aren't too many memtables already queued up, we can
// slide the ingested sstables on top of the existing memtables.
asFlushable = true
err = d.handleIngestAsFlushable(meta, seqNum)
}
var ve *versionEdit
apply := func(seqNum uint64) {
if err != nil {
if err != nil || asFlushable {
// An error occurred during prepare.
if mut != nil {
mut.writerUnref()
}
return
}
@@ -737,6 +952,9 @@ func (d *DB) ingest(
if err = ingestUpdateSeqNum(
d.cmp, d.opts.Comparer.FormatKey, seqNum, meta,
); err != nil {
if mut != nil {
mut.writerUnref()
}
return
}
@@ -748,16 +966,18 @@ func (d *DB) ingest(
// Assign the sstables to the correct level in the LSM and apply the
// version edit.
ve, err = d.ingestApply(jobID, meta, targetLevelFunc)
ve, err = d.ingestApply(jobID, meta, targetLevelFunc, mut)
}
d.commit.AllocateSeqNum(len(meta), prepare, apply)
if err != nil {
if err2 := ingestCleanup(d.opts.FS, d.dirname, meta); err2 != nil {
if err2 := ingestCleanup(d.objProvider, meta); err2 != nil {
d.opts.Logger.Infof("ingest cleanup failed: %v", err2)
}
} else {
// Since we either created a hard link to the ingesting files, or copied
// them over, it is safe to remove the originals paths.
for _, path := range paths {
if err2 := d.opts.FS.Remove(path); err2 != nil {
d.opts.Logger.Infof("ingest failed to remove original file: %s", err2)
@@ -769,6 +989,7 @@ func (d *DB) ingest(
JobID: jobID,
GlobalSeqNum: meta[0].SmallestSeqNum,
Err: err,
flushable: asFlushable,
}
var stats IngestOperationStats
if ve != nil {
@@ -784,6 +1005,31 @@ func (d *DB) ingest(
if e.Level == 0 {
stats.ApproxIngestedIntoL0Bytes += e.Meta.Size
}
if metaFlushableOverlaps[i] {
stats.MemtableOverlappingFiles++
}
}
} else if asFlushable {
info.Tables = make([]struct {
TableInfo
Level int
}, len(meta))
for i, f := range meta {
info.Tables[i].Level = -1
info.Tables[i].TableInfo = f.TableInfo()
stats.Bytes += f.Size
// We don't have exact stats on which files will be ingested into
// L0, because actual ingestion into the LSM has been deferred until
// flush time. Instead, we infer based on memtable overlap.
//
// TODO(jackson): If we optimistically compute data overlap (#2112)
// before entering the commit pipeline, we can use that overlap to
// improve our approximation by incorporating overlap with L0, not
// just memtables.
if metaFlushableOverlaps[i] {
stats.ApproxIngestedIntoL0Bytes += f.Size
stats.MemtableOverlappingFiles++
}
}
}
d.opts.EventListener.TableIngested(info)
@@ -793,6 +1039,7 @@ func (d *DB) ingest(
type ingestTargetLevelFunc func(
newIters tableNewIters,
newRangeKeyIter keyspan.TableNewSpanIter,
iterOps IterOptions,
cmp Compare,
v *version,
@@ -802,7 +1049,7 @@ type ingestTargetLevelFunc func(
) (int, error)
func (d *DB) ingestApply(
jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc,
jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc, mut *memTable,
) (*versionEdit, error) {
d.mu.Lock()
defer d.mu.Unlock()
@@ -819,6 +1066,16 @@ func (d *DB) ingestApply(
// logAndApply unconditionally releases the manifest lock, but any earlier
// returns must unlock the manifest.
d.mu.versions.logLock()
if mut != nil {
// Unref the mutable memtable to allows its flush to proceed. Now that we've
// acquired the manifest lock, we can be certain that if the mutable
// memtable has received more recent conflicting writes, the flush won't
// beat us to applying to the manifest resulting in sequence number
// inversion.
mut.writerUnref()
}
current := d.mu.versions.currentVersion()
baseLevel := d.mu.versions.picker.getBaseLevel()
iterOps := IterOptions{logger: d.opts.Logger}
@@ -828,7 +1085,7 @@ func (d *DB) ingestApply(
m := meta[i]
f := &ve.NewFiles[i]
var err error
f.Level, err = findTargetLevel(d.newIters, iterOps, d.cmp, current, baseLevel, d.mu.compact.inProgress, m)
f.Level, err = findTargetLevel(d.newIters, d.tableNewRangeKeyIter, iterOps, d.cmp, current, baseLevel, d.mu.compact.inProgress, m)
if err != nil {
d.mu.versions.logUnlock()
return nil, err
@@ -851,7 +1108,6 @@ func (d *DB) ingestApply(
}
d.updateReadStateLocked(d.opts.DebugCheck)
d.updateTableStatsLocked(ve.NewFiles)
d.deleteObsoleteFiles(jobID, false /* waitForOngoing */)
// The ingestion may have pushed a level over the threshold for compaction,
// so check to see if one is necessary and schedule it.
d.maybeScheduleCompaction()
@@ -862,12 +1118,21 @@ func (d *DB) ingestApply(
// maybeValidateSSTablesLocked adds the slice of newFileEntrys to the pending
// queue of files to be validated, when the feature is enabled.
// DB.mu must be locked when calling.
//
// TODO(bananabrick): Make sure that the ingestion step only passes in the
// physical sstables for validation here.
func (d *DB) maybeValidateSSTablesLocked(newFiles []newFileEntry) {
// Only add to the validation queue when the feature is enabled.
if !d.opts.Experimental.ValidateOnIngest {
return
}
for _, f := range newFiles {
if f.Meta.Virtual {
panic("pebble: invalid call to maybeValidateSSTablesLocked")
}
}
d.mu.tableValidation.pending = append(d.mu.tableValidation.pending, newFiles...)
if d.shouldValidateSSTablesLocked() {
go d.validateSSTables()
@@ -927,9 +1192,10 @@ func (d *DB) validateSSTables() {
}
}
err := d.tableCache.withReader(f.Meta, func(r *sstable.Reader) error {
return r.ValidateBlockChecksums()
})
err := d.tableCache.withReader(
f.Meta.PhysicalMeta(), func(r *sstable.Reader) error {
return r.ValidateBlockChecksums()
})
if err != nil {
// TODO(travers): Hook into the corruption reporting pipeline, once
// available. See pebble#1192.

12
vendor/github.com/cockroachdb/pebble/internal.go generated vendored
View File

@@ -22,6 +22,7 @@ const (
InternalKeyKindRangeKeySet = base.InternalKeyKindRangeKeySet
InternalKeyKindRangeKeyUnset = base.InternalKeyKindRangeKeyUnset
InternalKeyKindRangeKeyDelete = base.InternalKeyKindRangeKeyDelete
InternalKeyKindIngestSST = base.InternalKeyKindIngestSST
InternalKeyKindInvalid = base.InternalKeyKindInvalid
InternalKeySeqNumBatch = base.InternalKeySeqNumBatch
InternalKeySeqNumMax = base.InternalKeySeqNumMax
@@ -36,3 +37,14 @@ type internalIterator = base.InternalIterator
// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST,
// sstable) isn't in the expected format.
var ErrCorruption = base.ErrCorruption
// AttributeAndLen exports the base.AttributeAndLen type.
type AttributeAndLen = base.AttributeAndLen
// ShortAttribute exports the base.ShortAttribute type.
type ShortAttribute = base.ShortAttribute
// LazyFetcher exports the base.LazyFetcher type. This export is needed since
// LazyValue.Clone requires a pointer to a LazyFetcher struct to avoid
// allocations. No code outside Pebble needs to peer into a LazyFetcher.
type LazyFetcher = base.LazyFetcher

4
vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go generated vendored
View File

@@ -79,6 +79,10 @@ func (it *flushIterator) Next() (*base.InternalKey, base.LazyValue) {
return &it.key, base.MakeInPlaceValue(it.value())
}
func (it *flushIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
panic("pebble: NextPrefix unimplemented")
}
func (it *flushIterator) Prev() (*base.InternalKey, base.LazyValue) {
panic("pebble: Prev unimplemented")
}

7
vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go generated vendored
View File

@@ -194,6 +194,13 @@ func (it *Iterator) Next() (*base.InternalKey, base.LazyValue) {
return &it.key, base.MakeInPlaceValue(it.value())
}
// NextPrefix advances to the next position with a new prefix. Returns the key
// and value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise.
func (it *Iterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
return it.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext())
}
// Prev moves to the previous position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
func (it *Iterator) Prev() (*base.InternalKey, base.LazyValue) {

2
vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go generated vendored
View File

@@ -68,7 +68,7 @@ const (
// keys.
var ErrRecordExists = errors.New("record with this key already exists")
// Skiplist is a fast, cocnurrent skiplist implementation that supports forward
// Skiplist is a fast, concurrent skiplist implementation that supports forward
// and backward iteration. See batchskl.Skiplist for a non-concurrent
// skiplist. Keys and values are immutable once added to the skiplist and
// deletion is not supported. Instead, higher-level code is expected to add new

25
vendor/github.com/cockroachdb/pebble/internal/base/comparer.go generated vendored
View File

@@ -70,11 +70,11 @@ type Successor func(dst, a []byte) []byte
// returns the smallest key that is larger than the given prefix a.
// ImmediateSuccessor must return a prefix key k such that:
//
// Split(k) == len(k) and Compare(k, a) > 0
// Split(k) == len(k) and Compare(k, a) > 0
//
// and there exists no representable k2 such that:
//
// Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
// Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
//
// As an example, an implementation built on the natural byte ordering using
// bytes.Compare could append a `\0` to `a`.
@@ -98,24 +98,23 @@ type ImmediateSuccessor func(dst, a []byte) []byte
//
// The returned prefix must have the following properties:
//
// 1) The prefix must be a byte prefix:
// 1. The prefix must be a byte prefix:
//
// bytes.HasPrefix(a, prefix(a))
// bytes.HasPrefix(a, prefix(a))
//
// 2) A key consisting of just a prefix must sort before all other keys with
// that prefix:
// 2. A key consisting of just a prefix must sort before all other keys with
// that prefix:
//
// Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
// Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
//
// 3) Prefixes must be used to order keys before suffixes:
// 3. Prefixes must be used to order keys before suffixes:
//
// If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
// If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
//
// 4) Suffixes themselves must be valid keys and comparable, respecting the same
// ordering as within a key.
//
// If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
// 4. Suffixes themselves must be valid keys and comparable, respecting the same
// ordering as within a key.
//
// If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
type Split func(a []byte) int
// Comparer defines a total ordering over the space of []byte keys: a 'less

79
vendor/github.com/cockroachdb/pebble/internal/base/filenames.go generated vendored
View File

@@ -20,6 +20,31 @@ type FileNum uint64
// String returns a string representation of the file number.
func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
// called if the caller can ensure that the FileNum belongs to a physical file
// on disk. These could be manifests, log files, physical sstables on disk, the
// options file, but not virtual sstables.
func (fn FileNum) DiskFileNum() DiskFileNum {
return DiskFileNum{fn}
}
// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
// Note that a FileNum is an internal DB identifier and it could belong to files
// which don't exist on disk. An example would be virtual sstable FileNums.
// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
// FileNum to DiskFileNum may not be valid and care should be taken to prove
// that the FileNum actually exists on disk.
type DiskFileNum struct {
fn FileNum
}
func (dfn DiskFileNum) String() string { return dfn.fn.String() }
// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
func (dfn DiskFileNum) FileNum() FileNum {
return dfn.fn
}
// FileType enumerates the types of files found in a DB.
type FileType int
@@ -36,92 +61,92 @@ const (
)
// MakeFilename builds a filename from components.
func MakeFilename(fileType FileType, fileNum FileNum) string {
func MakeFilename(fileType FileType, dfn DiskFileNum) string {
switch fileType {
case FileTypeLog:
return fmt.Sprintf("%s.log", fileNum)
return fmt.Sprintf("%s.log", dfn)
case FileTypeLock:
return "LOCK"
case FileTypeTable:
return fmt.Sprintf("%s.sst", fileNum)
return fmt.Sprintf("%s.sst", dfn)
case FileTypeManifest:
return fmt.Sprintf("MANIFEST-%s", fileNum)
return fmt.Sprintf("MANIFEST-%s", dfn)
case FileTypeCurrent:
return "CURRENT"
case FileTypeOptions:
return fmt.Sprintf("OPTIONS-%s", fileNum)
return fmt.Sprintf("OPTIONS-%s", dfn)
case FileTypeOldTemp:
return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum)
return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
case FileTypeTemp:
return fmt.Sprintf("temporary.%s.dbtmp", fileNum)
return fmt.Sprintf("temporary.%s.dbtmp", dfn)
}
panic("unreachable")
}
// MakeFilepath builds a filepath from components.
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, fileNum))
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
}
// ParseFilename parses the components from a filename.
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) {
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
filename = fs.PathBase(filename)
switch {
case filename == "CURRENT":
return FileTypeCurrent, 0, true
return FileTypeCurrent, DiskFileNum{0}, true
case filename == "LOCK":
return FileTypeLock, 0, true
return FileTypeLock, DiskFileNum{0}, true
case strings.HasPrefix(filename, "MANIFEST-"):
fileNum, ok = parseFileNum(filename[len("MANIFEST-"):])
dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
if !ok {
break
}
return FileTypeManifest, fileNum, true
return FileTypeManifest, dfn, true
case strings.HasPrefix(filename, "OPTIONS-"):
fileNum, ok = parseFileNum(filename[len("OPTIONS-"):])
dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
if !ok {
break
}
return FileTypeOptions, fileNum, ok
return FileTypeOptions, dfn, ok
case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeOldTemp, fileNum, ok
return FileTypeOldTemp, dfn, ok
case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeTemp, fileNum, ok
return FileTypeTemp, dfn, ok
default:
i := strings.IndexByte(filename, '.')
if i < 0 {
break
}
fileNum, ok = parseFileNum(filename[:i])
dfn, ok = parseDiskFileNum(filename[:i])
if !ok {
break
}
switch filename[i+1:] {
case "sst":
return FileTypeTable, fileNum, true
return FileTypeTable, dfn, true
case "log":
return FileTypeLog, fileNum, true
return FileTypeLog, dfn, true
}
}
return 0, fileNum, false
return 0, dfn, false
}
func parseFileNum(s string) (fileNum FileNum, ok bool) {
func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
u, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return fileNum, false
return dfn, false
}
return FileNum(u), true
return DiskFileNum{FileNum(u)}, true
}
// A Fataler fatals a process with a message when called.

40
vendor/github.com/cockroachdb/pebble/internal/base/internal.go generated vendored
View File

@@ -60,6 +60,12 @@ const (
InternalKeyKindRangeKeyUnset InternalKeyKind = 20
InternalKeyKindRangeKeySet InternalKeyKind = 21
// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
// the WAL entry for ingested sstables that are added to the flushable
// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
// batch, or in an sstable.
InternalKeyKindIngestSST InternalKeyKind = 22
// This maximum value isn't part of the file format. It's unlikely,
// but future extensions may increase this value.
//
@@ -69,7 +75,7 @@ const (
// which sorts 'less than or equal to' any other valid internalKeyKind, when
// searching for any kind of internal key formed by a certain user key and
// seqNum.
InternalKeyKindMax InternalKeyKind = 21
InternalKeyKindMax InternalKeyKind = 22
// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
// zero sequence number.
@@ -111,6 +117,7 @@ var internalKeyKindNames = []string{
InternalKeyKindRangeKeySet: "RANGEKEYSET",
InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET",
InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
InternalKeyKindIngestSST: "INGESTSST",
InternalKeyKindInvalid: "INVALID",
}
@@ -187,6 +194,7 @@ var kindsMap = map[string]InternalKeyKind{
"DEL": InternalKeyKindDelete,
"SINGLEDEL": InternalKeyKindSingleDelete,
"RANGEDEL": InternalKeyKindRangeDelete,
"LOGDATA": InternalKeyKindLogData,
"SET": InternalKeyKindSet,
"MERGE": InternalKeyKindMerge,
"INVALID": InternalKeyKindInvalid,
@@ -195,6 +203,7 @@ var kindsMap = map[string]InternalKeyKind{
"RANGEKEYSET": InternalKeyKindRangeKeySet,
"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
"RANGEKEYDEL": InternalKeyKindRangeKeyDelete,
"INGESTSST": InternalKeyKindIngestSST,
}
// ParseInternalKey parses the string representation of an internal key. The
@@ -362,9 +371,14 @@ func (k *InternalKey) SetKind(kind InternalKeyKind) {
k.Trailer = (k.Trailer &^ 0xff) | uint64(kind)
}
// Kind returns the kind compoment of the key.
// Kind returns the kind component of the key.
func (k InternalKey) Kind() InternalKeyKind {
return InternalKeyKind(k.Trailer & 0xff)
return TrailerKind(k.Trailer)
}
// TrailerKind returns the key kind of the key trailer.
func TrailerKind(trailer uint64) InternalKeyKind {
return InternalKeyKind(trailer & 0xff)
}
// Valid returns true if the key has a valid kind.
@@ -383,6 +397,13 @@ func (k InternalKey) Clone() InternalKey {
}
}
// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
// reusing any space already used for the current UserKey.
func (k *InternalKey) CopyFrom(k2 InternalKey) {
k.UserKey = append(k.UserKey[:0], k2.UserKey...)
k.Trailer = k2.Trailer
}
// String returns a string representation of the key.
func (k InternalKey) String() string {
return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())
@@ -413,7 +434,11 @@ type prettyInternalKey struct {
}
func (k prettyInternalKey) Format(s fmt.State, c rune) {
fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
if seqNum := k.SeqNum(); seqNum == InternalKeySeqNumMax {
fmt.Fprintf(s, "%s#inf,%s", k.formatKey(k.UserKey), k.Kind())
} else {
fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
}
}
// ParsePrettyInternalKey parses the pretty string representation of an
@@ -425,6 +450,11 @@ func ParsePrettyInternalKey(s string) InternalKey {
if !ok {
panic(fmt.Sprintf("unknown kind: %q", x[2]))
}
seqNum, _ := strconv.ParseUint(x[1], 10, 64)
var seqNum uint64
if x[1] == "max" || x[1] == "inf" {
seqNum = InternalKeySeqNumMax
} else {
seqNum, _ = strconv.ParseUint(x[1], 10, 64)
}
return MakeInternalKey([]byte(ukey), seqNum, kind)
}

57
vendor/github.com/cockroachdb/pebble/internal/base/iterator.go generated vendored
View File

@@ -4,7 +4,10 @@
package base
import "fmt"
import (
"fmt"
"time"
)
// InternalIterator iterates over a DB's key/value pairs in key order. Unlike
// the Iterator interface, the returned keys are InternalKeys composed of the
@@ -40,7 +43,9 @@ import "fmt"
// context is not a strict byte prefix, but defined by byte equality for the
// result of the Comparer.Split method. An InternalIterator is not required to
// support prefix iteration mode, and can implement SeekPrefixGE by forwarding
// to SeekGE.
// to SeekGE. When the iteration prefix is exhausted, it is not valid to call
// Next on an internal iterator that's already returned (nil,nilv) or a key
// beyond the prefix.
//
// Bounds, [lower, upper), can be set on iterators, either using the SetBounds()
// function in the interface, or in implementation specific ways during iterator
@@ -152,6 +157,22 @@ type InternalIterator interface {
// SeekPrefixGE or Next returned (nil, nilv).
Next() (*InternalKey, LazyValue)
// NextPrefix moves the iterator to the next key/value pair with a different
// prefix than the key at the current iterator position. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise. Note that NextPrefix only checks the upper bound. It is up to
// the caller to ensure that key is greater than or equal to the lower
// bound.
//
// NextPrefix is passed the immediate successor to the current prefix key. A
// valid implementation of NextPrefix is to call SeekGE with succKey.
//
// It is not allowed to call NextPrefix when the previous call was a reverse
// positioning operation or a call to a forward positioning method that
// returned (nil, nilv). It is also not allowed to call NextPrefix when the
// iterator is in prefix iteration mode.
NextPrefix(succKey []byte) (*InternalKey, LazyValue)
// Prev moves the iterator to the previous key/value pair. Returns the key
// and value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that Prev only checks the lower bound. It is up to the
@@ -330,7 +351,11 @@ type InternalIteratorStats struct {
BlockBytes uint64
// Subset of BlockBytes that were in the block cache.
BlockBytesInCache uint64
// BlockReadDuration accumulates the duration spent fetching blocks
// due to block cache misses.
// TODO(sumeer): this currently excludes the time spent in Reader creation,
// and in reading the rangedel and rangekey blocks. Fix that.
BlockReadDuration time.Duration
// The following can repeatedly count the same points if they are iterated
// over multiple times. Additionally, they may count a point twice when
// switching directions. The latter could be improved if needed.
@@ -339,7 +364,7 @@ type InternalIteratorStats struct {
// included.
KeyBytes uint64
// Bytes in values that were iterated over. Currently, only point values are
// included.
// included. For separated values, this is the size of the handle.
ValueBytes uint64
// The count of points iterated over.
PointCount uint64
@@ -347,14 +372,38 @@ type InternalIteratorStats struct {
// can be useful for discovering instances of
// https://github.com/cockroachdb/pebble/issues/1070.
PointsCoveredByRangeTombstones uint64
// Stats related to points in value blocks encountered during iteration.
// These are useful to understand outliers, since typical user facing
// iteration should tend to only look at the latest point, and hence have
// the following stats close to 0.
SeparatedPointValue struct {
// Count is a count of points that were in value blocks. This is not a
// subset of PointCount: PointCount is produced by mergingIter and if
// positioned once, and successful in returning a point, will have a
// PointCount of 1, regardless of how many sstables (and memtables etc.)
// in the heap got positioned. The count here includes every sstable
// iterator that got positioned in the heap.
Count uint64
// ValueBytes represent the total byte length of the values (in value
// blocks) of the points corresponding to Count.
ValueBytes uint64
// ValueBytesFetched is the total byte length of the values (in value
// blocks) that were retrieved.
ValueBytesFetched uint64
}
}
// Merge merges the stats in from into the given stats.
func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
s.BlockBytes += from.BlockBytes
s.BlockBytesInCache += from.BlockBytesInCache
s.BlockReadDuration += from.BlockReadDuration
s.KeyBytes += from.KeyBytes
s.ValueBytes += from.ValueBytes
s.PointCount += from.PointCount
s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones
s.SeparatedPointValue.Count += from.SeparatedPointValue.Count
s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
}

175
vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go generated vendored
View File

@@ -16,12 +16,8 @@ import "github.com/cockroachdb/pebble/internal/invariants"
// it can call the ShortAttributeExtractor to extract the attribute and store
// it together with the key. This allows for cheap retrieval of
// AttributeAndLen on the read-path, without doing a more expensive retrieval
// of the value.
//
// In general, the extraction code may want to also look at the key to decide
// how to treat the value. Our current needs can be satisfied by configuring a
// keyspan for which the ShortAttributeExtractor should be used, so we do not
// expose the key to the extractor.
// of the value. In general, the extraction code may want to also look at the
// key to decide how to treat the value, hence the key* parameters.
//
// Write path performance: The ShortAttributeExtractor func cannot be inlined,
// so we will pay the cost of this function call. However, we will only pay
@@ -38,7 +34,8 @@ const MaxShortAttribute = 7
// ShortAttributeExtractor is an extractor that given the value, will return
// the ShortAttribute.
type ShortAttributeExtractor func(value []byte) (ShortAttribute, error)
type ShortAttributeExtractor func(
key []byte, keyPrefixLen int, value []byte) (ShortAttribute, error)
// AttributeAndLen represents the pair of value length and the short
// attribute.
@@ -70,24 +67,26 @@ type AttributeAndLen struct {
// Memory management:
// This is subtle, but important for performance.
//
// - A LazyValue returned by an InternalIterator or Iterator is unstable in
// that repositioning the iterator will invalidate the memory inside it. A
// caller wishing to maintain that LazyValue needs to call
// LazyValue.Clone(). Note that this does not fetch the value if it is not
// in-place. Additionally, Clone() *must* only be called if
// LazyValue.Value() has *not* been called, since (a) it only makes a
// promise about stability of LazyValue and not the underlying value, (b) if
// LazyValue.Value() has been called already, the caller has the value
// []byte slice so it is unnecessary to fiddle around with LazyValue.
// A LazyValue returned by an InternalIterator or Iterator is unstable in that
// repositioning the iterator will invalidate the memory inside it. A caller
// wishing to maintain that LazyValue needs to call LazyValue.Clone(). Note
// that this does not fetch the value if it is not in-place. Clone() should
// ideally not be called if LazyValue.Value() has been called, since the
// cloned LazyValue will forget the extracted/fetched value, and calling
// Value() on this clone will cause the value to be extracted again. That is,
// Clone() does not make any promise about the memory stability of the
// underlying value.
//
// - A user of an iterator that calls LazyValue.Value() wants as much as
// possible for the returned value []byte to point to iterator owned memory.
// - [P1] The underlying iterator that owns that memory also needs a promise
// A user of an iterator that calls LazyValue.Value() wants as much as
// possible for the returned value []byte to point to iterator owned memory.
//
// 1. [P1] The underlying iterator that owns that memory also needs a promise
// from that user that at any time there is at most one value []byte slice
// that the caller is expecting it to maintain. Otherwise, the underlying
// iterator has to maintain multiple such []byte slices which results in
// more complicated and inefficient code.
// - [P2] The underlying iterator, in order to make the promise that it is
//
// 2. [P2] The underlying iterator, in order to make the promise that it is
// maintaining the one value []byte slice, also needs a way to know when
// it is relieved of that promise. One way it is relieved of that promise
// is by being told that it is being repositioned. Typically, the owner of
@@ -105,28 +104,31 @@ type AttributeAndLen struct {
// the caller. This will be true if the callee attempted to use buf and
// either successfully used it or allocated a new []byte slice.
//
// To ground the above in reality, we consider three examples of callers of
// LazyValue.Value():
// - Iterator: it calls LazyValue.Value for its own use when merging values.
// When merging during reverse iteration, it may have cloned the LazyValue.
// In this case it calls LazyValue.Value() on the cloned value, merges it,
// and then calls LazyValue.Value() on the current iterator position and
// merges it. So it is honoring P1.
// - Iterator on behalf of Iterator clients: The Iterator.Value() method
// needs to call LazyValue.Value(). The client of Iterator is satisfying P1
// because of the inherent Iterator interface constraint, i.e., it is calling
// Iterator.Value() on the current Iterator position. It is possible that
// the Iterator has cloned this LazyValue (for the reverse iteration case),
// which the client is unaware of, so the underlying sstable iterator may
// not be able to satisfy P2. This is ok because Iterator will call
// LazyValue.Value with its (reusable) owned buffer.
// - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
// since during reverse iteration in order to find the highest version that
// satisfies a read it needs to clone the LazyValue, step back the iterator
// and then decide whether it needs the value from the previously cloned
// LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
// similar to the previous case in that it will call LazyValue.Value with
// its (reusable) owned buffer.
// To ground the above in reality, we consider three examples of callers of
// LazyValue.Value():
//
// - Iterator: it calls LazyValue.Value for its own use when merging values.
// When merging during reverse iteration, it may have cloned the LazyValue.
// In this case it calls LazyValue.Value() on the cloned value, merges it,
// and then calls LazyValue.Value() on the current iterator position and
// merges it. So it is honoring P1.
//
// - Iterator on behalf of Iterator clients: The Iterator.Value() method
// needs to call LazyValue.Value(). The client of Iterator is satisfying P1
// because of the inherent Iterator interface constraint, i.e., it is calling
// Iterator.Value() on the current Iterator position. It is possible that
// the Iterator has cloned this LazyValue (for the reverse iteration case),
// which the client is unaware of, so the underlying sstable iterator may
// not be able to satisfy P2. This is ok because Iterator will call
// LazyValue.Value with its (reusable) owned buffer.
//
// - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
// since during reverse iteration in order to find the highest version that
// satisfies a read it needs to clone the LazyValue, step back the iterator
// and then decide whether it needs the value from the previously cloned
// LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
// similar to the previous case in that it will call LazyValue.Value with
// its (reusable) owned buffer.
//
// Corollary: callers that directly use InternalIterator can know that they
// have done nothing to interfere with promise P2 can pass in a nil buf and be
@@ -154,9 +156,24 @@ type LazyValue struct {
}
// LazyFetcher supports fetching a lazy value.
//
// Fetcher and Attribute are to be initialized at creation time. The fields
// are arranged to reduce the sizeof this struct.
type LazyFetcher struct {
// ValueFetcher, given a handle, returns the value. It is acceptable to call
// the ValueFetcher as long as the DB is open. However, one should assume
// Fetcher, given a handle, returns the value.
Fetcher ValueFetcher
err error
value []byte
// Attribute includes the short attribute and value length.
Attribute AttributeAndLen
fetched bool
callerOwned bool
}
// ValueFetcher is an interface for fetching a value.
type ValueFetcher interface {
// Fetch returns the value, given the handle. It is acceptable to call the
// ValueFetcher.Fetch as long as the DB is open. However, one should assume
// there is a fast-path when the iterator tree has not moved off the sstable
// iterator that initially provided this LazyValue. Hence, to utilize this
// fast-path the caller should try to decide whether it needs the value or
@@ -166,13 +183,8 @@ type LazyFetcher struct {
// If the fetcher attempted to use buf *and* len(buf) was insufficient, it
// will allocate a new slice for the value. In either case it will set
// callerOwned to true.
ValueFetcher func(handle []byte, buf []byte) (val []byte, callerOwned bool, err error)
// Attribute includes the short attribute and value length.
Attribute AttributeAndLen
fetched bool
value []byte
callerOwned bool
err error
Fetch(
handle []byte, valLen int32, buf []byte) (val []byte, callerOwned bool, err error)
}
// Value returns the underlying value.
@@ -197,7 +209,8 @@ func (lv *LazyValue) fetchValue(buf []byte) (val []byte, callerOwned bool, err e
f := lv.Fetcher
if !f.fetched {
f.fetched = true
f.value, f.callerOwned, f.err = lv.Fetcher.ValueFetcher(lv.ValueOrHandle, buf)
f.value, f.callerOwned, f.err = f.Fetcher.Fetch(
lv.ValueOrHandle, lv.Fetcher.Attribute.ValueLen, buf)
}
return f.value, f.callerOwned, f.err
}
@@ -219,18 +232,46 @@ func (lv *LazyValue) Len() int {
return int(lv.Fetcher.Attribute.ValueLen)
}
// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
// REQUIRES: LazyValue.Value() has not been called.
// Ensure that this is only called before LazyValue.Value is called. We don't
// actually care if it was in-place but we don't want confusing things about whether
// we need to clone the value inside the fetcher.
func (lv *LazyValue) Clone(buf []byte) (LazyValue, []byte) {
if invariants.Enabled && lv.Fetcher != nil && lv.Fetcher.fetched {
panic("value has already been fetched")
// TryGetShortAttribute returns the ShortAttribute and a bool indicating
// whether the ShortAttribute was populated.
func (lv *LazyValue) TryGetShortAttribute() (ShortAttribute, bool) {
if lv.Fetcher == nil {
return 0, false
}
return lv.Fetcher.Attribute.ShortAttribute, true
}
// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
// The fetcher parameter must be non-nil and may be over-written and used
// inside the returned LazyValue -- this is needed to avoid an allocation.
// Most callers have at most K cloned LazyValues, where K is hard-coded, so
// they can have a pool of exactly K LazyFetcher structs they can reuse in
// these calls. The alternative of allocating LazyFetchers from a sync.Pool is
// not viable since we have no code trigger for returning to the pool
// (LazyValues are simply GC'd).
//
// NB: It is highly preferable that LazyValue.Value() has not been called,
// since the Clone will forget any previously extracted value, and a future
// call to Value will cause it to be fetched again. We do this since we don't
// want to reason about whether or not to clone an already extracted value
// inside the Fetcher (we don't). Property P1 applies here too: if lv1.Value()
// has been called, and then lv2 is created as a clone of lv1, then calling
// lv2.Value() can invalidate any backing memory maintained inside the fetcher
// for lv1 (even though these are the same values). We initially prohibited
// calling LazyValue.Clone() if LazyValue.Value() has been called, but there
// is at least one complex caller (pebbleMVCCScanner inside CockroachDB) where
// it is not easy to prove this invariant.
func (lv *LazyValue) Clone(buf []byte, fetcher *LazyFetcher) (LazyValue, []byte) {
var lvCopy LazyValue
if lv.Fetcher != nil {
*fetcher = LazyFetcher{
Fetcher: lv.Fetcher.Fetcher,
Attribute: lv.Fetcher.Attribute,
// Not copying anything that has been extracted.
}
lvCopy.Fetcher = fetcher
}
// INVARIANT: LazyFetcher.value is nil, so there is nothing to copy there.
vLen := len(lv.ValueOrHandle)
lvCopy := LazyValue{Fetcher: lv.Fetcher}
if vLen == 0 {
return lvCopy, buf
}
@@ -244,13 +285,3 @@ func (lv *LazyValue) Clone(buf []byte) (LazyValue, []byte) {
func MakeInPlaceValue(val []byte) LazyValue {
return LazyValue{ValueOrHandle: val}
}
// TODO(sumeer): test that callers are adhering to promise P1. In the sstable
// iterators, when invariants are enabled, create a testingFetcher struct that
// contains the sstable name and the []byte backing for the last fetched
// value. The sstable iterator, instead of returning an in-place value, will
// return a copy of the key in ValueOrHandle, and a ValueFetcher implemented
// by testingFetcher.valueFetcher. Each time this method is called, it will
// mangle the last fetched value []byte slice, allocate a new one, open the
// sstable, seek to the key, and copy the value into this new byte slice and
// return.

144
vendor/github.com/cockroachdb/pebble/internal/base/logger.go generated vendored Normal file
View File

@@ -0,0 +1,144 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"bytes"
"context"
"fmt"
"log"
"os"
"runtime"
"sync"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Logger defines an interface for writing log messages.
type Logger interface {
Infof(format string, args ...interface{})
Fatalf(format string, args ...interface{})
}
type defaultLogger struct{}
// DefaultLogger logs to the Go stdlib logs.
var DefaultLogger defaultLogger
var _ Logger = DefaultLogger
// Infof implements the Logger.Infof interface.
func (defaultLogger) Infof(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
}
// Fatalf implements the Logger.Fatalf interface.
func (defaultLogger) Fatalf(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
os.Exit(1)
}
// InMemLogger implements Logger using an in-memory buffer (used for testing).
// The buffer can be read via String() and cleared via Reset().
type InMemLogger struct {
mu struct {
sync.Mutex
buf bytes.Buffer
}
}
var _ Logger = (*InMemLogger)(nil)
// Reset clears the internal buffer.
func (b *InMemLogger) Reset() {
b.mu.Lock()
defer b.mu.Unlock()
b.mu.buf.Reset()
}
// String returns the current internal buffer.
func (b *InMemLogger) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.mu.buf.String()
}
// Infof is part of the Logger interface.
func (b *InMemLogger) Infof(format string, args ...interface{}) {
s := fmt.Sprintf(format, args...)
b.mu.Lock()
defer b.mu.Unlock()
b.mu.buf.Write([]byte(s))
if n := len(s); n == 0 || s[n-1] != '\n' {
b.mu.buf.Write([]byte("\n"))
}
}
// Fatalf is part of the Logger interface.
func (b *InMemLogger) Fatalf(format string, args ...interface{}) {
b.Infof(format, args...)
runtime.Goexit()
}
// LoggerAndTracer defines an interface for logging and tracing.
type LoggerAndTracer interface {
Logger
// Eventf formats and emits a tracing log, if tracing is enabled in the
// current context.
Eventf(ctx context.Context, format string, args ...interface{})
// IsTracingEnabled returns true if tracing is enabled. It can be used as an
// optimization to avoid calling Eventf (which will be a noop when tracing
// is not enabled) to avoid the overhead of boxing the args.
IsTracingEnabled(ctx context.Context) bool
}
// LoggerWithNoopTracer wraps a logger and does no tracing.
type LoggerWithNoopTracer struct {
Logger
}
var _ LoggerAndTracer = &LoggerWithNoopTracer{}
// Eventf implements LoggerAndTracer.
func (*LoggerWithNoopTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
if invariants.Enabled && ctx == nil {
panic("Eventf context is nil")
}
}
// IsTracingEnabled implements LoggerAndTracer.
func (*LoggerWithNoopTracer) IsTracingEnabled(ctx context.Context) bool {
if invariants.Enabled && ctx == nil {
panic("IsTracingEnabled ctx is nil")
}
return false
}
// NoopLoggerAndTracer does no logging and tracing. Remember that struct{} is
// special cased in Go and does not incur an allocation when it backs the
// interface LoggerAndTracer.
type NoopLoggerAndTracer struct{}
var _ LoggerAndTracer = NoopLoggerAndTracer{}
// Infof implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Infof(format string, args ...interface{}) {}
// Fatalf implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Fatalf(format string, args ...interface{}) {}
// Eventf implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
if invariants.Enabled && ctx == nil {
panic("Eventf context is nil")
}
}
// IsTracingEnabled implements LoggerAndTracer.
func (l NoopLoggerAndTracer) IsTracingEnabled(ctx context.Context) bool {
if invariants.Enabled && ctx == nil {
panic("IsTracingEnabled ctx is nil")
}
return false
}

2
vendor/github.com/cockroachdb/pebble/internal/base/merger.go generated vendored
View File

@@ -23,7 +23,7 @@ type Merge func(key, value []byte) (ValueMerger, error)
//
// The merge operation must be associative. That is, for the values A, B, C:
//
// Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
// Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
//
// Examples of merge operators are integer addition, list append, and string
// concatenation.

54
vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go generated vendored Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
// This file defines sequence numbers that are reserved for foreign keys i.e.
// internal keys coming from other Pebble instances and existing in shared
// storage, as those will "slot below" any internal keys added by our own Pebble
// instance. Any keys created by this Pebble instance need to be greater than
// all reserved sequence numbers (i.e. >= SeqNumStart).
const (
// SeqNumZero is the zero sequence number, set by compactions if they can
// guarantee there are no keys underneath an internal key.
SeqNumZero = uint64(0)
// SeqNumL6Point is the sequence number reserved for foreign point keys in L6.
// This sequence number must be lower than SeqNumL6RangeKey for range key
// masking to work correctly.
SeqNumL6Point = uint64(1)
// SeqNumL6RangeKey is the sequence number reserved for foreign range keys in
// L6. Only RangeKeySets are expected at this level.
SeqNumL6RangeKey = uint64(2)
// SeqNumL5RangeDel is the sequence number reserved for foreign range deletes
// in L5. These keys could delete L6 points.
SeqNumL5RangeDel = uint64(3)
// SeqNumL5Point is the sequence number reserved for foreign point keys in L5.
// Any sst-local range deletions would have already been applied to these keys,
// so they can safely get a sequence number higher than SeqNumL5RangeDel.
// However they must have a sequence number lower than SeqNumL5RangeKey* for
// range key masking to work correctly.
SeqNumL5Point = uint64(4)
// SeqNumL5RangeKeyUnsetDel is the sequence number reserved for foreign
// range key unsets/deletes in L5. These operations could apply to L6
// RangeKeySets, so this sequence number must be > SeqNumL6RangeKey.
SeqNumL5RangeKeyUnsetDel = uint64(5)
// SeqNumL5RangeKeySet is the sequence number reserved for foreign range key
// Sets in L5. These operations could apply to L6 RangeKeySets, so this
// sequence number must be > SeqNumL6RangeKey. Any SST-local rangekey
// unsets/dels have already been applied to them, so their sequence number must
// be > SeqNumL5RangeKeyUnsetDel.
SeqNumL5RangeKeySet = uint64(6)
// Sequence numbers 7-9 are reserved for future use.
// SeqNumStart is the first sequence number assigned to a key written by
// ourselves.
SeqNumStart = uint64(10)
)

10
vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go generated vendored
View File

@@ -26,7 +26,7 @@ import "github.com/cockroachdb/pebble/internal/rawalloc"
type A []byte
const chunkAllocMinSize = 512
const chunkAllocMaxSize = 16384
const chunkAllocMaxSize = 512 << 10 // 512 KB
func (a A) reserve(n int) A {
allocSize := cap(a) * 2
@@ -59,3 +59,11 @@ func (a A) Copy(src []byte) (A, []byte) {
copy(alloc, src)
return a, alloc
}
// Reset returns the current chunk, resetting allocated memory back to none.
// Future allocations will use memory previously allocated by previous calls to
// Alloc or Copy, so the caller must know know that none of the previously
// allocated byte slices are still in use.
func (a A) Reset() A {
return a[:0]
}

33
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go generated vendored
View File

@@ -33,7 +33,7 @@ import (
type fileKey struct {
// id is the namespace for fileNums.
id uint64
fileNum base.FileNum
fileNum base.DiskFileNum
}
type key struct {
@@ -113,7 +113,7 @@ type shard struct {
countTest int64
}
func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
c.mu.RLock()
var value *Value
if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
@@ -131,7 +131,7 @@ func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
return Handle{value: value}
}
func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
if n := value.refs(); n != 1 {
panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
}
@@ -222,7 +222,7 @@ func (c *shard) checkConsistency() {
}
// Delete deletes the cached value for the specified file and offset.
func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
// The common case is there is nothing to delete, so do a quick check with
// shared lock.
k := key{fileKey{id, fileNum}, offset}
@@ -246,7 +246,7 @@ func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
}
// EvictFile evicts all of the cache values for the specified file.
func (c *shard) EvictFile(id uint64, fileNum base.FileNum) {
func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -603,7 +603,7 @@ type Metrics struct {
// efficient eviction of all of the blocks for a file which is used when an
// sstable is deleted from disk.
//
// Memory Management
// # Memory Management
//
// In order to reduce pressure on the Go GC, manual memory management is
// performed for the data stored in the cache. Manual memory management is
@@ -639,9 +639,9 @@ type Cache struct {
// creator of the cache should usually release their reference after the DB is
// created.
//
// c := cache.New(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
// c := cache.New(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func New(size int64) *Cache {
return newShards(size, 2*runtime.GOMAXPROCS(0))
}
@@ -683,7 +683,7 @@ func newShards(size int64, shards int) *Cache {
return c
}
func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard {
func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
@@ -698,10 +698,11 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
h ^= uint64(id & 0xff)
id >>= 8
}
fileNumVal := uint64(fileNum.FileNum())
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(fileNum & 0xff)
fileNum >>= 8
h ^= uint64(fileNumVal) & 0xff
fileNumVal >>= 8
}
for i := 0; i < 8; i++ {
h *= prime64
@@ -738,7 +739,7 @@ func (c *Cache) Unref() {
// Get retrieves the cache value for the specified file and offset, returning
// nil if no value is present.
func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
}
@@ -746,17 +747,17 @@ func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
// existing value if present. A Handle is returned which provides faster
// retrieval of the cached value than Get (lock-free and avoidance of the map
// lookup). The value must have been allocated by Cache.Alloc.
func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
}
// Delete deletes the cached value for the specified file and offset.
func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) {
func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
}
// EvictFile evicts all of the cache values for the specified file.
func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) {
func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}

2
vendor/github.com/cockroachdb/pebble/internal/cache/entry.go generated vendored
View File

@@ -30,7 +30,7 @@ func (p entryType) String() string {
// Using manual memory management for entries is technically a volation of the
// Cgo pointer rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// Specifically, Go pointers should not be stored in C allocated memory. The
// reason for this rule is that the Go GC will not look at C allocated memory

12
vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go generated vendored
View File

@@ -24,7 +24,7 @@ func robinHoodHash(k key, shift uint32) uint32 {
const m = 11400714819323198485
h := hashSeed
h ^= k.id * m
h ^= uint64(k.fileNum) * m
h ^= uint64(k.fileNum.FileNum()) * m
h ^= k.offset * m
return uint32(h >> shift)
}
@@ -95,11 +95,11 @@ func (e robinHoodEntries) free() {
// table is 2, maxDist is computed as 4 and the actual size of the entry slice
// is 6.
//
// +---+---+---+---+---+---+
// | 0 | 1 | 2 | 3 | 4 | 5 |
// +---+---+---+---+---+---+
// ^
// size
// +---+---+---+---+---+---+
// | 0 | 1 | 2 | 3 | 4 | 5 |
// +---+---+---+---+---+---+
// ^
// size
//
// In this scenario, the target entry for a key will always be in the range
// [0,1]. Valid entries may reside in the range [0,4] due to the linear probing

2
vendor/github.com/cockroachdb/pebble/internal/crc/crc.go generated vendored
View File

@@ -10,7 +10,9 @@
// look like a checksum.
//
// To calculate the uint32 checksum of some data:
//
// var u uint32 = crc.New(data).Value()
//
// In pebble, the uint32 value is then stored in little-endian format.
package crc // import "github.com/cockroachdb/pebble/internal/crc"

2
vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go generated vendored
View File

@@ -7,9 +7,11 @@ package fastrand
import _ "unsafe" // required by go:linkname
// Uint32 returns a lock free uint32 value.
//
//go:linkname Uint32 runtime.fastrand
func Uint32() uint32
// Uint32n returns a lock free uint32 value in the interval [0, n).
//
//go:linkname Uint32n runtime.fastrandn
func Uint32n(n uint32) uint32

11
vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go generated vendored
View File

@@ -66,6 +66,9 @@ type BoundedIter struct {
// indicating whether the iterator is in prefix iteration mode and the prefix
// key if it is. This is used to exclude spans that are outside the iteration
// prefix.
//
// hasPrefix and prefix are allowed to be nil, however if hasPrefix != nil,
// prefix must also not be nil.
func (i *BoundedIter) Init(
cmp base.Compare,
split base.Split,
@@ -146,7 +149,7 @@ func (i *BoundedIter) Next() *Span {
}
// Similarly, if the span extends to the next prefix and we're in prefix
// iteration mode, we can avoid advancing.
if i.iterSpan != nil && *i.hasPrefix {
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
ei := i.split(i.iterSpan.End)
if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 {
i.pos = posAtUpperLimit
@@ -178,7 +181,7 @@ func (i *BoundedIter) Prev() *Span {
}
// Similarly, if the span extends to or beyond the current prefix and
// we're in prefix iteration mode, we can avoid advancing.
if i.iterSpan != nil && *i.hasPrefix {
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(i.iterSpan.Start)
if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 {
i.pos = posAtLowerLimit
@@ -215,7 +218,7 @@ func (i *BoundedIter) SetBounds(lower, upper []byte) {
func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && *i.hasPrefix {
if span != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(span.Start)
if i.cmp(span.Start[:si], *i.prefix) > 0 {
// This span starts at a prefix that sorts after our current prefix.
@@ -242,7 +245,7 @@ func (i *BoundedIter) checkForwardBound(span *Span) *Span {
func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
if span != nil && i.hasPrefix != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
// This span ends before the current prefix.
span = nil
}

141
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored
View File

@@ -8,14 +8,20 @@ import (
"bytes"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/invariants"
)
// bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
// that DefragmentingIter will reuse. Buffers greater than this will be
// that DefragmentingIter will reuse. Buffers larger than this will be
// discarded and reallocated as necessary.
const bufferReuseMaxCapacity = 10 << 10 // 10 KB
// keysReuseMaxCapacity is the maximum capacity of a []keyspan.Key buffer that
// DefragmentingIter will reuse. Buffers larger than this will be discarded and
// reallocated as necessary.
const keysReuseMaxCapacity = 100
// DefragmentMethod configures the defragmentation performed by the
// DefragmentingIter.
type DefragmentMethod interface {
@@ -113,30 +119,21 @@ const (
//
// Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may
// land on a physical fragment in the middle of several fragments that must be
// defragmented. A seek first degfragments in the opposite direction of
// iteration to find the beginning of the defragmented span, and then
// defragments in the iteration direction, ensuring it's found a whole
// defragmented span.
// defragmented. A seek that lands in a fragment straddling the seek key must
// first degfragment in the opposite direction of iteration to find the
// beginning of the defragmented span, and then defragments in the iteration
// direction, ensuring it's found a whole defragmented span.
type DefragmentingIter struct {
// DefragmentingBuffers holds buffers used for copying iterator state.
*DefragmentingBuffers
comparer *base.Comparer
equal base.Equal
iter FragmentIterator
iterSpan *Span
iterPos iterPos
// curr holds the span at the current iterator position. currBuf is a buffer
// for use when copying user keys for curr. keysBuf is a buffer for use when
// copying Keys for curr. currBuf is cleared between positioning methods.
//
// keyBuf is a buffer specifically for the defragmented start key when
// defragmenting backwards or the defragmented end key when defragmenting
// forwards. These bounds are overwritten repeatedly during defragmentation,
// and the defragmentation routines overwrite keyBuf repeatedly to store
// these extended bounds.
curr Span
currBuf []byte
keysBuf []Key
keyBuf []byte
// curr holds the span at the current iterator position.
curr Span
// method is a comparison function for two spans. method is called when two
// spans are abutting to determine whether they may be defragmented.
@@ -148,20 +145,53 @@ type DefragmentingIter struct {
reduce DefragmentReducer
}
// DefragmentingBuffers holds buffers used for copying iterator state.
type DefragmentingBuffers struct {
// currBuf is a buffer for use when copying user keys for curr. currBuf is
// cleared between positioning methods.
currBuf bytealloc.A
// keysBuf is a buffer for use when copying Keys for DefragmentingIter.curr.
keysBuf []Key
// keyBuf is a buffer specifically for the defragmented start key when
// defragmenting backwards or the defragmented end key when defragmenting
// forwards. These bounds are overwritten repeatedly during defragmentation,
// and the defragmentation routines overwrite keyBuf repeatedly to store
// these extended bounds.
keyBuf []byte
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *DefragmentingBuffers) PrepareForReuse() {
if cap(bufs.currBuf) > bufferReuseMaxCapacity {
bufs.currBuf = nil
}
if cap(bufs.keyBuf) > bufferReuseMaxCapacity {
bufs.keyBuf = nil
}
if cap(bufs.keysBuf) > keysReuseMaxCapacity {
bufs.keysBuf = nil
}
}
// Assert that *DefragmentingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*DefragmentingIter)(nil)
// Init initializes the defragmenting iter using the provided defragment
// method.
func (i *DefragmentingIter) Init(
comparer *base.Comparer, iter FragmentIterator, equal DefragmentMethod, reducer DefragmentReducer,
comparer *base.Comparer,
iter FragmentIterator,
equal DefragmentMethod,
reducer DefragmentReducer,
bufs *DefragmentingBuffers,
) {
*i = DefragmentingIter{
comparer: comparer,
equal: comparer.Equal,
iter: iter,
method: equal,
reduce: reducer,
DefragmentingBuffers: bufs,
comparer: comparer,
equal: comparer.Equal,
iter: iter,
method: equal,
reduce: reducer,
}
}
@@ -175,8 +205,9 @@ func (i *DefragmentingIter) Close() error {
return i.iter.Close()
}
// SeekGE seeks the iterator to the first span with a start key greater than or
// equal to key and returns it.
// SeekGE moves the iterator to the first span covering a key greater than or
// equal to the given key. This is equivalent to seeking to the first span with
// an end key greater than the given key.
func (i *DefragmentingIter) SeekGE(key []byte) *Span {
i.iterSpan = i.iter.SeekGE(key)
if i.iterSpan == nil {
@@ -186,30 +217,28 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
i.iterPos = iterPosCurr
return i.iterSpan
}
// Save the current span and peek backwards.
i.saveCurrent()
i.iterSpan = i.iter.Prev()
if i.iterSpan != nil && i.equal(i.curr.Start, i.iterSpan.End) && i.checkEqual(i.iterSpan, &i.curr) {
// A continuation. The span we originally landed on and defragmented
// backwards has a true Start key < key. To obey the FragmentIterator
// contract, we must not return this defragmented span. Defragment
// forward to finish defragmenting the span in the forward direction.
i.defragmentForward()
// Now we must be on a span that truly has a defragmented Start key >
// key.
// If the span starts strictly after key, we know there mustn't be an
// earlier span that ends at i.iterSpan.Start, otherwise i.iter would've
// returned that span instead.
if i.comparer.Compare(i.iterSpan.Start, key) > 0 {
return i.defragmentForward()
}
// The span previous to i.curr does not defragment, so we should return it.
// Next the underlying iterator back onto the span we previously saved to
// i.curr and then defragment forward.
i.iterSpan = i.iter.Next()
// The span we landed on has a Start bound ≤ key. There may be additional
// fragments before this span. Defragment backward to find the start of the
// defragmented span.
i.defragmentBackward()
if i.iterPos == iterPosPrev {
// Next once back onto the span.
i.iterSpan = i.iter.Next()
}
// Defragment the full span from its start.
return i.defragmentForward()
}
// SeekLT seeks the iterator to the last span with a start key less than
// key and returns it.
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
func (i *DefragmentingIter) SeekLT(key []byte) *Span {
i.iterSpan = i.iter.SeekLT(key)
if i.iterSpan == nil {
@@ -219,7 +248,16 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
i.iterPos = iterPosCurr
return i.iterSpan
}
// Defragment forward to find the end of the defragmented span.
// If the span ends strictly before key, we know there mustn't be a later
// span that starts at i.iterSpan.End, otherwise i.iter would've returned
// that span instead.
if i.comparer.Compare(i.iterSpan.End, key) < 0 {
return i.defragmentBackward()
}
// The span we landed on has a End bound ≥ key. There may be additional
// fragments after this span. Defragment forward to find the end of the
// defragmented span.
i.defragmentForward()
if i.iterPos == iterPosNext {
// Prev once back onto the span.
@@ -437,15 +475,9 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
}
func (i *DefragmentingIter) saveCurrent() {
i.currBuf = i.currBuf[:0]
i.currBuf.Reset()
i.keysBuf = i.keysBuf[:0]
i.keyBuf = i.keyBuf[:0]
if cap(i.currBuf) > bufferReuseMaxCapacity {
i.currBuf = nil
}
if cap(i.keyBuf) > bufferReuseMaxCapacity {
i.keyBuf = nil
}
if i.iterSpan == nil {
return
}
@@ -468,7 +500,6 @@ func (i *DefragmentingIter) saveBytes(b []byte) []byte {
if b == nil {
return nil
}
ret := append(i.currBuf, b...)
i.currBuf = ret[len(ret):]
return ret
i.currBuf, b = i.currBuf.Copy(b)
return b
}

124
vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go generated vendored
View File

@@ -108,20 +108,20 @@ func (f *Fragmenter) checkInvariants(buf []Span) {
// increasing start key order. That is, Add must be called with a series
// of spans like:
//
// a---e
// c---g
// c-----i
// j---n
// j-l
// a---e
// c---g
// c-----i
// j---n
// j-l
//
// We need to fragment the spans at overlap points. In the above
// example, we'd create:
//
// a-c-e
// c-e-g
// c-e-g-i
// j-l-n
// j-l
// a-c-e
// c-e-g
// c-e-g-i
// j-l-n
// j-l
//
// The fragments need to be output sorted by start key, and for equal start
// keys, sorted by descending sequence number. This last part requires a mild
@@ -134,42 +134,42 @@ func (f *Fragmenter) checkInvariants(buf []Span) {
//
// Walking through the example above, we start with:
//
// a---e
// a---e
//
// Next we add [c,g) resulting in:
//
// a-c-e
// c---g
// a-c-e
// c---g
//
// The fragment [a,c) is flushed leaving the pending spans as:
//
// c-e
// c---g
// c-e
// c---g
//
// The next span is [c,i):
//
// c-e
// c---g
// c-----i
// c-e
// c---g
// c-----i
//
// No fragments are flushed. The next span is [j,n):
//
// c-e
// c---g
// c-----i
// j---n
// c-e
// c---g
// c-----i
// j---n
//
// The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments
// by their end key, then split the fragments on the end keys:
//
// c-e
// c-e-g
// c-e---i
// c-e
// c-e-g
// c-e---i
//
// The [c,e) fragments all get flushed leaving:
//
// e-g
// e---i
// e-g
// e---i
//
// This process continues until there are no more fragments to flush.
//
@@ -229,16 +229,35 @@ func (f *Fragmenter) Add(s Span) {
f.pending = append(f.pending, s)
}
// Covers returns true if the specified key is covered by one of the pending
// spans. The key must be consistent with the ordering of the
// spans. That is, it is invalid to specify a key here that is out of
// order with the span start keys passed to Add.
func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
// Cover is returned by Framenter.Covers and describes a span's relationship to
// a key at a particular snapshot.
type Cover int8
const (
// NoCover indicates the tested key does not fall within the span's bounds,
// or the span contains no keys with sequence numbers higher than the key's.
NoCover Cover = iota
// CoversInvisibly indicates the tested key does fall within the span's
// bounds and the span contains at least one key with a higher sequence
// number, but none visible at the provided snapshot.
CoversInvisibly
// CoversVisibly indicates the tested key does fall within the span's
// bounds, and the span constains at least one key with a sequence number
// higher than the key's sequence number that is visible at the provided
// snapshot.
CoversVisibly
)
// Covers returns an enum indicating whether the specified key is covered by one
// of the pending keys. The provided key must be consistent with the ordering of
// the spans. That is, it is invalid to specify a key here that is out of order
// with the span start keys passed to Add.
func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) Cover {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if len(f.pending) == 0 {
return false
return NoCover
}
if f.Cmp(f.pending[0].Start, key.UserKey) > 0 {
@@ -246,18 +265,31 @@ func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
f.Format(f.pending[0].Start), key.Pretty(f.Format)))
}
cover := NoCover
seqNum := key.SeqNum()
for _, s := range f.pending {
if f.Cmp(key.UserKey, s.End) < 0 {
// NB: A range deletion tombstone does not delete a point operation
// at the same sequence number, and broadly a span is not considered
// to cover a point operation at the same sequence number.
if s.CoversAt(snapshot, seqNum) {
return true
for i := range s.Keys {
if kseq := s.Keys[i].SeqNum(); kseq > seqNum {
// This key from the span has a higher sequence number than
// `key`. It covers `key`, although the span's key might not
// be visible if its snapshot is too high.
//
// Batch keys are always be visible.
if kseq < snapshot || kseq&base.InternalKeySeqNumBatch != 0 {
return CoversVisibly
}
// s.Keys[i] is not visible.
cover = CoversInvisibly
}
}
}
}
return false
return cover
}
// Empty returns true if all fragments added so far have finished flushing.
@@ -270,20 +302,20 @@ func (f *Fragmenter) Empty() bool {
// emitting of spans which straddle an sstable boundary. Consider
// the scenario:
//
// a---------k#10
// f#8
// f#7
///
// a---------k#10
// f#8
// f#7
//
// Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will
// flush this span:
//
// a-------g#10
// f#8
// f#7
// a-------g#10
// f#8
// f#7
//
// And leave this one in f.pending:
//
// g----k#10
// g----k#10
//
// WARNING: The fragmenter could hold on to the specified end key. Ensure it's
// a safe byte slice that could outlast the current sstable output, and one
@@ -374,8 +406,8 @@ func (f *Fragmenter) truncateAndFlush(key []byte) {
// we want to flush (but not truncate) all range tombstones that start at or
// before the first key in the next sstable. Consider:
//
// a---e#10
// a------h#9
// a---e#10
// a------h#9
//
// If a compaction splits the sstables at key c we want the first sstable to
// contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally

83
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored
View File

@@ -29,9 +29,9 @@ type SpanMask interface {
SpanChanged(*Span)
// SkipPoint is invoked by the interleaving iterator whenever the iterator
// encounters a point key covered by a Span. If SkipPoint returns true, the
// interleaving iterator skips the point key without returning it. This is
// used during range key iteration to skip over point keys 'masked' by range
// keys.
// interleaving iterator skips the point key and all larger keys with the
// same prefix. This is used during range key iteration to skip over point
// keys 'masked' by range keys.
SkipPoint(userKey []byte) bool
}
@@ -79,13 +79,13 @@ type SpanMask interface {
// SeekGE or SeekPrefixGE search key. Consider, for example SetBounds('c', 'e'),
// with an iterator containing the Span [a,z):
//
// First() = `c#72057594037927935,21` Span() = [c,e)
// SeekGE('d') = `d#72057594037927935,21` Span() = [c,e)
// First() = `c#72057594037927935,21` Span() = [c,e)
// SeekGE('d') = `d#72057594037927935,21` Span() = [c,e)
//
// InterleavedIter does not interleave synthetic markers for spans that do not
// contain any keys.
//
// SpanMask
// # SpanMask
//
// InterelavingIter takes a SpanMask parameter that may be used to configure the
// behavior of the iterator. See the documentation on the SpanMask type.
@@ -479,6 +479,30 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) {
return i.interleaveForward(i.lower, nil /* prefix */)
}
// NextPrefix implements (base.InternalIterator).NextPrefix.
func (i *InterleavingIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
if i.dir == -1 {
panic("pebble: cannot switch directions with NextPrefix")
}
// Refresh the point key if the current point key has already been
// interleaved, OR if the next uninterleaved point key has the old suffix
// (it's less than succKey).
if i.pointKeyInterleaved || (i.pointKey != nil && i.cmp(i.pointKey.UserKey, succKey) < 0) {
i.pointKey, i.pointVal = i.pointIter.NextPrefix(succKey)
i.pointKeyInterleaved = false
}
// If we already interleaved the current span start key, and the point key
// is ≥ the span's end key, move to the next span.
if i.keyspanInterleaved && i.pointKey != nil && i.span != nil &&
i.cmp(i.pointKey.UserKey, i.span.End) >= 0 {
i.span = i.keyspanIter.Next()
i.checkForwardBound(nil)
i.savedKeyspan()
}
return i.interleaveForward(i.lower, nil)
}
// Prev implements (base.InternalIterator).Prev.
func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
if i.dir == +1 {
@@ -677,9 +701,29 @@ func (i *InterleavingIter) interleaveForward(
// if we have stepped outside of the span last saved as a mask,
// so that the decision to skip is made with the correct
// knowledge of the covering span.
i.maybeUpdateMask(true /*covered */)
i.maybeUpdateMask(true /* covered */)
if i.mask != nil && i.mask.SkipPoint(i.pointKey.UserKey) {
if i.prefix {
// During prefix-iteration node, once a point is masked,
// all subsequent keys with the same prefix must also be
// masked according to the key ordering. We can stop and
// return nil.
//
// NB: The above is not just an optimization. During
// prefix-iteration mode, the internal iterator contract
// prohibits us from Next-ing beyond the first key
// beyond the iteration prefix. If we didn't already
// stop early, we would need to check if this masked
// point is already beyond the prefix.
return i.yieldNil()
}
// TODO(jackson): If we thread a base.Comparer through to
// InterleavingIter so that we have access to
// ImmediateSuccessor, we could use NextPrefix. We'd need to
// tweak the SpanMask interface slightly, but it's probably
// worthwhile.
i.pointKey, i.pointVal = i.pointIter.Next()
// We may have just invalidated the invariant that
// ensures the span's End is > the point key, so
@@ -780,27 +824,16 @@ func (i *InterleavingIter) interleaveBackward() (*base.InternalKey, base.LazyVal
}
}
// keyspanSeekGE seeks the keyspan iterator to the first span covering k ≥ key.
// Note that this differs from the FragmentIterator.SeekGE semantics, which
// seek to the first span with a start key ≥ key.
func (i *InterleavingIter) keyspanSeekGE(key []byte, prefix []byte) {
// Seek using SeekLT to look for a span that starts before key, with an end
// boundary extending beyond key.
i.span = i.keyspanIter.SeekLT(key)
if i.span == nil || i.cmp(i.span.End, key) <= 0 {
// The iterator is exhausted in the reverse direction, or the span we
// found ends before key. Next to the first key with a start ≥ key.
i.span = i.keyspanIter.Next()
}
// keyspanSeekGE seeks the keyspan iterator to the first span covering a key ≥ k.
func (i *InterleavingIter) keyspanSeekGE(k []byte, prefix []byte) {
i.span = i.keyspanIter.SeekGE(k)
i.checkForwardBound(prefix)
i.savedKeyspan()
}
// keyspanSeekLT seeks the keyspan iterator to the last span covering k < key.
// Note that this differs from the FragmentIterator.SeekLT semantics, which
// seek to the last span with a start key < key.
func (i *InterleavingIter) keyspanSeekLT(key []byte) {
i.span = i.keyspanIter.SeekLT(key)
// keyspanSeekLT seeks the keyspan iterator to the last span covering a key < k.
func (i *InterleavingIter) keyspanSeekLT(k []byte) {
i.span = i.keyspanIter.SeekLT(k)
i.checkBackwardBound()
// The current span's start key is not guaranteed to be less than key,
// because of the bounds enforcement. Consider the following example:
@@ -813,7 +846,7 @@ func (i *InterleavingIter) keyspanSeekLT(key []byte) {
//
// This problem is a consequence of the SeekLT's exclusive search key and
// the fact that we don't perform bounds truncation at every leaf iterator.
if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, key) >= 0 {
if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, k) >= 0 {
i.span = nil
}
i.savedKeyspan()

8
vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go generated vendored
View File

@@ -16,6 +16,7 @@ import "github.com/cockroachdb/pebble/internal/base"
// keyspan.FragmentIterator with point keys.
type InternalIteratorShim struct {
miter MergingIter
mbufs MergingBuffers
span *Span
iterKey base.InternalKey
}
@@ -26,7 +27,7 @@ var _ base.InternalIterator = &InternalIteratorShim{}
// Init initializes the internal iterator shim to merge the provided fragment
// iterators.
func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) {
i.miter.Init(cmp, noopTransform, iters...)
i.miter.Init(cmp, noopTransform, &i.mbufs, iters...)
}
// Span returns the span containing the full set of keys over the key span at
@@ -87,6 +88,11 @@ func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) {
return &i.iterKey, base.MakeInPlaceValue(i.span.End)
}
// NextPrefix implements (base.InternalIterator).NextPrefix.
func (i *InternalIteratorShim) NextPrefix([]byte) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Prev implements (base.InternalIterator).Prev.
func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")

19
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored
View File

@@ -18,12 +18,14 @@ import (
// longer lifetimes but implementations need only guarantee stability until the
// next positioning method.
type FragmentIterator interface {
// SeekGE moves the iterator to the first span whose start key is greater
// than or equal to the given key.
// SeekGE moves the iterator to the first span covering a key greater than
// or equal to the given key. This is equivalent to seeking to the first
// span with an end key greater than the given key.
SeekGE(key []byte) *Span
// SeekLT moves the iterator to the last span whose start key is less than
// the given key.
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
SeekLT(key []byte) *Span
// First moves the iterator to the first span.
@@ -58,7 +60,8 @@ type FragmentIterator interface {
Close() error
}
// TableNewSpanIter creates a new iterator for spans for the given file.
// TableNewSpanIter creates a new iterator for range key spans for the given
// file.
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
@@ -104,6 +107,9 @@ func (i *Iter) Init(cmp base.Compare, spans []Span) {
func (i *Iter) SeekGE(key []byte) *Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(j) = false iff the span i.spans[j] is strictly before `key`
// (equivalently, i.spans[j].End ≤ key.)
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
@@ -111,12 +117,13 @@ func (i *Iter) SeekGE(key []byte) *Span {
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].Start) > 0 {
if i.cmp(key, i.spans[h].End) >= 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
if i.index >= len(i.spans) {

118
vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go generated vendored
View File

@@ -8,25 +8,23 @@ import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
)
// Logger defines an interface for writing log messages.
type Logger interface {
Infof(format string, args ...interface{})
Fatalf(format string, args ...interface{})
}
// LevelIter provides a merged view of spans from sstables in a level.
// It takes advantage of level invariants to only have one sstable span block
// open at one time, opened using the newIter function passed in.
type LevelIter struct {
logger Logger
cmp base.Compare
// Denotes if this level iter should read point key spans (i.e. rangedels,
// or range keys. If key type is Point, no straddle spans are emitted between
// files, and point key bounds are used to find files instead of range key
// bounds.
cmp base.Compare
// Denotes the kind of key the level iterator should read. If the key type
// is KeyTypePoint, the level iterator will read range tombstones (which
// only affect point keys). If the key type is KeyTypeRange, the level
// iterator will read range keys. It is invalid to configure an iterator
// with the KeyTypePointAndRange key type.
//
// If key type is KeyTypePoint, no straddle spans are emitted between files,
// and point key bounds are used to find files instead of range key bounds.
//
// TODO(bilal): Straddle spans can safely be produced in rangedel mode once
// we can guarantee that we will never read sstables in a level that split
@@ -50,14 +48,17 @@ type LevelIter struct {
dir int
straddle Span
straddleDir int
// The iter for the current file. It is nil under any of the following conditions:
// The iter for the current file (iterFile). It is nil under any of the
// following conditions:
// - files.Current() == nil
// - err != nil
// - straddleDir != 0, in which case iterFile is not nil and points to the
// next file (in the straddleDir direction).
// - some other constraint, like the bounds in opts, caused the file at index to not
// be relevant to the iteration.
iter FragmentIterator
iter FragmentIterator
// iterFile holds the current file.
// INVARIANT: iterFile = files.Current()
iterFile *manifest.FileMetadata
newIter TableNewSpanIter
files manifest.LevelIterator
@@ -72,18 +73,17 @@ type LevelIter struct {
// LevelIter implements the keyspan.FragmentIterator interface.
var _ FragmentIterator = (*LevelIter)(nil)
// newLevelIter returns a LevelIter.
func newLevelIter(
// NewLevelIter returns a LevelIter.
func NewLevelIter(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
logger Logger,
keyType manifest.KeyType,
) *LevelIter {
l := &LevelIter{}
l.Init(opts, cmp, newIter, files, level, logger, keyType)
l.Init(opts, cmp, newIter, files, level, keyType)
return l
}
@@ -94,18 +94,19 @@ func (l *LevelIter) Init(
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
logger Logger,
keyType manifest.KeyType,
) {
l.err = nil
l.level = level
l.logger = logger
l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
l.cmp = cmp
l.iterFile = nil
l.newIter = newIter
switch keyType {
case manifest.KeyTypePoint, manifest.KeyTypeRange:
case manifest.KeyTypePoint:
l.keyType = keyType
l.files = files.Filter(keyType)
case manifest.KeyTypeRange:
l.keyType = keyType
l.files = files.Filter(keyType)
default:
@@ -196,7 +197,9 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
f := l.findFileGE(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
// Peek at the previous file.
prevFile := l.files.Prev()
l.files.Next()
if prevFile != nil {
// We could unconditionally return an empty span between the seek key and
// f.SmallestRangeKey, however if this span is to the left of all range
@@ -212,11 +215,10 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
l.files.Next()
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return nil
return l.verify(nil)
}
l.straddleDir = +1
l.straddle = Span{
@@ -224,15 +226,15 @@ func (l *LevelIter) SeekGE(key []byte) *Span {
End: f.SmallestRangeKey.UserKey,
Keys: nil,
}
return &l.straddle
return l.verify(&l.straddle)
}
}
loadFileIndicator := l.loadFile(f, +1)
if loadFileIndicator == noFileLoaded {
return nil
return l.verify(nil)
}
if span := l.iter.SeekGE(key); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileForward()
}
@@ -246,7 +248,9 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
f := l.findFileLT(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
// Peek at the next file.
nextFile := l.files.Next()
l.files.Prev()
if nextFile != nil {
// We could unconditionally return an empty span between f.LargestRangeKey
// and the seek key, however if this span is to the right of all range keys
@@ -262,11 +266,10 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
l.files.Prev()
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return nil
return l.verify(nil)
}
l.straddleDir = -1
l.straddle = Span{
@@ -274,14 +277,14 @@ func (l *LevelIter) SeekLT(key []byte) *Span {
End: nextFile.SmallestRangeKey.UserKey,
Keys: nil,
}
return &l.straddle
return l.verify(&l.straddle)
}
}
if l.loadFile(l.findFileLT(key), -1) == noFileLoaded {
return nil
if l.loadFile(f, -1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.SeekLT(key); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
@@ -294,10 +297,10 @@ func (l *LevelIter) First() *Span {
l.err = nil // clear cached iteration error
if l.loadFile(l.files.First(), +1) == noFileLoaded {
return nil
return l.verify(nil)
}
if span := l.iter.First(); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileForward()
}
@@ -310,10 +313,10 @@ func (l *LevelIter) Last() *Span {
l.err = nil // clear cached iteration error
if l.loadFile(l.files.Last(), -1) == noFileLoaded {
return nil
return l.verify(nil)
}
if span := l.iter.Last(); span != nil {
return span
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
@@ -321,7 +324,7 @@ func (l *LevelIter) Last() *Span {
// Next implements keyspan.FragmentIterator.
func (l *LevelIter) Next() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) {
return nil
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir <= 0
@@ -331,7 +334,7 @@ func (l *LevelIter) Next() *Span {
if l.iter != nil {
if span := l.iter.Next(); span != nil {
return span
return l.verify(span)
}
}
return l.skipEmptyFileForward()
@@ -340,7 +343,7 @@ func (l *LevelIter) Next() *Span {
// Prev implements keyspan.FragmentIterator.
func (l *LevelIter) Prev() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) {
return nil
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir >= 0
@@ -350,7 +353,7 @@ func (l *LevelIter) Prev() *Span {
if l.iter != nil {
if span := l.iter.Prev(); span != nil {
return span
return l.verify(span)
}
}
return l.skipEmptyFileBackward()
@@ -367,7 +370,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
// Straddle spans are not created in rangedel mode.
if err := l.Close(); err != nil {
l.err = err
return nil
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -375,7 +378,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
// which it should be due to the Close() call above.
l.iterFile = l.files.Next()
if l.iterFile == nil {
return nil
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
@@ -386,7 +389,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
End: endKey,
}
l.straddleDir = +1
return &l.straddle
return l.verify(&l.straddle)
}
} else if l.straddleDir < 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
@@ -405,7 +408,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
fileToLoad = l.files.Next()
}
if l.loadFile(fileToLoad, +1) == noFileLoaded {
return nil
return l.verify(nil)
}
span = l.iter.First()
// In rangedel mode, we can expect to get empty files that we'd need to
@@ -414,7 +417,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span {
break
}
}
return span
return l.verify(span)
}
func (l *LevelIter) skipEmptyFileBackward() *Span {
@@ -428,7 +431,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
l.iterFile != nil && l.iter != nil {
if err := l.Close(); err != nil {
l.err = err
return nil
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
@@ -436,7 +439,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
// which it should be due to the Close() call above.
l.iterFile = l.files.Prev()
if l.iterFile == nil {
return nil
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
@@ -447,7 +450,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
End: endKey,
}
l.straddleDir = -1
return &l.straddle
return l.verify(&l.straddle)
}
} else if l.straddleDir > 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
@@ -464,7 +467,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
fileToLoad = l.files.Prev()
}
if l.loadFile(fileToLoad, -1) == noFileLoaded {
return nil
return l.verify(nil)
}
span = l.iter.Last()
// In rangedel mode, we can expect to get empty files that we'd need to
@@ -474,7 +477,22 @@ func (l *LevelIter) skipEmptyFileBackward() *Span {
break
}
}
return span
return l.verify(span)
}
// verify is invoked whenever a span is returned from an iterator positioning
// method to a caller. During invariant builds, it asserts invariants to the
// caller.
func (l *LevelIter) verify(s *Span) *Span {
// NB: Do not add any logic outside the invariants.Enabled conditional to
// ensure that verify is always compiled away in production builds.
if invariants.Enabled {
if f := l.files.Current(); f != l.iterFile {
panic(fmt.Sprintf("LevelIter.files.Current (%s) and l.iterFile (%s) diverged",
f, l.iterFile))
}
}
return s
}
// Error implements keyspan.FragmentIterator.

423
vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go generated vendored
View File

@@ -21,6 +21,10 @@ import (
// seeks would require introducing key comparisons to switchTo{Min,Max}Heap
// where there currently are none.
// TODO(jackson): There are several opportunities to use base.Equal in the
// MergingIter implementation, but will require a bit of plumbing to thread the
// Equal function.
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
@@ -45,9 +49,9 @@ var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst
return nil
})
// visibleTransform filters keys that are invisible at the provided snapshot
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func visibleTransform(snapshot uint64) Transformer {
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
@@ -76,34 +80,34 @@ func visibleTransform(snapshot uint64) Transformer {
// trailer descending. If the MergingIter is configured with a Transformer, it's
// permitted to modify the ordering of the spans' keys returned by MergingIter.
//
// Algorithm
// # Algorithm
//
// The merging iterator wraps child iterators, merging and fragmenting spans
// across levels. The high-level algorithm is:
//
// 1. Initialize the heap with bound keys from child iterators' spans.
// 2. Find the next [or previous] two unique user keys' from bounds.
// 3. Consider the span formed between the two unique user keys a candidate
// span.
// 4. Determine if any of the child iterators' spans overlap the candidate
// span.
// 4a. If any of the child iterator's current bounds are end keys
// (during forward iteration) or start keys (during reverse
// iteration), then all the spans with that bound overlap the
// candidate span.
// 4b. Apply the configured transform, which may remove keys.
// 4b. If no spans overlap, forget the smallest (forward iteration)
// or largest (reverse iteration) unique user key and advance
// the iterators to the next unique user key. Start again from 3.
// 1. Initialize the heap with bound keys from child iterators' spans.
// 2. Find the next [or previous] two unique user keys' from bounds.
// 3. Consider the span formed between the two unique user keys a candidate
// span.
// 4. Determine if any of the child iterators' spans overlap the candidate
// span.
// 4a. If any of the child iterator's current bounds are end keys
// (during forward iteration) or start keys (during reverse
// iteration), then all the spans with that bound overlap the
// candidate span.
// 4b. Apply the configured transform, which may remove keys.
// 4c. If no spans overlap, forget the smallest (forward iteration)
// or largest (reverse iteration) unique user key and advance
// the iterators to the next unique user key. Start again from 3.
//
// Detailed algorithm
// # Detailed algorithm
//
// Each level (i0, i1, ...) has a user-provided input FragmentIterator. The
// merging iterator steps through individual boundaries of the underlying
// spans separately. If the underlying FragmentIterator has fragments
// [a,b){#2,#1} [b,c){#1} the mergingIterLevel.{next,prev} step through:
//
// (a, start), (b, end), (b, start), (c, end)
// (a, start), (b, end), (b, start), (c, end)
//
// Note that (a, start) and (b, end) are observed ONCE each, despite two keys
// sharing those bounds. Also note that (b, end) and (b, start) are two distinct
@@ -119,18 +123,18 @@ func visibleTransform(snapshot uint64) Transformer {
// determine which span is next, but it's also responsible for fragmenting
// overlapping spans. Consider the example:
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
//
// fragments: a-b-c-d-e-----h-----k----------p
// fragments: a-b-c-d-e-----h-----k----------p
//
// None of the individual child iterators contain a span with the exact bounds
// [c,d), but the merging iterator must produce a span [c,d). To accomplish
// this, the merging iterator visits every span between unique boundary user
// keys. In the above example, this is:
//
// [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p)
// [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p)
//
// The merging iterator first initializes the heap to prepare for iteration.
// The description below discusses the mechanics of forward iteration after a
@@ -141,9 +145,9 @@ func visibleTransform(snapshot uint64) Transformer {
// mergingIterLevel to the first bound of the first fragment. In the above
// example, this seeks the child iterators to:
//
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (a, boundKindFragmentStart, [ [a,c) ])
// i2: (a, boundKindFragmentStart, [ [a,p) ])
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (a, boundKindFragmentStart, [ [a,c) ])
// i2: (a, boundKindFragmentStart, [ [a,p) ])
//
// After fixing up the heap, the root of the heap is a boundKey with the
// smallest user key ('a' in the example). Once the heap is setup for iteration
@@ -160,9 +164,9 @@ func visibleTransform(snapshot uint64) Transformer {
// In the above example, this results in m.start = 'a', m.end = 'b' and child
// iterators in the following positions:
//
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (c, boundKindFragmentEnd, [ [a,c) ])
// i2: (p, boundKindFragmentEnd, [ [a,p) ])
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (c, boundKindFragmentEnd, [ [a,c) ])
// i2: (p, boundKindFragmentEnd, [ [a,p) ])
//
// With the user key bounds of the next merged span established,
// findNextFragmentSet must determine which, if any, fragments overlap the span.
@@ -191,7 +195,7 @@ func visibleTransform(snapshot uint64) Transformer {
// findNextFragmentSet loops, repeating the above process again until it finds a
// span that does contain keys.
//
// Memory safety
// # Memory safety
//
// The FragmentIterator interface only guarantees stability of a Span and its
// associated slices until the next positioning method is called. Adjacent Spans
@@ -207,24 +211,14 @@ func visibleTransform(snapshot uint64) Transformer {
// accommodate this, find{Next,Prev}FragmentSet copy the initial boundary if the
// subsequent Next/Prev would move to the next span.
type MergingIter struct {
levels []mergingIterLevel
heap mergingIterHeap
*MergingBuffers
// start and end hold the bounds for the span currently under the
// iterator position.
//
// Invariant: None of the levels' iterators contain spans with a bound
// between start and end. For all bounds b, b ≤ start || b ≥ end.
start, end []byte
// buf is a buffer used to save [start, end) boundary keys.
buf []byte
// keys holds all of the keys across all levels that overlap the key span
// [start, end), sorted by Trailer descending. This slice is reconstituted
// in synthesizeKeys from each mergingIterLevel's keys every time the
// [start, end) bounds change.
//
// Each element points into a child iterator's memory, so the keys may not
// be directly modified.
keys keysBySeqNumKind
// transformer defines a transformation to be applied to a span before it's
// yielded to the user. Transforming may filter individual keys contained
// within the span.
@@ -233,9 +227,8 @@ type MergingIter struct {
// destination for transforms. Every tranformed span overwrites the
// previous.
span Span
err error
dir int8
err error
dir int8
// alloc preallocates mergingIterLevel and mergingIterItems for use by the
// merging iterator. As long as the merging iterator is used with
@@ -249,6 +242,35 @@ type MergingIter struct {
}
}
// MergingBuffers holds buffers used while merging keyspans.
type MergingBuffers struct {
// keys holds all of the keys across all levels that overlap the key span
// [start, end), sorted by Trailer descending. This slice is reconstituted
// in synthesizeKeys from each mergingIterLevel's keys every time the
// [start, end) bounds change.
//
// Each element points into a child iterator's memory, so the keys may not
// be directly modified.
keys keysBySeqNumKind
// levels holds levels allocated by MergingIter.init. The MergingIter will
// prefer use of its `manifest.NumLevels+3` array, so this slice will be
// longer if set.
levels []mergingIterLevel
// heap holds a slice for the merging iterator heap allocated by
// MergingIter.init. The MergingIter will prefer use of its
// `manifest.NumLevels+3` items array, so this slice will be longer if set.
heap mergingIterHeap
// buf is a buffer used to save [start, end) boundary keys.
buf []byte
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *MergingBuffers) PrepareForReuse() {
if cap(bufs.buf) > bufferReuseMaxCapacity {
bufs.buf = nil
}
}
// MergingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*MergingIter)(nil)
@@ -300,13 +322,15 @@ func (l *mergingIterLevel) prev() {
}
// Init initializes the merging iterator with the provided fragment iterators.
func (m *MergingIter) Init(cmp base.Compare, transformer Transformer, iters ...FragmentIterator) {
levels, items := m.levels, m.heap.items
func (m *MergingIter) Init(
cmp base.Compare, transformer Transformer, bufs *MergingBuffers, iters ...FragmentIterator,
) {
*m = MergingIter{
heap: mergingIterHeap{cmp: cmp},
transformer: transformer,
MergingBuffers: bufs,
transformer: transformer,
}
m.heap.cmp = cmp
levels, items := m.levels, m.heap.items
// Invariant: cap(levels) >= cap(items)
// Invariant: cap(alloc.levels) == cap(alloc.items)
@@ -334,38 +358,70 @@ func (m *MergingIter) AddLevel(iter FragmentIterator) {
m.levels = append(m.levels, mergingIterLevel{iter: iter})
}
// SeekGE moves the iterator to the first span with a start key greater than or
// equal to key.
// SeekGE moves the iterator to the first span covering a key greater than
// or equal to the given key. This is equivalent to seeking to the first
// span with an end key greater than the given key.
func (m *MergingIter) SeekGE(key []byte) *Span {
m.invalidate() // clear state about current position
// SeekGE(k) seeks to the first span with an end key greater than the given
// key. The merged span M that we're searching for might straddle the seek
// `key`. In this case, the M.Start may be a key ≤ the seek key.
//
// Consider a SeekGE(dog) in the following example.
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
// merged: a-b-c-d-e-----h-----k----------p
//
// The merged span M containing 'dog' is [d,e). The 'd' of the merged span
// comes from i0's [b,d)'s end boundary. The [b,d) span does not cover any
// key >= dog, so we cannot find the span by positioning the child iterators
// using a SeekGE(dog).
//
// Instead, if we take all the child iterators' spans bounds:
// a b c d e h k p
// We want to partition them into keys ≤ `key` and keys > `key`.
// dog
// │
// a b c d│e h k p
// │
// The largest key on the left of the partition forms the merged span's
// start key, and the smallest key on the right of the partition forms the
// merged span's end key. Recharacterized:
//
// M.Start: the largest boundary ≤ k of any child span
// M.End: the smallest boundary > k of any child span
//
// The FragmentIterator interface doesn't implement seeking by all bounds,
// it implements seeking by containment. A SeekGE(k) will ensure we observe
// all start boundaries ≥ k and all end boundaries > k but does not ensure
// we observe end boundaries = k or any boundaries < k. A SeekLT(k) will
// ensure we observe all start boundaries < k and all end boundaries ≤ k but
// does not ensure we observe any start boundaries = k or any boundaries >
// k. This forces us to seek in one direction and step in the other.
//
// In a SeekGE, we want to end up oriented in the forward direction when
// complete, so we begin with searching for M.Start by SeekLT-ing every
// child iterator to `k`. For every child span found, we determine the
// largest bound ≤ `k` and use it to initialize our max heap. The resulting
// root of the max heap is a preliminary value for `M.Start`.
for i := range m.levels {
l := &m.levels[i]
// A SeekGE requires we position each level at the smallest bound ≥ key.
// We must search through both inclusive start and exclusive end bounds.
// Note that this search requirement differs from FragmentIterator's
// .SeekGE'semantics, which returns the span with the smallest start key
// ≥ key. To remedy this difference, we find the last span less than
// key. If its end boundary is greater than or equal to key, we use it.
// Otherwise we use the start boundary of the next span which
// necessarily has a start ≥ key.
s := l.iter.SeekLT(key)
if s != nil && m.cmp(s.End, key) >= 0 {
// s.End ≥ key
// We need to use this span's end bound.
if s == nil {
l.heapKey = boundKey{kind: boundKindInvalid}
} else if m.cmp(s.End, key) <= 0 {
l.heapKey = boundKey{
kind: boundKindFragmentEnd,
key: s.End,
span: s,
}
continue
}
// s.End < key
// The span `s` ends before key. Next to the first span with a Start ≥
// key, and use that.
if s = l.iter.Next(); s == nil {
l.heapKey = boundKey{kind: boundKindInvalid}
} else {
// s.End > key && s.Start < key
// We need to use this span's start bound, since that's the largest
// bound ≤ key.
l.heapKey = boundKey{
kind: boundKindFragmentStart,
key: s.Start,
@@ -373,36 +429,201 @@ func (m *MergingIter) SeekGE(key []byte) *Span {
}
}
}
m.initMinHeap()
return m.findNextFragmentSet()
}
m.initMaxHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
// There are no spans covering any key < `key`. There is no span that
// straddles the seek key. Reorient the heap into a min heap and return
// the first span we find in the forward direction.
m.switchToMinHeap()
return m.findNextFragmentSet()
}
// SeekLT moves the iterator to the last span with a start key less than key.
func (m *MergingIter) SeekLT(key []byte) *Span {
// TODO(jackson): Evaluate whether there's an implementation of SeekLT
// independent of SeekGE that is more efficient. It's tricky, because the
// span we should return might straddle `key` itself.
// The heap root is now the largest boundary key b such that:
// 1. b < k
// 2. b = k, and b is an end boundary
// There's a third case that we will need to consider later, after we've
// switched to a min heap:
// 3. there exists a start boundary key b such that b = k.
// A start boundary key equal to k would not be surfaced when we seeked all
// the levels using SeekLT(k), since no key <k would be covered within a
// span within an inclusive `k` start boundary.
//
// Consider the scenario:
// a----------l #2
// b-----------m #1
//
// The merged, fully-fragmented spans that MergingIter exposes to the caller
// have bounds:
// a-b #2
// b--------l #2
// b--------l #1
// l-m #1
//
// A call SeekLT(c) must return the largest of the above spans with a
// Start user key < key: [b,l)#1. This requires examining bounds both < 'c'
// (the 'b' of [b,m)#1's start key) and bounds ≥ 'c' (the 'l' of ([a,l)#2's
// end key).
if s := m.SeekGE(key); s == nil && m.err != nil {
// Assume that the tightest boundary ≤ k is the current heap root (cases 1 &
// 2). After we switch to a min heap, we'll check for the third case and
// adjust the start boundary if necessary.
m.start = m.heap.items[0].boundKey.key
// Before switching the direction of the heap, save a copy of the start
// boundary if it's the end boundary of some child span. Next-ing the child
// iterator might switch files and invalidate the memory of the bound.
if m.heap.items[0].boundKey.kind == boundKindFragmentEnd {
m.buf = append(m.buf[:0], m.start...)
m.start = m.buf
}
// Switch to a min heap. This will move each level to the next bound in
// every level, and then establish a min heap. This allows us to obtain the
// smallest boundary key > `key`, which will serve as our candidate end
// bound.
m.switchToMinHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
return nil
}
// Prev to the previous span.
return m.Prev()
// Check for the case 3 described above. It's possible that when we switch
// heap directions, we discover a start boundary of some child span that is
// equal to the seek key `key`. In this case, we want this key to be our
// start boundary.
if m.heap.items[0].boundKey.kind == boundKindFragmentStart &&
m.cmp(m.heap.items[0].boundKey.key, key) == 0 {
// Call findNextFragmentSet, which will set m.start to the heap root and
// proceed forward.
return m.findNextFragmentSet()
}
m.end = m.heap.items[0].boundKey.key
if found, s := m.synthesizeKeys(+1); found && s != nil {
return s
}
return m.findNextFragmentSet()
}
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
func (m *MergingIter) SeekLT(key []byte) *Span {
m.invalidate() // clear state about current position
// SeekLT(k) seeks to the last span with a start key less than the given
// key. The merged span M that we're searching for might straddle the seek
// `key`. In this case, the M.End may be a key ≥ the seek key.
//
// Consider a SeekLT(dog) in the following example.
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
// merged: a-b-c-d-e-----h-----k----------p
//
// The merged span M containing the largest key <'dog' is [d,e). The 'e' of
// the merged span comes from i0's [e,h)'s start boundary. The [e,h) span
// does not cover any key < dog, so we cannot find the span by positioning
// the child iterators using a SeekLT(dog).
//
// Instead, if we take all the child iterators' spans bounds:
// a b c d e h k p
// We want to partition them into keys < `key` and keys ≥ `key`.
// dog
// │
// a b c d│e h k p
// │
// The largest key on the left of the partition forms the merged span's
// start key, and the smallest key on the right of the partition forms the
// merged span's end key. Recharacterized:
//
// M.Start: the largest boundary < k of any child span
// M.End: the smallest boundary ≥ k of any child span
//
// The FragmentIterator interface doesn't implement seeking by all bounds,
// it implements seeking by containment. A SeekGE(k) will ensure we observe
// all start boundaries ≥ k and all end boundaries > k but does not ensure
// we observe end boundaries = k or any boundaries < k. A SeekLT(k) will
// ensure we observe all start boundaries < k and all end boundaries ≤ k but
// does not ensure we observe any start boundaries = k or any boundaries >
// k. This forces us to seek in one direction and step in the other.
//
// In a SeekLT, we want to end up oriented in the backward direction when
// complete, so we begin with searching for M.End by SeekGE-ing every
// child iterator to `k`. For every child span found, we determine the
// smallest bound ≥ `k` and use it to initialize our min heap. The resulting
// root of the min heap is a preliminary value for `M.End`.
for i := range m.levels {
l := &m.levels[i]
s := l.iter.SeekGE(key)
if s == nil {
l.heapKey = boundKey{kind: boundKindInvalid}
} else if m.cmp(s.Start, key) >= 0 {
l.heapKey = boundKey{
kind: boundKindFragmentStart,
key: s.Start,
span: s,
}
} else {
// s.Start < key
// We need to use this span's end bound, since that's the smallest
// bound > key.
l.heapKey = boundKey{
kind: boundKindFragmentEnd,
key: s.End,
span: s,
}
}
}
m.initMinHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
// There are no spans covering any key ≥ `key`. There is no span that
// straddles the seek key. Reorient the heap into a max heap and return
// the first span we find in the reverse direction.
m.switchToMaxHeap()
return m.findPrevFragmentSet()
}
// The heap root is now the smallest boundary key b such that:
// 1. b > k
// 2. b = k, and b is a start boundary
// There's a third case that we will need to consider later, after we've
// switched to a max heap:
// 3. there exists an end boundary key b such that b = k.
// An end boundary key equal to k would not be surfaced when we seeked all
// the levels using SeekGE(k), since k would not be contained within the
// exclusive end boundary.
//
// Assume that the tightest boundary ≥ k is the current heap root (cases 1 &
// 2). After we switch to a max heap, we'll check for the third case and
// adjust the end boundary if necessary.
m.end = m.heap.items[0].boundKey.key
// Before switching the direction of the heap, save a copy of the end
// boundary if it's the start boundary of some child span. Prev-ing the
// child iterator might switch files and invalidate the memory of the bound.
if m.heap.items[0].boundKey.kind == boundKindFragmentStart {
m.buf = append(m.buf[:0], m.end...)
m.end = m.buf
}
// Switch to a max heap. This will move each level to the previous bound in
// every level, and then establish a max heap. This allows us to obtain the
// largest boundary key < `key`, which will serve as our candidate start
// bound.
m.switchToMaxHeap()
if m.err != nil {
return nil
} else if len(m.heap.items) == 0 {
return nil
}
// Check for the case 3 described above. It's possible that when we switch
// heap directions, we discover an end boundary of some child span that is
// equal to the seek key `key`. In this case, we want this key to be our end
// boundary.
if m.heap.items[0].boundKey.kind == boundKindFragmentEnd &&
m.cmp(m.heap.items[0].boundKey.key, key) == 0 {
// Call findPrevFragmentSet, which will set m.end to the heap root and
// proceed backwards.
return m.findPrevFragmentSet()
}
m.start = m.heap.items[0].boundKey.key
if found, s := m.synthesizeKeys(-1); found && s != nil {
return s
}
return m.findPrevFragmentSet()
}
// First seeks the iterator to the first span.
@@ -823,8 +1044,8 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) {
// sorted by trailer descending for the range key iteration use case.
sort.Sort(&m.keys)
// Apply the configured transform. See visibleTransform.
s := Span{
// Apply the configured transform. See VisibleTransform.
m.span = Span{
Start: m.start,
End: m.end,
Keys: m.keys,
@@ -832,7 +1053,7 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) {
}
// NB: m.heap.cmp is a base.Compare, whereas m.cmp is a method on
// MergingIter.
if err := m.transformer.Transform(m.heap.cmp, s, &m.span); err != nil {
if err := m.transformer.Transform(m.heap.cmp, m.span, &m.span); err != nil {
m.err = err
return false, nil
}

28
vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go generated vendored
View File

@@ -6,34 +6,6 @@ package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// SeekGE seeks to the span that contains the target key or the first span past
// the target key.
func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were to
// use SeekGE then the search key `c` would land on the span [e,i) and
// we'd have to backtrack. The one complexity here is what happens for the
// search key `e`. In that case SeekLT will land us on the span [a,e)
// and we'll have to move forward.
iterSpan := iter.SeekLT(key)
// Invariant: key > iterSpan.Start
if iterSpan == nil || cmp(key, iterSpan.End) >= 0 {
// The current span lies entirely before the search key, or the iterator
// is exhausted. Advance the iterator to the next span which is
// guaranteed to lie at or past the search key.
iterSpan = iter.Next()
}
return iterSpan
}
// SeekLE seeks to the span that contains or is before the target key.
func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search

23
vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go generated vendored
View File

@@ -24,7 +24,8 @@ import (
// Note that the start user key is inclusive and the end user key is exclusive.
//
// Currently the only supported key kinds are:
// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
//
// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
type Span struct {
// Start and End encode the user key range of all the contained items, with
// an inclusive start key and exclusive end key. Both Start and End must be
@@ -75,6 +76,15 @@ func (k Key) SeqNum() uint64 {
return k.Trailer >> 8
}
// VisibleAt returns true if the provided key is visible at the provided
// snapshot sequence number. It interprets batch sequence numbers as always
// visible, because non-visible batch span keys are filtered when they're
// fragmented.
func (k Key) VisibleAt(snapshot uint64) bool {
seq := k.SeqNum()
return seq < snapshot || seq&base.InternalKeySeqNumBatch != 0
}
// Kind returns the kind component of the key.
func (k Key) Kind() base.InternalKeyKind {
return base.InternalKeyKind(k.Trailer & 0xff)
@@ -396,6 +406,17 @@ func SortKeysByTrailer(keys *[]Key) {
sort.Sort(sorted)
}
// KeysBySuffix implements sort.Interface, sorting its member Keys slice to by
// Suffix in the order dictated by Cmp.
type KeysBySuffix struct {
Cmp base.Compare
Keys []Key
}
func (s *KeysBySuffix) Len() int { return len(s.Keys) }
func (s *KeysBySuffix) Less(i, j int) bool { return s.Cmp(s.Keys[i].Suffix, s.Keys[j].Suffix) < 0 }
func (s *KeysBySuffix) Swap(i, j int) { s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] }
// ParseSpan parses the string representation of a Span. It's intended for
// tests. ParseSpan panics if passed a malformed span representation.
func ParseSpan(input string) Span {

18
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored
View File

@@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base"
// truncated to be contained within the range [lower, upper). If start and end
// are specified, filter out any spans that are completely outside those bounds.
func Truncate(
cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
cmp base.Compare,
iter FragmentIterator,
lower, upper []byte,
start, end *base.InternalKey,
panicOnPartialOverlap bool,
) FragmentIterator {
return Filter(iter, func(in *Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
// Ignore this span if it lies completely outside [start, end].
// Ignore this span if it lies completely outside start, end. Note that
// end endInclusive indicated whether end is inclusive.
//
// The comparison between s.End and start is by user key only, as
// the span is exclusive at s.End, so comparing by user keys
@@ -48,13 +53,22 @@ func Truncate(
// Wholly within the end bound. Keep it.
}
}
var truncated bool
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
truncated = true
out.Start = lower
}
if cmp(in.End, upper) > 0 {
truncated = true
out.End = upper
}
if panicOnPartialOverlap && truncated {
panic("pebble: bounds should not be truncated")
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}

324
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored
View File

@@ -12,6 +12,7 @@ import (
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/invariants"
)
// The Annotator type defined below is used by other packages to lazily
@@ -110,7 +111,16 @@ type leafNode struct {
ref int32
count int16
leaf bool
items [maxItems]*FileMetadata
// subtreeCount holds the count of files in the entire subtree formed by
// this node. For leaf nodes, subtreeCount is always equal to count. For
// non-leaf nodes, it's the sum of count plus all the children's
// subtreeCounts.
//
// NB: We could move this field to the end of the node struct, since leaf =>
// count=subtreeCount, however the unsafe casting [leafToNode] performs make
// it risky and cumbersome.
subtreeCount int
items [maxItems]*FileMetadata
// annot contains one annotation per annotator, merged over the entire
// node's files (and all descendants for non-leaf nodes).
annot []annotation
@@ -167,7 +177,7 @@ func mut(n **node) *node {
// reference count to be greater than 1, we might be racing
// with another call to decRef on this node.
c := (*n).clone()
(*n).decRef(true /* recursive */, nil)
(*n).decRef(true /* contentsToo */, nil)
*n = c
// NB: We don't need to clear annotations, because (*node).clone does not
// copy them.
@@ -179,19 +189,25 @@ func (n *node) incRef() {
atomic.AddInt32(&n.ref, 1)
}
// decRef releases a reference to the node. If requested, the method
// will recurse into child nodes and decrease their refcounts as well.
// decRef releases a reference to the node. If requested, the method will unref
// its items and recurse into child nodes and decrease their refcounts as well.
// Some internal codepaths that manually copy the node's items or children to
// new nodes pass contentsToo=false to preserve existing reference counts during
// operations that should yield a net-zero change to descendant refcounts.
// When a node is released, its contained files are dereferenced.
func (n *node) decRef(recursive bool, obsolete *[]*FileMetadata) {
func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if atomic.AddInt32(&n.ref, -1) > 0 {
// Other references remain. Can't free.
return
}
// Dereference the node's metadata and release child references.
if recursive {
// Dereference the node's metadata and release child references if
// requested. Some internal callers may not want to propagate the deref
// because they're manually copying the filemetadata and children to other
// nodes, and they want to preserve the existing reference count.
if contentsToo {
for _, f := range n.items[:n.count] {
if atomic.AddInt32(&f.refs, -1) == 0 {
if f.Unref() == 0 {
// There are two sources of node dereferences: tree mutations
// and Version dereferences. Files should only be made obsolete
// during Version dereferences, during which `obsolete` will be
@@ -204,7 +220,7 @@ func (n *node) decRef(recursive bool, obsolete *[]*FileMetadata) {
}
if !n.leaf {
for i := int16(0); i <= n.count; i++ {
n.children[i].decRef(true /* recursive */, obsolete)
n.children[i].decRef(true /* contentsToo */, obsolete)
}
}
}
@@ -222,9 +238,10 @@ func (n *node) clone() *node {
// triggering the race detector and looking like a data race.
c.count = n.count
c.items = n.items
c.subtreeCount = n.subtreeCount
// Increase the refcount of each contained item.
for _, f := range n.items[:n.count] {
atomic.AddInt32(&f.refs, 1)
f.Ref()
}
if !c.leaf {
// Copy children and increase each refcount.
@@ -236,6 +253,9 @@ func (n *node) clone() *node {
return c
}
// insertAt inserts the provided file and node at the provided index. This
// function is for use only as a helper function for internal B-Tree code.
// Clients should not invoke it directly.
func (n *node) insertAt(index int, item *FileMetadata, nd *node) {
if index < int(n.count) {
copy(n.items[index+1:n.count+1], n.items[index:n.count])
@@ -250,6 +270,9 @@ func (n *node) insertAt(index int, item *FileMetadata, nd *node) {
n.count++
}
// pushBack inserts the provided file and node at the tail of the node's items.
// This function is for use only as a helper function for internal B-Tree code.
// Clients should not invoke it directly.
func (n *node) pushBack(item *FileMetadata, nd *node) {
n.items[n.count] = item
if !n.leaf {
@@ -258,6 +281,9 @@ func (n *node) pushBack(item *FileMetadata, nd *node) {
n.count++
}
// pushFront inserts the provided file and node at the head of the
// node's items. This function is for use only as a helper function for internal B-Tree
// code. Clients should not invoke it directly.
func (n *node) pushFront(item *FileMetadata, nd *node) {
if !n.leaf {
copy(n.children[1:n.count+2], n.children[:n.count+1])
@@ -269,7 +295,8 @@ func (n *node) pushFront(item *FileMetadata, nd *node) {
}
// removeAt removes a value at a given index, pulling all subsequent values
// back.
// back. This function is for use only as a helper function for internal B-Tree
// code. Clients should not invoke it directly.
func (n *node) removeAt(index int) (*FileMetadata, *node) {
var child *node
if !n.leaf {
@@ -284,7 +311,9 @@ func (n *node) removeAt(index int) (*FileMetadata, *node) {
return out, child
}
// popBack removes and returns the last element in the list.
// popBack removes and returns the last element in the list. This function is
// for use only as a helper function for internal B-Tree code. Clients should
// not invoke it directly.
func (n *node) popBack() (*FileMetadata, *node) {
n.count--
out := n.items[n.count]
@@ -297,7 +326,9 @@ func (n *node) popBack() (*FileMetadata, *node) {
return out, child
}
// popFront removes and returns the first element in the list.
// popFront removes and returns the first element in the list. This function is
// for use only as a helper function for internal B-Tree code. Clients should
// not invoke it directly.
func (n *node) popFront() (*FileMetadata, *node) {
n.count--
var child *node
@@ -315,6 +346,9 @@ func (n *node) popFront() (*FileMetadata, *node) {
// find returns the index where the given item should be inserted into this
// list. 'found' is true if the item already exists in the list at the given
// index.
//
// This function is for use only as a helper function for internal B-Tree code.
// Clients should not invoke it directly.
func (n *node) find(cmp btreeCmp, item *FileMetadata) (index int, found bool) {
// Logic copied from sort.Search. Inlining this gave
// an 11% speedup on BenchmarkBTreeDeleteInsert.
@@ -338,23 +372,34 @@ func (n *node) find(cmp btreeCmp, item *FileMetadata) (index int, found bool) {
// and this function returns the item that existed at that index and a new
// node containing all items/children after it.
//
// Before:
// split is called when we want to perform a transformation like the one
// depicted in the following diagram.
//
// +-----------+
// | x y z |
// +--/-/-\-\--+
// Before:
// +-----------+
// n *node | x y z |
// +--/-/-\-\--+
//
// After:
// After:
// +-----------+
// | y | n's parent
// +----/-\----+
// / \
// v v
// +-----------+ +-----------+
// n *node | x | | z | next *node
// +-----------+ +-----------+
//
// +-----------+
// | y |
// +----/-\----+
// / \
// v v
// +-----------+ +-----------+
// | x | | z |
// +-----------+ +-----------+
// split does not perform the complete transformation; the caller is responsible
// for updating the parent appropriately. split splits `n` into two nodes, `n`
// and `next`, returning `next` and the file that separates them. In the diagram
// above, `n.split` removes y and z from `n`, returning y in the first return
// value and `next` in the second return value. The caller is responsible for
// updating n's parent to now contain `y` as the separator between nodes `n` and
// `next`.
//
// This function is for use only as a helper function for internal B-Tree code.
// Clients should not invoke it directly.
func (n *node) split(i int) (*FileMetadata, *node) {
out := n.items[i]
var next *node
@@ -370,17 +415,27 @@ func (n *node) split(i int) (*FileMetadata, *node) {
}
if !n.leaf {
copy(next.children[:], n.children[i+1:n.count+1])
descendantsMoved := 0
for j := int16(i + 1); j <= n.count; j++ {
descendantsMoved += n.children[j].subtreeCount
n.children[j] = nil
}
n.subtreeCount -= descendantsMoved
next.subtreeCount += descendantsMoved
}
n.count = int16(i)
// NB: We subtract one more than `next.count` from n's subtreeCount because
// the item at index `i` was removed from `n.items`. We'll return the item
// at index `i`, and the caller is responsible for updating the subtree
// count of whichever node adopts it.
n.subtreeCount -= int(next.count) + 1
next.subtreeCount += int(next.count)
return out, next
}
// insert inserts a item into the subtree rooted at this node, making sure no
// Insert inserts a item into the subtree rooted at this node, making sure no
// nodes in the subtree exceed maxItems items.
func (n *node) insert(cmp btreeCmp, item *FileMetadata) error {
func (n *node) Insert(cmp btreeCmp, item *FileMetadata) error {
i, found := n.find(cmp, item)
if found {
// cmp provides a total ordering of the files within a level.
@@ -391,6 +446,7 @@ func (n *node) insert(cmp btreeCmp, item *FileMetadata) error {
}
if n.leaf {
n.insertAt(i, item, nil)
n.subtreeCount++
return nil
}
if n.children[i].count >= maxItems {
@@ -410,14 +466,21 @@ func (n *node) insert(cmp btreeCmp, item *FileMetadata) error {
errors.Safe(item.FileNum), errors.Safe(n.items[i].FileNum))
}
}
return mut(&n.children[i]).insert(cmp, item)
err := mut(&n.children[i]).Insert(cmp, item)
if err == nil {
n.subtreeCount++
}
return err
}
// removeMax removes and returns the maximum item from the subtree rooted
// at this node.
// removeMax removes and returns the maximum item from the subtree rooted at
// this node. This function is for use only as a helper function for internal
// B-Tree code. Clients should not invoke it directly.
func (n *node) removeMax() *FileMetadata {
if n.leaf {
n.count--
n.subtreeCount--
out := n.items[n.count]
n.items[n.count] = nil
return out
@@ -427,16 +490,18 @@ func (n *node) removeMax() *FileMetadata {
n.rebalanceOrMerge(int(n.count))
return n.removeMax()
}
n.subtreeCount--
return child.removeMax()
}
// remove removes a item from the subtree rooted at this node. Returns
// Remove removes a item from the subtree rooted at this node. Returns
// the item that was removed or nil if no matching item was found.
func (n *node) remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) {
func (n *node) Remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) {
i, found := n.find(cmp, item)
if n.leaf {
if found {
out, _ = n.removeAt(i)
n.subtreeCount--
return out
}
return nil
@@ -444,22 +509,28 @@ func (n *node) remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) {
if n.children[i].count <= minItems {
// Child not large enough to remove from.
n.rebalanceOrMerge(i)
return n.remove(cmp, item)
return n.Remove(cmp, item)
}
child := mut(&n.children[i])
if found {
// Replace the item being removed with the max item in our left child.
out = n.items[i]
n.items[i] = child.removeMax()
n.subtreeCount--
return out
}
// Latch is not in this node and child is large enough to remove from.
out = child.remove(cmp, item)
// File is not in this node and child is large enough to remove from.
out = child.Remove(cmp, item)
if out != nil {
n.subtreeCount--
}
return out
}
// rebalanceOrMerge grows child 'i' to ensure it has sufficient room to remove
// a item from it while keeping it at or above minItems.
// rebalanceOrMerge grows child 'i' to ensure it has sufficient room to remove a
// item from it while keeping it at or above minItems. This function is for use
// only as a helper function for internal B-Tree code. Clients should not invoke
// it directly.
func (n *node) rebalanceOrMerge(i int) {
switch {
case i > 0 && n.children[i-1].count > minItems:
@@ -497,6 +568,12 @@ func (n *node) rebalanceOrMerge(i int) {
yLa := n.items[i-1]
child.pushFront(yLa, grandChild)
n.items[i-1] = xLa
child.subtreeCount++
left.subtreeCount--
if grandChild != nil {
child.subtreeCount += grandChild.subtreeCount
left.subtreeCount -= grandChild.subtreeCount
}
case i < int(n.count) && n.children[i+1].count > minItems:
// Rebalance from right sibling.
@@ -532,6 +609,12 @@ func (n *node) rebalanceOrMerge(i int) {
xLa, grandChild := right.popFront()
yLa := n.items[i]
child.pushBack(yLa, grandChild)
child.subtreeCount++
right.subtreeCount--
if grandChild != nil {
child.subtreeCount += grandChild.subtreeCount
right.subtreeCount -= grandChild.subtreeCount
}
n.items[i] = xLa
default:
@@ -570,12 +653,15 @@ func (n *node) rebalanceOrMerge(i int) {
copy(child.children[child.count+1:], mergeChild.children[:mergeChild.count+1])
}
child.count += mergeChild.count + 1
child.subtreeCount += mergeChild.subtreeCount + 1
mergeChild.decRef(false /* recursive */, nil)
mergeChild.decRef(false /* contentsToo */, nil)
}
}
func (n *node) invalidateAnnotation(a Annotator) {
// InvalidateAnnotation removes any existing cached annotations for the provided
// annotator from this node's subtree.
func (n *node) InvalidateAnnotation(a Annotator) {
// Find this annotator's annotation on this node.
var annot *annotation
for i := range n.annot {
@@ -590,12 +676,17 @@ func (n *node) invalidateAnnotation(a Annotator) {
}
if !n.leaf {
for i := int16(0); i <= n.count; i++ {
n.children[i].invalidateAnnotation(a)
n.children[i].InvalidateAnnotation(a)
}
}
}
func (n *node) annotation(a Annotator) (interface{}, bool) {
// Annotation retrieves, computing if not already computed, the provided
// annotator's annotation of this node. The second return value indicates
// whether the future reads of this annotation may use the first return value
// as-is. If false, the annotation is not stable and may change on a subsequent
// computation.
func (n *node) Annotation(a Annotator) (interface{}, bool) {
// Find this annotator's annotation on this node.
var annot *annotation
for i := range n.annot {
@@ -627,7 +718,7 @@ func (n *node) annotation(a Annotator) (interface{}, bool) {
annot.valid = true
for i := int16(0); i <= n.count; i++ {
if !n.leaf {
v, ok := n.children[i].annotation(a)
v, ok := n.children[i].Annotation(a)
annot.v = a.Merge(v, annot.v)
annot.valid = annot.valid && ok
}
@@ -640,6 +731,20 @@ func (n *node) annotation(a Annotator) (interface{}, bool) {
return annot.v, annot.valid
}
func (n *node) verifyInvariants() {
recomputedSubtreeCount := int(n.count)
if !n.leaf {
for i := int16(0); i <= n.count; i++ {
n.children[i].verifyInvariants()
recomputedSubtreeCount += n.children[i].subtreeCount
}
}
if recomputedSubtreeCount != n.subtreeCount {
panic(fmt.Sprintf("recomputed subtree count (%d) ≠ n.subtreeCount (%d)",
recomputedSubtreeCount, n.subtreeCount))
}
}
// btree is an implementation of a B-Tree.
//
// btree stores FileMetadata in an ordered structure, allowing easy insertion,
@@ -650,25 +755,23 @@ func (n *node) annotation(a Annotator) (interface{}, bool) {
// Write operations are not safe for concurrent mutation by multiple
// goroutines, but Read operations are.
type btree struct {
root *node
length int
cmp btreeCmp
root *node
cmp btreeCmp
}
// release dereferences and clears the root node of the btree, removing all
// Release dereferences and clears the root node of the btree, removing all
// items from the btree. In doing so, it decrements contained file counts.
// It returns a slice of newly obsolete files, if any.
func (t *btree) release() (obsolete []*FileMetadata) {
func (t *btree) Release() (obsolete []*FileMetadata) {
if t.root != nil {
t.root.decRef(true /* recursive */, &obsolete)
t.root.decRef(true /* contentsToo */, &obsolete)
t.root = nil
}
t.length = 0
return obsolete
}
// clone clones the btree, lazily. It does so in constant time.
func (t *btree) clone() btree {
// Clone clones the btree, lazily. It does so in constant time.
func (t *btree) Clone() btree {
c := *t
if c.root != nil {
// Incrementing the reference count on the root node is sufficient to
@@ -690,15 +793,17 @@ func (t *btree) clone() btree {
return c
}
// delete removes the provided file from the tree.
// Delete removes the provided file from the tree.
// It returns true if the file now has a zero reference count.
func (t *btree) delete(item *FileMetadata) (obsolete bool) {
func (t *btree) Delete(item *FileMetadata) (obsolete bool) {
if t.root == nil || t.root.count == 0 {
return false
}
if out := mut(&t.root).remove(t.cmp, item); out != nil {
t.length--
obsolete = atomic.AddInt32(&out.refs, -1) == 0
if out := mut(&t.root).Remove(t.cmp, item); out != nil {
obsolete = out.Unref() == 0
}
if invariants.Enabled {
t.root.verifyInvariants()
}
if t.root.count == 0 {
old := t.root
@@ -707,14 +812,14 @@ func (t *btree) delete(item *FileMetadata) (obsolete bool) {
} else {
t.root = t.root.children[0]
}
old.decRef(false /* recursive */, nil)
old.decRef(false /* contentsToo */, nil)
}
return obsolete
}
// insert adds the given item to the tree. If a item in the tree already
// equals the given one, insert panics.
func (t *btree) insert(item *FileMetadata) error {
// Insert adds the given item to the tree. If a item in the tree already
// equals the given one, Insert panics.
func (t *btree) Insert(item *FileMetadata) error {
if t.root == nil {
t.root = newLeafNode()
} else if t.root.count >= maxItems {
@@ -724,39 +829,36 @@ func (t *btree) insert(item *FileMetadata) error {
newRoot.items[0] = splitLa
newRoot.children[0] = t.root
newRoot.children[1] = splitNode
newRoot.subtreeCount = t.root.subtreeCount + splitNode.subtreeCount + 1
t.root = newRoot
}
atomic.AddInt32(&item.refs, 1)
err := mut(&t.root).insert(t.cmp, item)
t.length++
item.Ref()
err := mut(&t.root).Insert(t.cmp, item)
if invariants.Enabled {
t.root.verifyInvariants()
}
return err
}
// iter returns a new iterator object. It is not safe to continue using an
// Iter returns a new iterator object. It is not safe to continue using an
// iterator after modifications are made to the tree. If modifications are made,
// create a new iterator.
func (t *btree) iter() iterator {
func (t *btree) Iter() iterator {
return iterator{r: t.root, pos: -1, cmp: t.cmp}
}
// height returns the height of the tree.
func (t *btree) height() int {
// Count returns the number of files contained within the B-Tree.
func (t *btree) Count() int {
if t.root == nil {
return 0
}
h := 1
n := t.root
for !n.leaf {
n = n.children[0]
h++
}
return h
return t.root.subtreeCount
}
// String returns a string description of the tree. The format is
// similar to the https://en.wikipedia.org/wiki/Newick_format.
func (t *btree) String() string {
if t.length == 0 {
if t.Count() == 0 {
return ";"
}
var b strings.Builder
@@ -873,6 +975,8 @@ type iterator struct {
r *node
// n and pos make up the current position of the iterator.
// If valid, n.items[pos] is the current value of the iterator.
//
// n may be nil iff i.r is nil.
n *node
pos int16
// cmp dictates the ordering of the FileMetadata.
@@ -883,6 +987,59 @@ type iterator struct {
s iterStack
}
// countLeft returns the count of files that are to the left of the current
// iterator position.
func (i *iterator) countLeft() int {
if i.r == nil {
return 0
}
// Each iterator has a stack of frames marking the path from the root node
// to the current iterator position. All files (n.items) and all subtrees
// (n.children) with indexes less than [pos] are to the left of the current
// iterator position.
//
// +------------------------+ -
// | Root pos:5 | |
// +------------------------+ | stack
// | Root/5 pos:3 | | frames
// +------------------------+ | [i.s]
// | Root/5/3 pos:9 | |
// +========================+ -
// | |
// | i.n: Root/5/3/9 i.pos:2|
// +------------------------+
//
var count int
// Walk all the ancestors in the iterator stack [i.s], tallying up all the
// files and subtrees to the left of the stack frame's position.
f, ok := i.s.nth(0)
for fi := 0; ok; fi++ {
// There are [f.pos] files contained within [f.n.items] that sort to the
// left of the subtree the iterator has descended.
count += int(f.pos)
// Any subtrees that fall before the stack frame's position are entirely
// to the left of the iterator's current position.
for j := int16(0); j < f.pos; j++ {
count += f.n.children[j].subtreeCount
}
f, ok = i.s.nth(fi + 1)
}
// The bottommost stack frame is inlined within the iterator struct. Again,
// [i.pos] files fall to the left of the current iterator position.
count += int(i.pos)
if !i.n.leaf {
// NB: Unlike above, we use a `<= i.pos` comparison. The iterator is
// positioned at item `i.n.items[i.pos]`, which sorts after everything
// in the subtree at `i.n.children[i.pos]`.
for j := int16(0); j <= i.pos; j++ {
count += i.n.children[j].subtreeCount
}
}
return count
}
func (i *iterator) clone() iterator {
c := *i
c.s = i.s.clone()
@@ -904,7 +1061,7 @@ func (i iterator) String() string {
}
fmt.Fprintf(&buf, "%p: %02d/%02d\n", f.n, f.pos, f.n.count)
}
if i.n == nil {
if i.r == nil {
fmt.Fprintf(&buf, "<nil>: %02d", i.pos)
} else {
fmt.Fprintf(&buf, "%p: %02d/%02d", i.n, i.pos, i.n.count)
@@ -1034,7 +1191,7 @@ func (i *iterator) ascend() {
// tree's files, and then true for the (possibly empty) remainder.
func (i *iterator) seek(fn func(*FileMetadata) bool) {
i.reset()
if i.n == nil {
if i.r == nil {
return
}
@@ -1066,7 +1223,7 @@ func (i *iterator) seek(fn func(*FileMetadata) bool) {
// first seeks to the first item in the btree.
func (i *iterator) first() {
i.reset()
if i.n == nil {
if i.r == nil {
return
}
for !i.n.leaf {
@@ -1078,7 +1235,7 @@ func (i *iterator) first() {
// last seeks to the last item in the btree.
func (i *iterator) last() {
i.reset()
if i.n == nil {
if i.r == nil {
return
}
for !i.n.leaf {
@@ -1090,7 +1247,7 @@ func (i *iterator) last() {
// next positions the iterator to the item immediately following
// its current position.
func (i *iterator) next() {
if i.n == nil {
if i.r == nil {
return
}
@@ -1117,7 +1274,7 @@ func (i *iterator) next() {
// prev positions the iterator to the item immediately preceding
// its current position.
func (i *iterator) prev() {
if i.n == nil {
if i.r == nil {
return
}
@@ -1148,5 +1305,8 @@ func (i *iterator) valid() bool {
// cur returns the item at the iterator's current position. It is illegal
// to call cur if the iterator is not valid.
func (i *iterator) cur() *FileMetadata {
if invariants.Enabled && !i.valid() {
panic("btree iterator.cur invoked on invalid iterator")
}
return i.n.items[i.pos]
}

609
vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go generated vendored
View File

@@ -112,13 +112,15 @@ func sortAndSweep(keys []intervalKeyTemp, cmp Compare) []intervalKeyTemp {
sorter := intervalKeySorter{keys: keys, cmp: cmp}
sort.Sort(sorter)
// intervalKeys are generated using the file bounds. Specifically, there are 2 intervalKeys
// for each file, and len(keys) = 2 * number of files. Each intervalKeyTemp stores information
// about which file it was generated from, and whether the key represents the end key of the file.
// So, as we're deduplicating the `keys` slice, we're guaranteed to iterate over the interval
// keys belonging to each of the files. Since the file.{min,max}IntervalIndex points to the position
// of the files bounds in the deduplicated `keys` slice, we can determine file.{min,max}IntervalIndex
// during the iteration.
// intervalKeys are generated using the file bounds. Specifically, there are
// 2 intervalKeys for each file, and len(keys) = 2 * number of files. Each
// `intervalKeyTemp` stores information about which file it was generated
// from, and whether the key represents the end key of the file. So, as
// we're deduplicating the `keys` slice, we're guaranteed to iterate over
// the interval keys belonging to each of the files. Since the
// file.{min,max}IntervalIndex points to the position of the files bounds in
// the deduplicated `keys` slice, we can determine
// file.{min,max}IntervalIndex during the iteration.
i := 0
j := 0
for i < len(keys) {
@@ -151,24 +153,24 @@ type fileInterval struct {
// cannot have any files participate in L0 -> Lbase compactions.
isBaseCompacting bool
// The min and max intervals index across all the files that overlap with this
// interval. Inclusive on both sides.
// The min and max intervals index across all the files that overlap with
// this interval. Inclusive on both sides.
filesMinIntervalIndex int
filesMaxIntervalIndex int
// True if another interval that has a file extending into this interval is
// undergoing a compaction into Lbase. In other words, this bool is true
// if any interval in [filesMinIntervalIndex,
// filesMaxIntervalIndex] has isBaseCompacting set to true. This
// lets the compaction picker de-prioritize this interval for picking
// compactions, since there's a high chance that a base compaction with a
// sufficient height of sublevels rooted at this interval could not be
// chosen due to the ongoing base compaction in the
// other interval. If the file straddling the two intervals is at a
// sufficiently high sublevel (with enough compactible files below it to
// satisfy minCompactionDepth), this is not an issue, but to optimize for
// quickly picking base compactions far away from other base compactions,
// this bool is used as a heuristic (but not as a complete disqualifier).
// undergoing a compaction into Lbase. In other words, this bool is true if
// any interval in [filesMinIntervalIndex, filesMaxIntervalIndex] has
// isBaseCompacting set to true. This lets the compaction picker
// de-prioritize this interval for picking compactions, since there's a high
// chance that a base compaction with a sufficient height of sublevels
// rooted at this interval could not be chosen due to the ongoing base
// compaction in the other interval. If the file straddling the two
// intervals is at a sufficiently high sublevel (with enough compactible
// files below it to satisfy minCompactionDepth), this is not an issue, but
// to optimize for quickly picking base compactions far away from other base
// compactions, this bool is used as a heuristic (but not as a complete
// disqualifier).
intervalRangeIsBaseCompacting bool
// All files in this interval, in increasing sublevel order.
@@ -178,9 +180,8 @@ type fileInterval struct {
// starting new compactions. This metric is not precise since the
// compactingFileCount can include files that are part of N (where N > 1)
// intra-L0 compactions, so the stack depth after those complete will be
// len(files) - compactingFileCount + N. We ignore this imprecision since
// we don't want to track which files are part of which intra-L0
// compaction.
// len(files) - compactingFileCount + N. We ignore this imprecision since we
// don't want to track which files are part of which intra-L0 compaction.
compactingFileCount int
// Interpolated from files in this interval. For files spanning multiple
@@ -332,17 +333,17 @@ func NewL0Sublevels(
for _, sublevelFiles := range s.levelFiles {
tr, ls := makeBTree(btreeCmpSmallestKey(cmp), sublevelFiles)
s.Levels = append(s.Levels, ls)
tr.release()
tr.Release()
}
s.calculateFlushSplitKeys(flushSplitMaxBytes)
return s, nil
}
// Helper function to merge new intervalKeys into an existing slice
// of old fileIntervals, into result. Returns the new result and a slice of ints
// mapping old interval indices to new ones. The added intervalKeys do not
// need to be sorted; they get sorted and deduped in this function.
// Helper function to merge new intervalKeys into an existing slice of old
// fileIntervals, into result. Returns the new result and a slice of ints
// mapping old interval indices to new ones. The added intervalKeys do not need
// to be sorted; they get sorted and deduped in this function.
func mergeIntervals(
old, result []fileInterval, added []intervalKeyTemp, compare Compare,
) ([]fileInterval, []int) {
@@ -413,35 +414,36 @@ func mergeIntervals(
return result, oldToNewMap
}
// AddL0Files incrementally builds a new L0Sublevels for when the only
// change since the receiver L0Sublevels was an addition of the specified files,
// with no L0 deletions. The common case of this is an ingestion or a flush.
// These files can "sit on top" of existing sublevels, creating at most one
// new sublevel for a flush (and possibly multiple for an ingestion), and at
// most 2*len(files) additions to s.orderedIntervals. No files must have been
// deleted from L0, and the added files must all be newer in sequence numbers
// than existing files in L0Sublevels. The files parameter must be sorted in
// seqnum order. The levelMetadata parameter corresponds to the new L0 post
// addition of files. This method is meant to be significantly more performant
// than NewL0Sublevels.
// AddL0Files incrementally builds a new L0Sublevels for when the only change
// since the receiver L0Sublevels was an addition of the specified files, with
// no L0 deletions. The common case of this is an ingestion or a flush. These
// files can "sit on top" of existing sublevels, creating at most one new
// sublevel for a flush (and possibly multiple for an ingestion), and at most
// 2*len(files) additions to s.orderedIntervals. No files must have been deleted
// from L0, and the added files must all be newer in sequence numbers than
// existing files in L0Sublevels. The files parameter must be sorted in seqnum
// order. The levelMetadata parameter corresponds to the new L0 post addition of
// files. This method is meant to be significantly more performant than
// NewL0Sublevels.
//
// Note that this function can only be called once on a given receiver; it
// appends to some slices in s which is only safe when done once. This is okay,
// as the common case (generating a new L0Sublevels after a flush/ingestion) is
// only going to necessitate one call of this method on a given receiver. The
// returned value, if non-nil, can then have AddL0Files called on it again, and
// so on. If errInvalidL0SublevelsOpt is returned as an error, it likely means
// the optimization could not be applied (i.e. files added were older than files
// already in the sublevels, which is possible around ingestions and in tests).
// Eg. it can happen when an ingested file was ingested without queueing a flush
// since it did not actually overlap with any keys in the memtable. Later on the
// memtable was flushed, and the memtable had keys spanning around the ingested
// file, producing a flushed file that overlapped with the ingested file in file
// bounds but not in keys. It's possible for that flushed file to have a lower
// LargestSeqNum than the ingested file if all the additions after the ingestion
// were to another flushed file that was split into a separate sstable during
// flush. Any other non-nil error means L0Sublevels generation failed in the same
// way as NewL0Sublevels would likely fail.
// returned value, if non-nil, can then have [*L0Sublevels.AddL0Files] called on
// it again, and so on. If [errInvalidL0SublevelsOpt] is returned as an error,
// it likely means the optimization could not be applied (i.e. files added were
// older than files already in the sublevels, which is possible around
// ingestions and in tests). Eg. it can happen when an ingested file was
// ingested without queueing a flush since it did not actually overlap with any
// keys in the memtable. Later on the memtable was flushed, and the memtable had
// keys spanning around the ingested file, producing a flushed file that
// overlapped with the ingested file in file bounds but not in keys. It's
// possible for that flushed file to have a lower LargestSeqNum than the
// ingested file if all the additions after the ingestion were to another
// flushed file that was split into a separate sstable during flush. Any other
// non-nil error means [L0Sublevels] generation failed in the same way as
// [NewL0Sublevels] would likely fail.
func (s *L0Sublevels) AddL0Files(
files []*FileMetadata, flushSplitMaxBytes int64, levelMetadata *LevelMetadata,
) (*L0Sublevels, error) {
@@ -485,8 +487,8 @@ func (s *L0Sublevels) AddL0Files(
keys := make([]fileInterval, 0, 2*levelMetadata.Len())
var oldToNewMap []int
// We can avoid the sortAndSweep step on the combined length of
// s.orderedIntervals and fileKeys by treating this as a merge of two
// sorted runs, fileKeys and s.orderedIntervals, into `keys` which will form
// s.orderedIntervals and fileKeys by treating this as a merge of two sorted
// runs, fileKeys and s.orderedIntervals, into `keys` which will form
// newVal.orderedIntervals.
keys, oldToNewMap = mergeIntervals(s.orderedIntervals, keys, fileKeys, s.cmp)
if invariants.Enabled {
@@ -502,20 +504,21 @@ func (s *L0Sublevels) AddL0Files(
newInterval := &keys[newIdx]
newInterval.index = newIdx
// This code, and related code in the for loop below, adjusts
// files{Min,Max}IntervalIndex just for interval indices shifting due to new
// intervals, and not for any of the new files being added to the same
// intervals. The goal is to produce a state of the system that's accurate
// for all existing files, and has all the new intervals to support new
// files. Once that's done, we can just call addFileToSublevel to adjust
// all relevant intervals for new files.
// files{Min,Max}IntervalIndex just for interval indices shifting due to
// new intervals, and not for any of the new files being added to the
// same intervals. The goal is to produce a state of the system that's
// accurate for all existing files, and has all the new intervals to
// support new files. Once that's done, we can just call
// addFileToSublevel to adjust all relevant intervals for new files.
newInterval.filesMinIntervalIndex = oldToNewMap[newInterval.filesMinIntervalIndex]
// maxIntervalIndexes are special. Since it's an inclusive end bound, we
// actually have to map it to the _next_ old interval's new previous
// interval. This logic is easier to understand if you see
// [f.minIntervalIndex, f.maxIntervalIndex] as [f.minIntervalIndex,
// f.maxIntervalIndex+1). The other case to remember is when the interval is
// completely empty (i.e. len(newInterval.files) == 0); in that case we want
// to refer back to ourselves regardless of additions to the right of us.
// f.maxIntervalIndex+1). The other case to remember is when the
// interval is completely empty (i.e. len(newInterval.files) == 0); in
// that case we want to refer back to ourselves regardless of additions
// to the right of us.
if newInterval.filesMaxIntervalIndex < len(oldToNewMap)-1 && len(newInterval.files) > 0 {
newInterval.filesMaxIntervalIndex = oldToNewMap[newInterval.filesMaxIntervalIndex+1] - 1
} else {
@@ -523,16 +526,16 @@ func (s *L0Sublevels) AddL0Files(
newInterval.filesMaxIntervalIndex = oldToNewMap[newInterval.filesMaxIntervalIndex]
}
}
// Loop through all instances of new intervals added between two old intervals
// and expand [filesMinIntervalIndex, filesMaxIntervalIndex] of new intervals
// to reflect that of adjacent old intervals.
// Loop through all instances of new intervals added between two old
// intervals and expand [filesMinIntervalIndex, filesMaxIntervalIndex] of
// new intervals to reflect that of adjacent old intervals.
{
// We can skip cases where new intervals were added to the left of all
// existing intervals (eg. if the first entry in oldToNewMap is
// oldToNewMap[0] >= 1). Those intervals will only contain newly added files
// and will have their parameters adjusted down in addFileToSublevels. The
// same can also be said about new intervals that are to the right of all
// existing intervals.
// oldToNewMap[0] >= 1). Those intervals will only contain newly added
// files and will have their parameters adjusted down in
// addFileToSublevels. The same can also be said about new intervals
// that are to the right of all existing intervals.
lastIdx := 0
for _, newIdx := range oldToNewMap {
for i := lastIdx + 1; i < newIdx; i++ {
@@ -555,28 +558,29 @@ func (s *L0Sublevels) AddL0Files(
// TODO(bilal): This is the only place in this method where we loop through
// all existing files, which could be much more in number than newly added
// files. See if we can avoid the need for this, either by getting rid of
// f.minIntervalIndex and f.maxIntervalIndex and calculating them on the
// fly with a binary search, or by only looping through files to the right
// of the first interval touched by this method.
// f.minIntervalIndex and f.maxIntervalIndex and calculating them on the fly
// with a binary search, or by only looping through files to the right of
// the first interval touched by this method.
for sublevel := range s.Levels {
s.Levels[sublevel].Each(func(f *FileMetadata) {
oldIntervalDelta := f.maxIntervalIndex - f.minIntervalIndex + 1
oldMinIntervalIndex := f.minIntervalIndex
f.minIntervalIndex = oldToNewMap[f.minIntervalIndex]
// maxIntervalIndex is special. Since it's an inclusive end bound, we
// actually have to map it to the _next_ old interval's new previous
// interval. This logic is easier to understand if you see
// [f.minIntervalIndex, f.maxIntervalIndex] as
// [f.minIntervalIndex, f.maxIntervalIndex+1).
// maxIntervalIndex is special. Since it's an inclusive end bound,
// we actually have to map it to the _next_ old interval's new
// previous interval. This logic is easier to understand if you see
// [f.minIntervalIndex, f.maxIntervalIndex] as [f.minIntervalIndex,
// f.maxIntervalIndex+1).
f.maxIntervalIndex = oldToNewMap[f.maxIntervalIndex+1] - 1
newIntervalDelta := f.maxIntervalIndex - f.minIntervalIndex + 1
// Recalculate estimatedBytes for all old files across new intervals, but
// only if new intervals were added in between.
// Recalculate estimatedBytes for all old files across new
// intervals, but only if new intervals were added in between.
if oldIntervalDelta != newIntervalDelta {
// j is incremented so that oldToNewMap[j] points to the next old
// interval. This is used to distinguish between old intervals (i.e.
// ones where we need to subtract f.Size/oldIntervalDelta) from new
// ones (where we don't need to subtract). In both cases we need to add
// j is incremented so that oldToNewMap[j] points to the next
// old interval. This is used to distinguish between old
// intervals (i.e. ones where we need to subtract
// f.Size/oldIntervalDelta) from new ones (where we don't need
// to subtract). In both cases we need to add
// f.Size/newIntervalDelta.
j := oldMinIntervalIndex
for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
@@ -628,7 +632,7 @@ func (s *L0Sublevels) AddL0Files(
// populated correctly.
newVal.Levels[sublevel] = ls
}
tr.release()
tr.Release()
}
newVal.flushSplitUserKeys = nil
@@ -636,12 +640,12 @@ func (s *L0Sublevels) AddL0Files(
return newVal, nil
}
// addFileToSublevels is called during L0Sublevels generation, and adds f to
// the correct sublevel's levelFiles, the relevant intervals' files slices, and
// sets interval indices on f. This method, if called successively on multiple
// files, _must_ be called on successively newer files (by seqnum). If
// checkInvariant is true, it could check for this in some cases and return
// errInvalidL0SublevelsOpt if that invariant isn't held.
// addFileToSublevels is called during L0Sublevels generation, and adds f to the
// correct sublevel's levelFiles, the relevant intervals' files slices, and sets
// interval indices on f. This method, if called successively on multiple files,
// _must_ be called on successively newer files (by seqnum). If checkInvariant
// is true, it could check for this in some cases and return
// [errInvalidL0SublevelsOpt] if that invariant isn't held.
func (s *L0Sublevels) addFileToSublevels(f *FileMetadata, checkInvariant bool) error {
// This is a simple and not very accurate estimate of the number of
// bytes this SSTable contributes to the intervals it is a part of.
@@ -654,14 +658,18 @@ func (s *L0Sublevels) addFileToSublevels(f *FileMetadata, checkInvariant bool) e
// Update state in every fileInterval for this file.
for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
interval := &s.orderedIntervals[i]
if len(interval.files) > 0 &&
subLevel <= interval.files[len(interval.files)-1].SubLevel {
if len(interval.files) > 0 {
if checkInvariant && interval.files[len(interval.files)-1].LargestSeqNum > f.LargestSeqNum {
// We are sliding this file "underneath" an existing file. Throw away
// and start over in NewL0Sublevels.
return errInvalidL0SublevelsOpt
}
subLevel = interval.files[len(interval.files)-1].SubLevel + 1
// interval.files is sorted by sublevels, from lowest to highest.
// AddL0Files can only add files at sublevels higher than existing files
// in the same key intervals.
if maxSublevel := interval.files[len(interval.files)-1].SubLevel; subLevel <= maxSublevel {
subLevel = maxSublevel + 1
}
}
interval.estimatedBytes += interpolatedBytes
if f.minIntervalIndex < interval.filesMinIntervalIndex {
@@ -738,11 +746,10 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
}
}
// Some intervals may be base compacting without the files contained
// within those intervals being marked as compacting. This is possible if
// the files were added after the compaction initiated, and the active
// compaction files straddle the input file. Mark these intervals as base
// compacting.
// Some intervals may be base compacting without the files contained within
// those intervals being marked as compacting. This is possible if the files
// were added after the compaction initiated, and the active compaction
// files straddle the input file. Mark these intervals as base compacting.
for _, c := range inProgress {
startIK := intervalKey{key: c.Smallest.UserKey, isLargest: false}
endIK := intervalKey{key: c.Largest.UserKey, isLargest: !c.Largest.IsExclusiveSentinel()}
@@ -924,35 +931,35 @@ func (s *L0Sublevels) InUseKeyRanges(smallest, largest []byte) []UserKeyRange {
curr = &keyRanges[len(keyRanges)-1]
}
// If the filesMaxIntervalIndex is not the current index, we can jump
// to the max index, knowing that all intermediary intervals are
// overlapped by some file.
// If the filesMaxIntervalIndex is not the current index, we can jump to
// the max index, knowing that all intermediary intervals are overlapped
// by some file.
if maxIdx := s.orderedIntervals[i].filesMaxIntervalIndex; maxIdx != i {
// Note that end may be less than or equal to maxIdx if we're
// concerned with a key range that ends before the interval at
// maxIdx starts. We must set curr.End now, before making that
// leap, because this iteration may be the last.
// maxIdx starts. We must set curr.End now, before making that leap,
// because this iteration may be the last.
i = maxIdx
curr.End = s.orderedIntervals[i+1].startKey.key
continue
}
// No files overlapping with this interval overlap with the next
// interval. Update the current end to be the next interval's start
// key. Note that curr is not necessarily finished, because there may
// be an abutting non-empty interval.
// interval. Update the current end to be the next interval's start key.
// Note that curr is not necessarily finished, because there may be an
// abutting non-empty interval.
curr.End = s.orderedIntervals[i+1].startKey.key
i++
}
return keyRanges
}
// FlushSplitKeys returns a slice of user keys to split flushes at.
// Used by flushes to avoid writing sstables that straddle these split keys.
// These should be interpreted as the keys to start the next sstable (not the
// last key to include in the prev sstable). These are user keys so that
// range tombstones can be properly truncated (untruncated range tombstones
// are not permitted for L0 files).
// FlushSplitKeys returns a slice of user keys to split flushes at. Used by
// flushes to avoid writing sstables that straddle these split keys. These
// should be interpreted as the keys to start the next sstable (not the last key
// to include in the prev sstable). These are user keys so that range tombstones
// can be properly truncated (untruncated range tombstones are not permitted for
// L0 files).
func (s *L0Sublevels) FlushSplitKeys() [][]byte {
return s.flushSplitUserKeys
}
@@ -978,6 +985,7 @@ func (s *L0Sublevels) MaxDepthAfterOngoingCompactions() int {
//
// TODO(bilal): Simplify away the debugging statements in this method, and make
// this a pure sanity checker.
//
//lint:ignore U1000 - useful for debugging
func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error {
includedFiles := newBitSet(s.levelMetadata.Len())
@@ -1074,7 +1082,7 @@ func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error {
// recently started compaction. isBase specifies if this is a base compaction;
// if false, this is assumed to be an intra-L0 compaction. The specified
// compaction must be involving L0 SSTables. It's assumed that the Compacting
// and IsIntraL0Compacting fields are already set on all FileMetadatas passed
// and IsIntraL0Compacting fields are already set on all [FileMetadata]s passed
// in.
func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs []LevelSlice, isBase bool) error {
minIntervalIndex := -1
@@ -1106,11 +1114,10 @@ func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs []LevelSlice, isBas
return nil
}
// L0CompactionFiles represents a candidate set of L0 files for compaction.
// Also referred to as "lcf". Contains state information useful
// for generating the compaction (such as Files), as well as for picking
// between candidate compactions (eg. fileBytes and
// seedIntervalStackDepthReduction).
// L0CompactionFiles represents a candidate set of L0 files for compaction. Also
// referred to as "lcf". Contains state information useful for generating the
// compaction (such as Files), as well as for picking between candidate
// compactions (eg. fileBytes and seedIntervalStackDepthReduction).
type L0CompactionFiles struct {
Files []*FileMetadata
@@ -1146,6 +1153,23 @@ type L0CompactionFiles struct {
filesAdded []*FileMetadata
}
// Clone allocates a new L0CompactionFiles, with the same underlying data. Note
// that the two fileMetadata slices contain values that point to the same
// underlying fileMetadata object. This is safe because these objects are read
// only.
func (l *L0CompactionFiles) Clone() *L0CompactionFiles {
oldLcf := *l
return &oldLcf
}
// String merely prints the starting address of the first file, if it exists.
func (l *L0CompactionFiles) String() string {
if len(l.Files) > 0 {
return fmt.Sprintf("First File Address: %p", &l.Files[0])
}
return ""
}
// addFile adds the specified file to the LCF.
func (l *L0CompactionFiles) addFile(f *FileMetadata) {
if l.FilesIncluded[f.L0Index] {
@@ -1180,12 +1204,11 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
// Compactions:
//
// The sub-levels and intervals can be visualized in 2 dimensions as the X
// axis containing intervals in increasing order and the Y axis containing
// sub-levels (older to younger). The intervals can be sparse wrt sub-levels.
// We observe that the system is typically under severe pressure in L0 during
// large numbers of ingestions where most files added to L0 are narrow and
// non-overlapping.
// The sub-levels and intervals can be visualized in 2 dimensions as the X axis
// containing intervals in increasing order and the Y axis containing sub-levels
// (older to younger). The intervals can be sparse wrt sub-levels. We observe
// that the system is typically under severe pressure in L0 during large numbers
// of ingestions where most files added to L0 are narrow and non-overlapping.
//
// L0.1 d---g
// L0.0 c--e g--j o--s u--x
@@ -1203,23 +1226,23 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
// it amenable to concurrent L0 -> Lbase compactions.
//
// L0 -> Lbase: The high-level goal of a L0 -> Lbase compaction is to reduce
// stack depth, by compacting files in the intervals with the highest
// (fileCount - compactingCount). Additionally, we would like compactions to
// not involve a huge number of files, so that they finish quickly, and to
// allow for concurrent L0 -> Lbase compactions when needed. In order to
// achieve these goals we would like compactions to visualize as capturing
// thin and tall rectangles. The approach below is to consider intervals in
// some order and then try to construct a compaction using the interval. The
// first interval we can construct a compaction for is the compaction that is
// started. There can be multiple heuristics in choosing the ordering of the
// intervals -- the code uses one heuristic that worked well for a large
// ingestion stemming from a cockroachdb import, but additional experimentation
// is necessary to pick a general heuristic. Additionally, the compaction that
// gets picked may be not as desirable as one that could be constructed later
// in terms of reducing stack depth (since adding more files to the compaction
// can get blocked by needing to encompass files that are already being
// compacted). So an alternative would be to try to construct more than one
// compaction and pick the best one.
// stack depth, by compacting files in the intervals with the highest (fileCount
// - compactingCount). Additionally, we would like compactions to not involve a
// huge number of files, so that they finish quickly, and to allow for
// concurrent L0 -> Lbase compactions when needed. In order to achieve these
// goals we would like compactions to visualize as capturing thin and tall
// rectangles. The approach below is to consider intervals in some order and
// then try to construct a compaction using the interval. The first interval we
// can construct a compaction for is the compaction that is started. There can
// be multiple heuristics in choosing the ordering of the intervals -- the code
// uses one heuristic that worked well for a large ingestion stemming from a
// cockroachdb import, but additional experimentation is necessary to pick a
// general heuristic. Additionally, the compaction that gets picked may be not
// as desirable as one that could be constructed later in terms of reducing
// stack depth (since adding more files to the compaction can get blocked by
// needing to encompass files that are already being compacted). So an
// alternative would be to try to construct more than one compaction and pick
// the best one.
//
// Here's a visualization of an ideal L0->LBase compaction selection:
//
@@ -1243,9 +1266,9 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
// Lbase a--------i m---------w
//
// Note that running this compaction will mark the a--i file in Lbase as
// compacting, and when ExtendL0ForBaseCompactionTo is called with the bounds
// of that base file, it'll expand the compaction to also include all L0 files
// in the a-d interval. The resultant compaction would then be:
// compacting, and when ExtendL0ForBaseCompactionTo is called with the bounds of
// that base file, it'll expand the compaction to also include all L0 files in
// the a-d interval. The resultant compaction would then be:
//
// _____________
// L0.3 |a--d g-j|
@@ -1255,11 +1278,10 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
//
// Lbase a--------i m---------w
//
// The next best interval for base compaction would therefore
// be the one including r--t in L0.2 and p--x in L0.0, and both this compaction
// and the one picked earlier can run in parallel. This is assuming
// minCompactionDepth >= 2, otherwise the second compaction has too little
// depth to pick.
// The next best interval for base compaction would therefore be the one
// including r--t in L0.2 and p--x in L0.0, and both this compaction and the one
// picked earlier can run in parallel. This is assuming minCompactionDepth >= 2,
// otherwise the second compaction has too little depth to pick.
//
// _____________
// L0.3 |a--d g-j| _________
@@ -1292,24 +1314,23 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) {
//
// Lbase a--------ij--lm---------w
//
// Intra-L0: If the L0 score is high, but PickBaseCompaction() is unable to
// pick a compaction, PickIntraL0Compaction will be used to pick an intra-L0
// compaction. Similar to L0 -> Lbase compactions, we want to allow for
// multiple intra-L0 compactions and not generate wide output files that
// hinder later concurrency of L0 -> Lbase compactions. Also compactions
// that produce wide files don't reduce stack depth -- they represent wide
// rectangles in our visualization, which means many intervals have their
// depth reduced by a small amount. Typically, L0 files have non-overlapping
// sequence numbers, and sticking to that invariant would require us to
// consider intra-L0 compactions that proceed from youngest to oldest files,
// which could result in the aforementioned undesirable wide rectangle
// shape. But this non-overlapping sequence number is already relaxed in
// RocksDB -- sstables are primarily ordered by their largest sequence
// number. So we can arrange for intra-L0 compactions to capture thin and
// tall rectangles starting with the top of the stack (youngest files).
// Like the L0 -> Lbase case we order the intervals using a heuristic and
// consider each in turn. The same comment about better L0 -> Lbase heuristics
// and not being greedy applies here.
// Intra-L0: If the L0 score is high, but PickBaseCompaction() is unable to pick
// a compaction, PickIntraL0Compaction will be used to pick an intra-L0
// compaction. Similar to L0 -> Lbase compactions, we want to allow for multiple
// intra-L0 compactions and not generate wide output files that hinder later
// concurrency of L0 -> Lbase compactions. Also compactions that produce wide
// files don't reduce stack depth -- they represent wide rectangles in our
// visualization, which means many intervals have their depth reduced by a small
// amount. Typically, L0 files have non-overlapping sequence numbers, and
// sticking to that invariant would require us to consider intra-L0 compactions
// that proceed from youngest to oldest files, which could result in the
// aforementioned undesirable wide rectangle shape. But this non-overlapping
// sequence number is already relaxed in RocksDB -- sstables are primarily
// ordered by their largest sequence number. So we can arrange for intra-L0
// compactions to capture thin and tall rectangles starting with the top of the
// stack (youngest files). Like the L0 -> Lbase case we order the intervals
// using a heuristic and consider each in turn. The same comment about better L0
// -> Lbase heuristics and not being greedy applies here.
//
// Going back to a modified version of our example from earlier, let's say these
// are the base compactions in progress:
@@ -1387,10 +1408,9 @@ func (s *L0Sublevels) PickBaseCompaction(
// Pick the seed file for the interval as the file
// in the lowest sub-level.
f := interval.files[0]
// Don't bother considering the intervals that are
// covered by the seed file since they are likely
// nearby. Note that it is possible that those intervals
// have seed files at lower sub-levels so could be
// Don't bother considering the intervals that are covered by the seed
// file since they are likely nearby. Note that it is possible that
// those intervals have seed files at lower sub-levels so could be
// viable for compaction.
if f == nil {
return nil, errors.New("no seed file found in sublevel intervals")
@@ -1398,23 +1418,22 @@ func (s *L0Sublevels) PickBaseCompaction(
consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1)
if f.IsCompacting() {
if f.IsIntraL0Compacting {
// If we're picking a base compaction and we came across a
// seed file candidate that's being intra-L0 compacted, skip
// the interval instead of erroring out.
// If we're picking a base compaction and we came across a seed
// file candidate that's being intra-L0 compacted, skip the
// interval instead of erroring out.
continue
}
// We chose a compaction seed file that should not be
// compacting. Usually means the score is not accurately
// accounting for files already compacting, or internal state is
// inconsistent.
// We chose a compaction seed file that should not be compacting.
// Usually means the score is not accurately accounting for files
// already compacting, or internal state is inconsistent.
return nil, errors.Errorf("file %s chosen as seed file for compaction should not be compacting", f.FileNum)
}
c := s.baseCompactionUsingSeed(f, interval.index, minCompactionDepth)
if c != nil {
// Check if the chosen compaction overlaps with any files
// in Lbase that have Compacting = true. If that's the case,
// this compaction cannot be chosen.
// Check if the chosen compaction overlaps with any files in Lbase
// that have Compacting = true. If that's the case, this compaction
// cannot be chosen.
baseIter := baseFiles.Iter()
// An interval starting at ImmediateSuccessor(key) can never be the
// first interval of a compaction since no file can start at that
@@ -1454,10 +1473,9 @@ func (s *L0Sublevels) baseCompactionUsingSeed(
c.addFile(f)
// The first iteration of this loop builds the compaction at the seed file's
// sublevel. Future iterations expand on this compaction by stacking
// more files from intervalIndex and repeating. This is an
// optional activity so when it fails we can fallback to the last
// successful candidate.
// sublevel. Future iterations expand on this compaction by stacking more
// files from intervalIndex and repeating. This is an optional activity so
// when it fails we can fallback to the last successful candidate.
var lastCandidate *L0CompactionFiles
interval := &s.orderedIntervals[intervalIndex]
@@ -1467,29 +1485,28 @@ func (s *L0Sublevels) baseCompactionUsingSeed(
c.seedIntervalStackDepthReduction++
c.seedIntervalMaxLevel = sl
c.addFile(f2)
// The seed file is in the lowest sublevel in the seed interval, but it may
// overlap with other files in even lower sublevels. For
// correctness we need to grow our interval to include those files, and
// capture all files in the next level that fall in this extended interval
// and so on. This can result in a triangular shape like the following
// where again the X axis is the key intervals and the Y axis
// is oldest to youngest. Note that it is not necessary for
// correctness to fill out the shape at the higher sub-levels
// to make it more rectangular since the invariant only requires
// that younger versions of a key not be moved to Lbase while
// leaving behind older versions.
// The seed file is in the lowest sublevel in the seed interval, but it
// may overlap with other files in even lower sublevels. For correctness
// we need to grow our interval to include those files, and capture all
// files in the next level that fall in this extended interval and so
// on. This can result in a triangular shape like the following where
// again the X axis is the key intervals and the Y axis is oldest to
// youngest. Note that it is not necessary for correctness to fill out
// the shape at the higher sub-levels to make it more rectangular since
// the invariant only requires that younger versions of a key not be
// moved to Lbase while leaving behind older versions.
// -
// ---
// -----
// It may be better for performance to have a more rectangular
// shape since the files being left behind will overlap with the
// same Lbase key range as that of this compaction. But there is
// also the danger that in trying to construct a more rectangular
// shape we will be forced to pull in a file that is already
// compacting. We expect extendCandidateToRectangle to eventually be called
// on this compaction if it's chosen, at which point we would iterate
// backward and choose those files. This logic is similar to compaction.grow
// for non-L0 compactions.
// It may be better for performance to have a more rectangular shape
// since the files being left behind will overlap with the same Lbase
// key range as that of this compaction. But there is also the danger
// that in trying to construct a more rectangular shape we will be
// forced to pull in a file that is already compacting. We expect
// extendCandidateToRectangle to eventually be called on this compaction
// if it's chosen, at which point we would iterate backward and choose
// those files. This logic is similar to compaction.grow for non-L0
// compactions.
done := false
for currLevel := sl - 1; currLevel >= 0; currLevel-- {
if !s.extendFiles(currLevel, math.MaxUint64, c) {
@@ -1509,8 +1526,8 @@ func (s *L0Sublevels) baseCompactionUsingSeed(
// depth, but long running compactions reduce flexibility in what can
// run concurrently in L0 and even Lbase -> Lbase+1. An increase more
// than 150% in bytes since the last candidate compaction (along with a
// total compaction size in excess of 100mb), or a total compaction
// size beyond a hard limit of 500mb, is criteria for rejecting this
// total compaction size in excess of 100mb), or a total compaction size
// beyond a hard limit of 500mb, is criteria for rejecting this
// candidate. This lets us prefer slow growths as we add files, while
// still having a hard limit. Note that if this is the first compaction
// candidate to reach a stack depth reduction of minCompactionDepth or
@@ -1568,7 +1585,7 @@ func (s *L0Sublevels) extendFiles(
// PickIntraL0Compaction picks an intra-L0 compaction for files in this
// sublevel. This method is only called when a base compaction cannot be chosen.
// See comment above PickBaseCompaction for heuristics involved in this
// See comment above [PickBaseCompaction] for heuristics involved in this
// selection.
func (s *L0Sublevels) PickIntraL0Compaction(
earliestUnflushedSeqNum uint64, minCompactionDepth int,
@@ -1584,9 +1601,8 @@ func (s *L0Sublevels) PickIntraL0Compaction(
}
sort.Sort(intervalSorterByDecreasingScore(scoredIntervals))
// Optimization to avoid considering different intervals that
// are likely to choose the same seed file. Again this is just
// to reduce wasted work.
// Optimization to avoid considering different intervals that are likely to
// choose the same seed file. Again this is just to reduce wasted work.
consideredIntervals := newBitSet(len(s.orderedIntervals))
for _, scoredInterval := range scoredIntervals {
interval := &s.orderedIntervals[scoredInterval.interval]
@@ -1595,8 +1611,8 @@ func (s *L0Sublevels) PickIntraL0Compaction(
}
var f *FileMetadata
// Pick the seed file for the interval as the file
// in the highest sub-level.
// Pick the seed file for the interval as the file in the highest
// sub-level.
stackDepthReduction := scoredInterval.score
for i := len(interval.files) - 1; i >= 0; i-- {
f = interval.files[i]
@@ -1604,9 +1620,8 @@ func (s *L0Sublevels) PickIntraL0Compaction(
break
}
consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1)
// Can this be the seed file? Files with newer sequence
// numbers than earliestUnflushedSeqNum cannot be in
// the compaction.
// Can this be the seed file? Files with newer sequence numbers than
// earliestUnflushedSeqNum cannot be in the compaction.
if f.LargestSeqNum >= earliestUnflushedSeqNum {
stackDepthReduction--
if stackDepthReduction == 0 {
@@ -1669,10 +1684,10 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed(
}
// The first iteration of this loop produces an intra-L0 compaction at the
// seed level. Iterations after that optionally add to the compaction by
// stacking more files from intervalIndex and repeating. This is an
// optional activity so when it fails we can fallback to the last
// successful candidate. The code stops adding when it can't add more, or
// when fileBytes grows too large.
// stacking more files from intervalIndex and repeating. This is an optional
// activity so when it fails we can fallback to the last successful
// candidate. The code stops adding when it can't add more, or when
// fileBytes grows too large.
for ; slIndex >= 0; slIndex-- {
f2 := interval.files[slIndex]
sl := f2.SubLevel
@@ -1684,23 +1699,24 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed(
c.addFile(f2)
// The seed file captures all files in the higher level that fall in the
// range of intervals. That may extend the range of intervals so for
// correctness we need to capture all files in the next higher level that
// fall in this extended interval and so on. This can result in an
// inverted triangular shape like the following where again the X axis is the
// key intervals and the Y axis is oldest to youngest. Note that it is not
// necessary for correctness to fill out the shape at lower sub-levels to
// make it more rectangular since the invariant only requires that if we
// move an older seqnum for key k into a file that has a higher seqnum, we
// also move all younger seqnums for that key k into that file.
// correctness we need to capture all files in the next higher level
// that fall in this extended interval and so on. This can result in an
// inverted triangular shape like the following where again the X axis
// is the key intervals and the Y axis is oldest to youngest. Note that
// it is not necessary for correctness to fill out the shape at lower
// sub-levels to make it more rectangular since the invariant only
// requires that if we move an older seqnum for key k into a file that
// has a higher seqnum, we also move all younger seqnums for that key k
// into that file.
// -----
// ---
// -
//
// It may be better for performance to have a more rectangular shape since
// it will reduce the stack depth for more intervals. But there is also
// the danger that in explicitly trying to construct a more rectangular
// shape we will be forced to pull in a file that is already compacting.
// We assume that the performance concern is not a practical issue.
// It may be better for performance to have a more rectangular shape
// since it will reduce the stack depth for more intervals. But there is
// also the danger that in explicitly trying to construct a more
// rectangular shape we will be forced to pull in a file that is already
// compacting. We assume that the performance concern is not a practical
// issue.
done := false
for currLevel := sl + 1; currLevel < len(s.levelFiles); currLevel++ {
if !s.extendFiles(currLevel, earliestUnflushedSeqNum, c) {
@@ -1734,11 +1750,11 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed(
}
// ExtendL0ForBaseCompactionTo extends the specified base compaction candidate
// L0CompactionFiles to optionally cover more files in L0 without "touching"
// any of the passed-in keys (i.e. the smallest/largest bounds are exclusive),
// as including any user keys for those internal keys
// could require choosing more files in LBase which is undesirable. Unbounded
// start/end keys are indicated by passing in the InvalidInternalKey.
// L0CompactionFiles to optionally cover more files in L0 without "touching" any
// of the passed-in keys (i.e. the smallest/largest bounds are exclusive), as
// including any user keys for those internal keys could require choosing more
// files in LBase which is undesirable. Unbounded start/end keys are indicated
// by passing in the InvalidInternalKey.
func (s *L0Sublevels) ExtendL0ForBaseCompactionTo(
smallest, largest InternalKey, candidate *L0CompactionFiles,
) bool {
@@ -1794,65 +1810,66 @@ func (s *L0Sublevels) ExtendL0ForBaseCompactionTo(
// Consider this scenario (original candidate is inside the rectangle), with
// isBase = true and interval bounds a-j (from the union of base file bounds and
// that of compaction candidate):
// _______
// L0.3 a--d | g-j|
// L0.2 | f--j| r-t
// L0.1 b-d |e---j|
// L0.0 a--d | f--j| l--o p-----x
//
// Lbase a--------i m---------w
// _______
// L0.3 a--d | g-j|
// L0.2 | f--j| r-t
// L0.1 b-d |e---j|
// L0.0 a--d | f--j| l--o p-----x
//
// Lbase a--------i m---------w
//
// This method will iterate from the bottom up. At L0.0, it will add a--d since
// it's in the bounds, then add b-d, then a--d, and so on, to produce this:
//
// _____________
// L0.3 |a--d g-j|
// L0.2 | f--j| r-t
// L0.1 | b-d e---j|
// L0.0 |a--d f--j| l--o p-----x
// _____________
// L0.3 |a--d g-j|
// L0.2 | f--j| r-t
// L0.1 | b-d e---j|
// L0.0 |a--d f--j| l--o p-----x
//
// Lbase a-------i m---------w
// Lbase a-------i m---------w
//
// Let's assume that, instead of a--d in the top sublevel, we had 3 files, a-b,
// bb-c, and cc-d, of which bb-c is compacting. Let's also add another sublevel
// L0.4 with some files, all of which aren't compacting:
//
// L0.4 a------c ca--d _______
// L0.3 a-b bb-c cc-d | g-j|
// L0.2 | f--j| r-t
// L0.1 b----------d |e---j|
// L0.0 a------------d | f--j| l--o p-----x
// L0.4 a------c ca--d _______
// L0.3 a-b bb-c cc-d | g-j|
// L0.2 | f--j| r-t
// L0.1 b----------d |e---j|
// L0.0 a------------d | f--j| l--o p-----x
//
// Lbase a------------------i m---------w
// Lbase a------------------i m---------w
//
// This method then needs to choose between the left side of L0.3 bb-c
// (i.e. a-b), or the right side (i.e. cc-d and g-j) for inclusion in this
// compaction. Since the right side has more files as well as one file that has
// already been picked, it gets chosen at that sublevel, resulting in this
// intermediate compaction:
// This method then needs to choose between the left side of L0.3 bb-c (i.e.
// a-b), or the right side (i.e. cc-d and g-j) for inclusion in this compaction.
// Since the right side has more files as well as one file that has already been
// picked, it gets chosen at that sublevel, resulting in this intermediate
// compaction:
//
// L0.4 a------c ca--d
// ______________
// L0.3 a-b bb-c| cc-d g-j|
// L0.2 _________| f--j| r-t
// L0.1 | b----------d e---j|
// L0.0 |a------------d f--j| l--o p-----x
// L0.4 a------c ca--d
// ______________
// L0.3 a-b bb-c| cc-d g-j|
// L0.2 _________| f--j| r-t
// L0.1 | b----------d e---j|
// L0.0 |a------------d f--j| l--o p-----x
//
// Lbase a------------------i m---------w
// Lbase a------------------i m---------w
//
// Since bb-c had to be excluded at L0.3, the interval bounds for L0.4 are
// actually ca-j, since ca is the next interval start key after the end interval
// of bb-c. This would result in only ca-d being chosen at that sublevel, even
// though a--c is also not compacting. This is the final result:
//
// ______________
// L0.4 a------c|ca--d |
// L0.3 a-b bb-c| cc-d g-j|
// L0.2 _________| f--j| r-t
// L0.1 | b----------d e---j|
// L0.0 |a------------d f--j| l--o p-----x
// ______________
// L0.4 a------c|ca--d |
// L0.3 a-b bb-c| cc-d g-j|
// L0.2 _________| f--j| r-t
// L0.1 | b----------d e---j|
// L0.0 |a------------d f--j| l--o p-----x
//
// Lbase a------------------i m---------w
// Lbase a------------------i m---------w
//
// TODO(bilal): Add more targeted tests for this method, through
// ExtendL0ForBaseCompactionTo and intraL0CompactionUsingSeed.
@@ -1883,19 +1900,19 @@ func (s *L0Sublevels) extendCandidateToRectangle(
}
// Stats for files.
addedCount := 0
// Iterate from the oldest sub-level for L0 -> Lbase and youngest
// sub-level for intra-L0. The idea here is that anything that can't
// be included from that level constrains what can be included from
// the next level. This change in constraint is directly incorporated
// into minIntervalIndex, maxIntervalIndex.
// Iterate from the oldest sub-level for L0 -> Lbase and youngest sub-level
// for intra-L0. The idea here is that anything that can't be included from
// that level constrains what can be included from the next level. This
// change in constraint is directly incorporated into minIntervalIndex,
// maxIntervalIndex.
for sl := startLevel; sl != endLevel; sl += increment {
files := s.levelFiles[sl]
// Find the first file that overlaps with minIntervalIndex.
index := sort.Search(len(files), func(i int) bool {
return minIntervalIndex <= files[i].maxIntervalIndex
})
// Track the files that are fully within the current constraint
// of [minIntervalIndex, maxIntervalIndex].
// Track the files that are fully within the current constraint of
// [minIntervalIndex, maxIntervalIndex].
firstIndex := -1
lastIndex := -1
for ; index < len(files); index++ {
@@ -1904,8 +1921,8 @@ func (s *L0Sublevels) extendCandidateToRectangle(
break
}
include := true
// Extends out on the left so can't be included. This narrows
// what we can included in the next level.
// Extends out on the left so can't be included. This narrows what
// we can included in the next level.
if f.minIntervalIndex < minIntervalIndex {
include = false
minIntervalIndex = f.maxIntervalIndex + 1
@@ -1932,13 +1949,13 @@ func (s *L0Sublevels) extendCandidateToRectangle(
continue
}
// We have the files in [firstIndex, lastIndex] as potential for
// inclusion. Some of these may already have been picked. Some
// of them may be already compacting. The latter is tricky since
// we have to decide whether to contract minIntervalIndex or
// maxIntervalIndex when we encounter an already compacting file.
// We pick the longest sequence between firstIndex
// and lastIndex of non-compacting files -- this is represented by
// [candidateNonCompactingFirst, candidateNonCompactingLast].
// inclusion. Some of these may already have been picked. Some of them
// may be already compacting. The latter is tricky since we have to
// decide whether to contract minIntervalIndex or maxIntervalIndex when
// we encounter an already compacting file. We pick the longest sequence
// between firstIndex and lastIndex of non-compacting files -- this is
// represented by [candidateNonCompactingFirst,
// candidateNonCompactingLast].
nonCompactingFirst := -1
currentRunHasAlreadyPickedFiles := false
candidateNonCompactingFirst := -1
@@ -1985,8 +2002,8 @@ func (s *L0Sublevels) extendCandidateToRectangle(
}
}
if candidateNonCompactingFirst == -1 {
// All files are compacting. There will be gaps that we could exploit
// to continue, but don't bother.
// All files are compacting. There will be gaps that we could
// exploit to continue, but don't bother.
break
}
// May need to shrink [minIntervalIndex, maxIntervalIndex] for the next level.

273
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored
View File

@@ -9,6 +9,7 @@ import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// LevelMetadata contains metadata for all of the files within
@@ -23,12 +24,12 @@ type LevelMetadata struct {
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
tree: lm.tree.clone(),
tree: lm.tree.Clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
return lm.tree.release()
return lm.tree.Release()
}
func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetadata {
@@ -46,29 +47,29 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
var t btree
t.cmp = cmp
for _, f := range files {
t.insert(f)
t.Insert(f)
}
return t, LevelSlice{iter: t.iter(), length: t.length}
return t, newLevelSlice(t.Iter())
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.length == 0
return lm.tree.Count() == 0
}
// Len returns the number of files within the level.
func (lm *LevelMetadata) Len() int {
return lm.tree.length
return lm.tree.Count()
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.iter()}
return LevelIterator{iter: lm.tree.Iter()}
}
// Slice constructs a slice containing the entire level.
func (lm *LevelMetadata) Slice() LevelSlice {
return LevelSlice{iter: lm.tree.iter(), length: lm.tree.length}
return newLevelSlice(lm.tree.Iter())
}
// Find finds the provided file in the level if it exists.
@@ -99,7 +100,7 @@ func (lm *LevelMetadata) Annotation(annotator Annotator) interface{} {
if lm.Empty() {
return annotator.Zero(nil)
}
v, _ := lm.tree.root.annotation(annotator)
v, _ := lm.tree.root.Annotation(annotator)
return v
}
@@ -112,7 +113,7 @@ func (lm *LevelMetadata) InvalidateAnnotation(annotator Annotator) {
if lm.Empty() {
return
}
lm.tree.root.invalidateAnnotation(annotator)
lm.tree.root.InvalidateAnnotation(annotator)
}
// LevelFile holds a file's metadata along with its position
@@ -133,7 +134,8 @@ func (lf LevelFile) Slice() LevelSlice {
// a slice constructor like this?
func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSeqNum, files)
tr.release()
tr.Release()
slice.verifyInvariants()
return slice
}
@@ -143,7 +145,8 @@ func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
// a slice constructor like this?
func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSmallestKey(cmp), files)
tr.release()
tr.Release()
slice.verifyInvariants()
return slice
}
@@ -153,13 +156,56 @@ func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice
// TODO(jackson): Update tests to avoid requiring this and remove it.
func NewLevelSliceSpecificOrder(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSpecificOrder(files), files)
tr.release()
tr.Release()
slice.verifyInvariants()
return slice
}
// newLevelSlice constructs a new LevelSlice backed by iter.
func newLevelSlice(iter iterator) LevelSlice {
s := LevelSlice{iter: iter}
if iter.r != nil {
s.length = iter.r.subtreeCount
}
s.verifyInvariants()
return s
}
// newBoundedLevelSlice constructs a new LevelSlice backed by iter and bounded
// by the provided start and end bounds. The provided startBound and endBound
// iterators must be iterators over the same B-Tree. Both start and end bounds
// are inclusive.
func newBoundedLevelSlice(iter iterator, startBound, endBound *iterator) LevelSlice {
s := LevelSlice{
iter: iter,
start: startBound,
end: endBound,
}
if iter.valid() {
s.length = endBound.countLeft() - startBound.countLeft()
// NB: The +1 is a consequence of the end bound being inclusive.
if endBound.valid() {
s.length++
}
// NB: A slice that's empty due to its bounds may have an endBound
// positioned before the startBound due to the inclusive bounds.
// TODO(jackson): Consider refactoring the end boundary to be exclusive;
// it would simplify some areas (eg, here) and complicate others (eg,
// Reslice-ing to grow compactions).
if s.length < 0 {
s.length = 0
}
}
s.verifyInvariants()
return s
}
// LevelSlice contains a slice of the files within a level of the LSM.
// A LevelSlice is immutable once created, but may be used to construct a
// mutable LevelIterator over the slice's files.
//
// LevelSlices should be constructed through one of the existing constructors,
// not manually initialized.
type LevelSlice struct {
iter iterator
length int
@@ -170,6 +216,19 @@ type LevelSlice struct {
end *iterator
}
func (ls LevelSlice) verifyInvariants() {
if invariants.Enabled {
i := ls.Iter()
var length int
for f := i.First(); f != nil; f = i.Next() {
length++
}
if ls.length != length {
panic(fmt.Sprintf("LevelSlice %s has length %d value; actual length is %d", ls, ls.length, length))
}
}
}
// Each invokes fn for each element in the slice.
func (ls LevelSlice) Each(fn func(*FileMetadata)) {
iter := ls.Iter()
@@ -181,6 +240,7 @@ func (ls LevelSlice) Each(fn func(*FileMetadata)) {
// String implements fmt.Stringer.
func (ls LevelSlice) String() string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d files: ", ls.length)
ls.Each(func(f *FileMetadata) {
if buf.Len() > 0 {
fmt.Fprintf(&buf, " ")
@@ -246,18 +306,7 @@ func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelS
end.iter = ls.end.clone()
}
resliceFunc(&start, &end)
s := LevelSlice{
iter: start.iter.clone(),
start: &start.iter,
end: &end.iter,
}
// Calculate the new slice's length.
iter := s.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
s.length++
}
return s
return newBoundedLevelSlice(start.iter.clone(), &start.iter, &end.iter)
}
// KeyType is used to specify the type of keys we're looking for in
@@ -387,8 +436,13 @@ func (i *LevelIterator) Clone() LevelIterator {
}
// Current returns the item at the current iterator position.
//
// Current is deprecated. Callers should instead use the return value of a
// positioning operation.
func (i *LevelIterator) Current() *FileMetadata {
if !i.iter.valid() {
if !i.iter.valid() ||
(i.end != nil && cmpIter(i.iter, *i.end) > 0) ||
(i.start != nil && cmpIter(i.iter, *i.start) < 0) {
return nil
}
return i.iter.cur()
@@ -426,7 +480,7 @@ func (i *LevelIterator) First() *FileMetadata {
if !i.iter.valid() {
return nil
}
return i.filteredNextFile(i.iter.cur())
return i.skipFilteredForward(i.iter.cur())
}
// Last seeks to the last file in the iterator and returns it.
@@ -442,31 +496,37 @@ func (i *LevelIterator) Last() *FileMetadata {
if !i.iter.valid() {
return nil
}
return i.filteredPrevFile(i.iter.cur())
return i.skipFilteredBackward(i.iter.cur())
}
// Next advances the iterator to the next file and returns it.
func (i *LevelIterator) Next() *FileMetadata {
if i.iter.r == nil {
return nil
}
if invariants.Enabled && (i.iter.pos >= i.iter.n.count || (i.end != nil && cmpIter(i.iter, *i.end) > 0)) {
panic("pebble: cannot next forward-exhausted iterator")
}
i.iter.next()
if !i.iter.valid() {
return nil
}
if i.end != nil && cmpIter(i.iter, *i.end) > 0 {
return nil
}
return i.filteredNextFile(i.iter.cur())
return i.skipFilteredForward(i.iter.cur())
}
// Prev moves the iterator the previous file and returns it.
func (i *LevelIterator) Prev() *FileMetadata {
if i.iter.r == nil {
return nil
}
if invariants.Enabled && (i.iter.pos < 0 || (i.start != nil && cmpIter(i.iter, *i.start) < 0)) {
panic("pebble: cannot prev backward-exhausted iterator")
}
i.iter.prev()
if !i.iter.valid() {
return nil
}
if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
return nil
}
return i.filteredPrevFile(i.iter.cur())
return i.skipFilteredBackward(i.iter.cur())
}
// SeekGE seeks to the first file in the iterator's file set with a largest
@@ -475,28 +535,23 @@ func (i *LevelIterator) Prev() *FileMetadata {
// be sorted by user keys and non-overlapping.
func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.empty() {
if i.iter.r == nil {
return nil
}
meta := i.seek(func(m *FileMetadata) bool {
m := i.seek(func(m *FileMetadata) bool {
return cmp(m.Largest.UserKey, userKey) >= 0
})
for meta != nil {
switch i.filter {
case KeyTypePointAndRange:
return meta
case KeyTypePoint:
if meta.HasPointKeys && cmp(meta.LargestPointKey.UserKey, userKey) >= 0 {
return meta
}
case KeyTypeRange:
if meta.HasRangeKeys && cmp(meta.LargestRangeKey.UserKey, userKey) >= 0 {
return meta
}
if i.filter != KeyTypePointAndRange && m != nil {
b, ok := m.LargestBound(i.filter)
if !ok {
m = i.Next()
} else if c := cmp(b.UserKey, userKey); c < 0 || c == 0 && b.IsExclusiveSentinel() {
// This file does not contain any keys of the type ≥ lower. It
// should be filtered, even though it does contain point keys.
m = i.Next()
}
meta = i.Next()
}
return i.filteredNextFile(meta)
return i.skipFilteredForward(m)
}
// SeekLT seeks to the last file in the iterator's file set with a smallest
@@ -505,71 +560,76 @@ func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
// by user keys and non-overlapping.
func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.empty() {
if i.iter.r == nil {
return nil
}
i.seek(func(m *FileMetadata) bool {
return cmp(m.Smallest.UserKey, userKey) >= 0
})
meta := i.Prev()
for meta != nil {
switch i.filter {
case KeyTypePointAndRange:
return meta
case KeyTypePoint:
if meta.HasPointKeys && cmp(meta.SmallestPointKey.UserKey, userKey) < 0 {
return meta
}
case KeyTypeRange:
if meta.HasRangeKeys && cmp(meta.SmallestRangeKey.UserKey, userKey) < 0 {
return meta
}
m := i.Prev()
// Although i.Prev() guarantees that the current file contains keys of the
// relevant type, it doesn't guarantee that the keys of the relevant type
// are < userKey.
if i.filter != KeyTypePointAndRange && m != nil {
b, ok := m.SmallestBound(i.filter)
if !ok {
panic("unreachable")
}
if c := cmp(b.UserKey, userKey); c >= 0 {
// This file does not contain any keys of the type ≥ lower. It
// should be filtered, even though it does contain point keys.
m = i.Prev()
}
meta = i.Prev()
}
return i.filteredPrevFile(meta)
return i.skipFilteredBackward(m)
}
func (i *LevelIterator) filteredNextFile(meta *FileMetadata) *FileMetadata {
switch i.filter {
case KeyTypePoint:
for meta != nil && !meta.HasPointKeys {
meta = i.Next()
// skipFilteredForward takes the file metadata at the iterator's current
// position, and skips forward if the current key-type filter (i.filter)
// excludes the file. It skips until it finds an unfiltered file or exhausts the
// level. If lower is != nil, skipFilteredForward skips any files that do not
// contain keys with the provided key-type ≥ lower.
//
// skipFilteredForward also enforces the upper bound, returning nil if at any
// point the upper bound is exceeded.
func (i *LevelIterator) skipFilteredForward(meta *FileMetadata) *FileMetadata {
for meta != nil && !meta.ContainsKeyType(i.filter) {
i.iter.next()
if !i.iter.valid() {
meta = nil
} else {
meta = i.iter.cur()
}
return meta
case KeyTypeRange:
// TODO(bilal): Range keys are expected to be rare and sparse. Add an
// optimization to annotate the tree and efficiently skip over files that
// do not contain range keys right at the seek step, to reduce iterations
// here.
for meta != nil && !meta.HasRangeKeys {
meta = i.Next()
}
return meta
default:
return meta
}
if meta != nil && i.end != nil && cmpIter(i.iter, *i.end) > 0 {
// Exceeded upper bound.
meta = nil
}
return meta
}
func (i *LevelIterator) filteredPrevFile(meta *FileMetadata) *FileMetadata {
switch i.filter {
case KeyTypePoint:
for meta != nil && !meta.HasPointKeys {
meta = i.Prev()
// skipFilteredBackward takes the file metadata at the iterator's current
// position, and skips backward if the current key-type filter (i.filter)
// excludes the file. It skips until it finds an unfiltered file or exhausts the
// level. If upper is != nil, skipFilteredBackward skips any files that do not
// contain keys with the provided key-type < upper.
//
// skipFilteredBackward also enforces the lower bound, returning nil if at any
// point the lower bound is exceeded.
func (i *LevelIterator) skipFilteredBackward(meta *FileMetadata) *FileMetadata {
for meta != nil && !meta.ContainsKeyType(i.filter) {
i.iter.prev()
if !i.iter.valid() {
meta = nil
} else {
meta = i.iter.cur()
}
return meta
case KeyTypeRange:
// TODO(bilal): Range keys are expected to be rare and sparse. Add an
// optimization to annotate the tree and efficiently skip over files that
// do not contain range keys right at the seek step, to reduce iterations
// here.
for meta != nil && !meta.HasRangeKeys {
meta = i.Prev()
}
return meta
default:
return meta
}
if meta != nil && i.start != nil && cmpIter(i.iter, *i.start) < 0 {
// Exceeded lower bound.
meta = nil
}
return meta
}
func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
@@ -594,7 +654,6 @@ func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
return nil
} else if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
i.iter = i.start.clone()
return i.iter.cur()
}
if !i.iter.valid() {
return nil
@@ -614,13 +673,9 @@ func (i *LevelIterator) Take() LevelFile {
// the same position for a LevelFile because they're inclusive, so we can
// share one iterator stack between the two bounds.
boundsIter := i.iter.clone()
s := newBoundedLevelSlice(i.iter.clone(), &boundsIter, &boundsIter)
return LevelFile{
FileMetadata: m,
slice: LevelSlice{
iter: i.iter.clone(),
start: &boundsIter,
end: &boundsIter,
length: 1,
},
slice: s,
}
}

542
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored
View File

@@ -17,7 +17,6 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/vfs"
)
// Compare exports the base.Compare type.
@@ -42,7 +41,8 @@ type TableInfo struct {
LargestSeqNum uint64
}
// TableStats contains statistics on a table used for compaction heuristics.
// TableStats contains statistics on a table used for compaction heuristics,
// and export via Metrics.
type TableStats struct {
// The total number of entries in the table.
NumEntries uint64
@@ -60,10 +60,14 @@ type TableStats struct {
// overlapping data in L0 and ignores L0 sublevels, but the error that
// introduces is expected to be small.
//
// Tables in the bottommost level of the LSM may have a nonzero estimate
// if snapshots or move compactions prevented the elision of their range
// tombstones.
// Tables in the bottommost level of the LSM may have a nonzero estimate if
// snapshots or move compactions prevented the elision of their range
// tombstones. A table in the bottommost level that was ingested into L6
// will have a zero estimate, because the file's sequence numbers indicate
// that the tombstone cannot drop any data contained within the file itself.
RangeDeletionsBytesEstimate uint64
// Total size of value blocks and value index block.
ValueBlocksSize uint64
}
// boundType represents the type of key (point or range) present as the smallest
@@ -79,10 +83,10 @@ const (
//
// The following shows the valid state transitions:
//
// NotCompacting --> Compacting --> Compacted
// ^ |
// | |
// +-------<-------+
// NotCompacting --> Compacting --> Compacted
// ^ |
// | |
// +-------<-------+
//
// Input files to a compaction transition to Compacting when a compaction is
// picked. A file that has finished compacting typically transitions into the
@@ -117,43 +121,81 @@ func (s CompactionState) String() string {
}
}
// FileMetadata holds the metadata for an on-disk table.
// FileMetadata is maintained for leveled-ssts, i.e., they belong to a level of
// some version. FileMetadata does not contain the actual level of the sst,
// since such leveled-ssts can move across levels in different versions, while
// sharing the same FileMetadata. There are two kinds of leveled-ssts, physical
// and virtual. Underlying both leveled-ssts is a backing-sst, for which the
// only state is FileBacking. A backing-sst is level-less. It is possible for a
// backing-sst to be referred to by a physical sst in one version and by one or
// more virtual ssts in one or more versions. A backing-sst becomes obsolete
// and can be deleted once it is no longer required by any physical or virtual
// sst in any version.
//
// We maintain some invariants:
//
// 1. Each physical and virtual sst will have a unique FileMetadata.FileNum,
// and there will be exactly one FileMetadata associated with the FileNum.
//
// 2. Within a version, a backing-sst is either only referred to by one
// physical sst or one or more virtual ssts.
//
// 3. Once a backing-sst is referred to by a virtual sst in the latest version,
// it cannot go back to being referred to by a physical sst in any future
// version.
//
// Once a physical sst is no longer needed by any version, we will no longer
// maintain the file metadata associated with it. We will still maintain the
// FileBacking associated with the physical sst if the backing sst is required
// by any virtual ssts in any version.
type FileMetadata struct {
// Atomic contains fields which are accessed atomically. Go allocations
// are guaranteed to be 64-bit aligned which we take advantage of by
// placing the 64-bit fields which we access atomically at the beginning
// of the FileMetadata struct. For more information, see
// https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
Atomic struct {
// AllowedSeeks is used to determine if a file should be picked for
// a read triggered compaction. It is decremented when read sampling
// in pebble.Iterator after every after every positioning operation
// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
AllowedSeeks int64
// AllowedSeeks is used to determine if a file should be picked for
// a read triggered compaction. It is decremented when read sampling
// in pebble.Iterator after every after every positioning operation
// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
AllowedSeeks atomic.Int64
// statsValid is 1 if stats have been loaded for the table. The
// TableStats structure is populated only if valid is 1.
statsValid uint32
}
// statsValid indicates if stats have been loaded for the table. The
// TableStats structure is populated only if valid is true.
statsValid atomic.Bool
// FileBacking is the state which backs either a physical or virtual
// sstables.
FileBacking *FileBacking
// InitAllowedSeeks is the inital value of allowed seeks. This is used
// to re-set allowed seeks on a file once it hits 0.
InitAllowedSeeks int64
// Reference count for the file: incremented when a file is added to a
// version and decremented when the version is unreferenced. The file is
// obsolete when the reference count falls to zero.
refs int32
// FileNum is the file number.
//
// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
//
// TODO(bananabrick): Consider creating separate types for
// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
// an indentifier for the FileMetadata in Pebble, and also as a handle to
// perform reads and writes. We should ensure through types that
// FileMetadata.FileNum isn't used to perform reads, and that
// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
FileNum base.FileNum
// Size is the size of the file, in bytes.
// Size is the size of the file, in bytes. Size is an approximate value for
// virtual sstables.
//
// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
//
// TODO(bananabrick): Size is currently used in metrics, and for many key
// Pebble level heuristics. Make sure that the heuristics will still work
// appropriately with an approximate value of size.
Size uint64
// File creation time in seconds since the epoch (1970-01-01 00:00:00
// UTC). For ingested sstables, this corresponds to the time the file was
// ingested.
// ingested. For virtual sstables, this corresponds to the wall clock time
// when the FileMetadata for the virtual sstable was first created.
CreationTime int64
// Smallest and largest sequence numbers in the table, across both point and
// range keys.
// Lower and upper bounds for the smallest and largest sequence numbers in
// the table, across both point and range keys. For physical sstables, these
// values are tight bounds. For virtual sstables, there is no guarantee that
// there will be keys with SmallestSeqNum or LargestSeqNum within virtual
// sstable bounds.
SmallestSeqNum uint64
LargestSeqNum uint64
// SmallestPointKey and LargestPointKey are the inclusive bounds for the
@@ -176,6 +218,13 @@ type FileMetadata struct {
Smallest InternalKey
Largest InternalKey
// Stats describe table statistics. Protected by DB.mu.
//
// For virtual sstables, set stats upon virtual sstable creation as
// asynchronous computation of stats is not currently supported.
//
// TODO(bananabrick): To support manifest replay for virtual sstables, we
// probably need to compute virtual sstable stats asynchronously. Otherwise,
// we'd have to write virtual sstable stats to the version edit.
Stats TableStats
SubLevel int
@@ -224,6 +273,173 @@ type FileMetadata struct {
// key type (point or range) corresponds to the smallest and largest overall
// table bounds.
boundTypeSmallest, boundTypeLargest boundType
// Virtual is true if the FileMetadata belongs to a virtual sstable.
Virtual bool
}
// PhysicalFileMeta is used by functions which want a guarantee that their input
// belongs to a physical sst and not a virtual sst.
//
// NB: This type should only be constructed by calling
// FileMetadata.PhysicalMeta.
type PhysicalFileMeta struct {
*FileMetadata
}
// VirtualFileMeta is used by functions which want a guarantee that their input
// belongs to a virtual sst and not a physical sst.
//
// NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
type VirtualFileMeta struct {
*FileMetadata
}
// PhysicalMeta should be the only source of creating the PhysicalFileMeta
// wrapper type.
func (m *FileMetadata) PhysicalMeta() PhysicalFileMeta {
if m.Virtual {
panic("pebble: file metadata does not belong to a physical sstable")
}
return PhysicalFileMeta{
m,
}
}
// VirtualMeta should be the only source of creating the VirtualFileMeta wrapper
// type.
func (m *FileMetadata) VirtualMeta() VirtualFileMeta {
if !m.Virtual {
panic("pebble: file metadata does not belong to a virtual sstable")
}
return VirtualFileMeta{
m,
}
}
// FileBacking either backs a single physical sstable, or one or more virtual
// sstables.
//
// See the comment above the FileMetadata type for sstable terminology.
type FileBacking struct {
// Reference count for the backing file on disk: incremented when a
// physical or virtual sstable which is backed by the FileBacking is
// added to a version and decremented when the version is unreferenced.
// We ref count in order to determine when it is safe to delete a
// backing sst file from disk. The backing file is obsolete when the
// reference count falls to zero.
refs atomic.Int32
// latestVersionRefs are the references to the FileBacking in the
// latest version. This reference can be through a single physical
// sstable in the latest version, or one or more virtual sstables in the
// latest version.
//
// INVARIANT: latestVersionRefs <= refs.
latestVersionRefs atomic.Int32
// VirtualizedSize is set iff the backing sst is only referred to by
// virtual ssts in the latest version. VirtualizedSize is the sum of the
// virtual sstable sizes of all of the virtual sstables in the latest
// version which are backed by the physical sstable. When a virtual
// sstable is removed from the latest version, we will decrement the
// VirtualizedSize. During compaction picking, we'll compensate a
// virtual sstable file size by
// (FileBacking.Size - FileBacking.VirtualizedSize) / latestVersionRefs.
// The intuition is that if FileBacking.Size - FileBacking.VirtualizedSize
// is high, then the space amplification due to virtual sstables is
// high, and we should pick the virtual sstable with a higher priority.
//
// TODO(bananabrick): Compensate the virtual sstable file size using
// the VirtualizedSize during compaction picking and test.
VirtualizedSize atomic.Uint64
DiskFileNum base.DiskFileNum
Size uint64
}
// InitPhysicalBacking allocates and sets the FileBacking which is required by a
// physical sstable FileMetadata.
//
// Ensure that the state required by FileBacking, such as the FileNum, is
// already set on the FileMetadata before InitPhysicalBacking is called.
// Calling InitPhysicalBacking only after the relevant state has been set in the
// FileMetadata is not necessary in tests which don't rely on FileBacking.
func (m *FileMetadata) InitPhysicalBacking() {
if m.Virtual {
panic("pebble: virtual sstables should use a pre-existing FileBacking")
}
if m.FileBacking == nil {
m.FileBacking = &FileBacking{Size: m.Size, DiskFileNum: m.FileNum.DiskFileNum()}
}
}
// ValidateVirtual should be called once the FileMetadata for a virtual sstable
// is created to verify that the fields of the virtual sstable are sound.
func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {
if !m.Virtual {
panic("pebble: invalid virtual sstable")
}
if createdFrom.SmallestSeqNum != m.SmallestSeqNum {
panic("pebble: invalid smallest sequence number for virtual sstable")
}
if createdFrom.LargestSeqNum != m.LargestSeqNum {
panic("pebble: invalid largest sequence number for virtual sstable")
}
if createdFrom.FileBacking != nil && createdFrom.FileBacking != m.FileBacking {
panic("pebble: invalid physical sstable state for virtual sstable")
}
if m.Size == 0 {
panic("pebble: virtual sstable size must be set upon creation")
}
}
// Refs returns the refcount of backing sstable.
func (m *FileMetadata) Refs() int32 {
return m.FileBacking.refs.Load()
}
// Ref increments the ref count associated with the backing sstable.
func (m *FileMetadata) Ref() {
m.FileBacking.refs.Add(1)
}
// Unref decrements the ref count associated with the backing sstable.
func (m *FileMetadata) Unref() int32 {
v := m.FileBacking.refs.Add(-1)
if invariants.Enabled && v < 0 {
panic("pebble: invalid FileMetadata refcounting")
}
return v
}
// LatestRef increments the latest ref count associated with the backing
// sstable.
func (m *FileMetadata) LatestRef() {
m.FileBacking.latestVersionRefs.Add(1)
if m.Virtual {
m.FileBacking.VirtualizedSize.Add(m.Size)
}
}
// LatestUnref decrements the latest ref count associated with the backing
// sstable.
func (m *FileMetadata) LatestUnref() int32 {
if m.Virtual {
m.FileBacking.VirtualizedSize.Add(-m.Size)
}
v := m.FileBacking.latestVersionRefs.Add(-1)
if invariants.Enabled && v < 0 {
panic("pebble: invalid FileMetadata latest refcounting")
}
return v
}
// LatestRefs returns the latest ref count associated with the backing sstable.
func (m *FileMetadata) LatestRefs() int32 {
return m.FileBacking.latestVersionRefs.Load()
}
// SetCompactionState transitions this file's compaction state to the given
@@ -261,22 +477,14 @@ func (m *FileMetadata) IsCompacting() bool {
// returns true, the Stats field may be read (with or without holding the
// database mutex).
func (m *FileMetadata) StatsValid() bool {
return atomic.LoadUint32(&m.Atomic.statsValid) == 1
}
// StatsValidLocked returns true if the table stats have been populated.
// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the
// overhead of an atomic load. This is possible because table stats validity is
// only set while DB.mu is held.
func (m *FileMetadata) StatsValidLocked() bool {
return m.Atomic.statsValid == 1
return m.statsValid.Load()
}
// StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu
// while populating TableStats and calling StatsMarkValud. Once stats are
// populated, they must not be mutated.
func (m *FileMetadata) StatsMarkValid() {
atomic.StoreUint32(&m.Atomic.statsValid, 1)
m.statsValid.Store(true)
}
// ExtendPointKeyBounds attempts to extend the lower and upper point key bounds
@@ -354,6 +562,66 @@ func (m *FileMetadata) extendOverallBounds(
}
}
// Overlaps returns true if the file key range overlaps with the given range.
func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusiveEnd bool) bool {
if c := cmp(m.Largest.UserKey, start); c < 0 || (c == 0 && m.Largest.IsExclusiveSentinel()) {
// f is completely before the specified range; no overlap.
return false
}
if c := cmp(m.Smallest.UserKey, end); c > 0 || (c == 0 && exclusiveEnd) {
// f is completely after the specified range; no overlap.
return false
}
return true
}
// ContainsKeyType returns whether or not the file contains keys of the provided
// type.
func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
switch kt {
case KeyTypePointAndRange:
return true
case KeyTypePoint:
return m.HasPointKeys
case KeyTypeRange:
return m.HasRangeKeys
default:
panic("unrecognized key type")
}
}
// SmallestBound returns the file's smallest bound of the key type. It returns a
// false second return value if the file does not contain any keys of the key
// type.
func (m *FileMetadata) SmallestBound(kt KeyType) (*InternalKey, bool) {
switch kt {
case KeyTypePointAndRange:
return &m.Smallest, true
case KeyTypePoint:
return &m.SmallestPointKey, m.HasPointKeys
case KeyTypeRange:
return &m.SmallestRangeKey, m.HasRangeKeys
default:
panic("unrecognized key type")
}
}
// LargestBound returns the file's largest bound of the key type. It returns a
// false second return value if the file does not contain any keys of the key
// type.
func (m *FileMetadata) LargestBound(kt KeyType) (*InternalKey, bool) {
switch kt {
case KeyTypePointAndRange:
return &m.Largest, true
case KeyTypePoint:
return &m.LargestPointKey, m.HasPointKeys
case KeyTypeRange:
return &m.LargestRangeKey, m.HasRangeKeys
default:
panic("unrecognized key type")
}
}
const (
maskContainsPointKeys = 1 << 0
maskSmallest = 1 << 1
@@ -417,7 +685,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
// ParseFileMetadataDebug parses a FileMetadata from its DebugString
// representation.
func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
// Split lines of the form:
// 000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
fields := strings.FieldsFunc(s, func(c rune) bool {
@@ -429,8 +697,9 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
}
})
if len(fields)%3 != 0 {
return m, errors.Newf("malformed input: %s", s)
return nil, errors.Newf("malformed input: %s", s)
}
m := &FileMetadata{}
for len(fields) > 0 {
prefix := fields[0]
smallest := base.ParsePrettyInternalKey(fields[1])
@@ -460,7 +729,8 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
m.HasPointKeys = true
}
return
m.InitPhysicalBacking()
return m, nil
}
// Validate validates the metadata for consistency with itself, returning an
@@ -522,6 +792,11 @@ func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error {
}
}
// Ensure that FileMetadata.Init was called.
if m.FileBacking == nil {
return base.CorruptionErrorf("file metadata FileBacking not set")
}
return nil
}
@@ -626,51 +901,50 @@ func SortBySmallest(files []*FileMetadata, cmp Compare) {
func overlaps(iter LevelIterator, cmp Compare, start, end []byte, exclusiveEnd bool) LevelSlice {
startIter := iter.Clone()
startIter.SeekGE(cmp, start)
// SeekGE compares user keys. The user key `start` may be equal to the
// f.Largest because f.Largest is a range deletion sentinel, indicating that
// the user key `start` is NOT contained within the file f. If that's the
// case, we can narrow the overlapping bounds to exclude the file with the
// sentinel.
if f := startIter.Current(); f != nil && f.Largest.IsExclusiveSentinel() &&
cmp(f.Largest.UserKey, start) == 0 {
startIter.Next()
{
startIterFile := startIter.SeekGE(cmp, start)
// SeekGE compares user keys. The user key `start` may be equal to the
// f.Largest because f.Largest is a range deletion sentinel, indicating
// that the user key `start` is NOT contained within the file f. If
// that's the case, we can narrow the overlapping bounds to exclude the
// file with the sentinel.
if startIterFile != nil && startIterFile.Largest.IsExclusiveSentinel() &&
cmp(startIterFile.Largest.UserKey, start) == 0 {
startIterFile = startIter.Next()
}
_ = startIterFile // Ignore unused assignment.
}
endIter := iter.Clone()
endIter.SeekGE(cmp, end)
{
endIterFile := endIter.SeekGE(cmp, end)
if !exclusiveEnd {
// endIter is now pointing at the *first* file with a largest key >= end.
// If there are multiple files including the user key `end`, we want all
// of them, so move forward.
for f := endIter.Current(); f != nil && cmp(f.Largest.UserKey, end) == 0; {
f = endIter.Next()
if !exclusiveEnd {
// endIter is now pointing at the *first* file with a largest key >= end.
// If there are multiple files including the user key `end`, we want all
// of them, so move forward.
for endIterFile != nil && cmp(endIterFile.Largest.UserKey, end) == 0 {
endIterFile = endIter.Next()
}
}
}
// LevelSlice uses inclusive bounds, so if we seeked to the end sentinel
// or nexted too far because Largest.UserKey equaled `end`, go back.
//
// Consider !exclusiveEnd and end = 'f', with the following file bounds:
//
// [b,d] [e, f] [f, f] [g, h]
//
// the above for loop will Next until it arrives at [g, h]. We need to
// observe that g > f, and Prev to the file with bounds [f, f].
if !endIter.iter.valid() {
endIter.Prev()
} else if c := cmp(endIter.Current().Smallest.UserKey, end); c > 0 || c == 0 && exclusiveEnd {
endIter.Prev()
}
iter = startIter.Clone()
return LevelSlice{
iter: iter.iter,
start: &startIter.iter,
end: &endIter.iter,
// LevelSlice uses inclusive bounds, so if we seeked to the end sentinel
// or nexted too far because Largest.UserKey equaled `end`, go back.
//
// Consider !exclusiveEnd and end = 'f', with the following file bounds:
//
// [b,d] [e, f] [f, f] [g, h]
//
// the above for loop will Next until it arrives at [g, h]. We need to
// observe that g > f, and Prev to the file with bounds [f, f].
if endIterFile == nil {
endIterFile = endIter.Prev()
} else if c := cmp(endIterFile.Smallest.UserKey, end); c > 0 || c == 0 && exclusiveEnd {
endIterFile = endIter.Prev()
}
_ = endIterFile // Ignore unused assignment.
}
return newBoundedLevelSlice(startIter.Clone().iter, &startIter.iter, &endIter.iter)
}
// NumLevels is the number of levels a Version contains.
@@ -744,7 +1018,7 @@ type Version struct {
// usage.
//
// L0Sublevels.Levels contains L0 files ordered by sublevels. All the files
// in Files[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
// in Levels[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
// a reference to that slice, as that slice is necessary for iterator
// creation and needs to outlast L0Sublevels.
L0Sublevels *L0Sublevels
@@ -759,7 +1033,7 @@ type Version struct {
// The callback to invoke when the last reference to a version is
// removed. Will be called with list.mu held.
Deleted func(obsolete []*FileMetadata)
Deleted func(obsolete []*FileBacking)
// Stats holds aggregated stats about the version maintained from
// version to version.
@@ -788,15 +1062,21 @@ func (v *Version) DebugString(format base.FormatKey) string {
return v.string(format, true)
}
func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string {
var buf bytes.Buffer
for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- {
fmt.Fprintf(&buf, "0.%d:\n", sublevel)
sublevels[sublevel].Each(func(f *FileMetadata) {
fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose))
})
}
return buf.String()
}
func (v *Version) string(format base.FormatKey, verbose bool) string {
var buf bytes.Buffer
if len(v.L0SublevelFiles) > 0 {
for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
fmt.Fprintf(&buf, "0.%d:\n", sublevel)
v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) {
fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose))
})
}
fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles))
}
for level := 1; level < NumLevels; level++ {
if v.Levels[level].Empty() {
@@ -837,7 +1117,7 @@ func ParseVersionDebug(
m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
m.HasPointKeys = true
}
files[level] = append(files[level], &m)
files[level] = append(files[level], m)
}
}
// Reverse the order of L0 files. This ensures we construct the same
@@ -865,11 +1145,15 @@ func (v *Version) Ref() {
// locked.
func (v *Version) Unref() {
if atomic.AddInt32(&v.refs, -1) == 0 {
obsolete := v.unrefFiles()
l := v.list
l.mu.Lock()
l.Remove(v)
v.Deleted(obsolete)
obsolete := v.unrefFiles()
fileBacking := make([]*FileBacking, len(obsolete))
for i, f := range obsolete {
fileBacking[i] = f.FileBacking
}
v.Deleted(fileBacking)
l.mu.Unlock()
}
}
@@ -881,7 +1165,12 @@ func (v *Version) Unref() {
func (v *Version) UnrefLocked() {
if atomic.AddInt32(&v.refs, -1) == 0 {
v.list.Remove(v)
v.Deleted(v.unrefFiles())
obsolete := v.unrefFiles()
fileBacking := make([]*FileBacking, len(obsolete))
for i, f := range obsolete {
fileBacking[i] = f.FileBacking
}
v.Deleted(fileBacking)
}
}
@@ -933,11 +1222,11 @@ func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool {
}
// Overlaps returns all elements of v.files[level] whose user key range
// intersects the inclusive range [start, end]. If level is non-zero then the
// user key ranges of v.files[level] are assumed to not overlap (although they
// may touch). If level is zero then that assumption cannot be made, and the
// [start, end] range is expanded to the union of those matching ranges so far
// and the computation is repeated until [start, end] stabilizes.
// intersects the given range. If level is non-zero then the user key ranges of
// v.files[level] are assumed to not overlap (although they may touch). If level
// is zero then that assumption cannot be made, and the [start, end] range is
// expanded to the union of those matching ranges so far and the computation is
// repeated until [start, end] stabilizes.
// The returned files are a subsequence of the input files, i.e., the ordering
// is not changed.
func (v *Version) Overlaps(
@@ -957,20 +1246,16 @@ func (v *Version) Overlaps(
if selected {
continue
}
smallest := meta.Smallest.UserKey
largest := meta.Largest.UserKey
if c := cmp(largest, start); c < 0 || c == 0 && meta.Largest.IsExclusiveSentinel() {
// meta is completely before the specified range; skip it.
continue
}
if c := cmp(smallest, end); c > 0 || c == 0 && exclusiveEnd {
// meta is completely after the specified range; skip it.
if !meta.Overlaps(cmp, start, end, exclusiveEnd) {
// meta is completely outside the specified range; skip it.
continue
}
// Overlaps.
selectedIndices[i] = true
numSelected++
smallest := meta.Smallest.UserKey
largest := meta.Largest.UserKey
// Since level == 0, check if the newly added fileMetadata has
// expanded the range. We expand the range immediately for files
// we have remaining to check in this loop. All already checked
@@ -998,17 +1283,17 @@ func (v *Version) Overlaps(
tr.cmp = v.Levels[level].tree.cmp
for i, meta := 0, l0Iter.First(); meta != nil; i, meta = i+1, l0Iter.Next() {
if selectedIndices[i] {
err := tr.insert(meta)
err := tr.Insert(meta)
if err != nil {
panic(err)
}
}
}
slice = LevelSlice{iter: tr.iter(), length: tr.length}
slice = newLevelSlice(tr.Iter())
// TODO(jackson): Avoid the oddity of constructing and
// immediately releasing a B-Tree. Make LevelSlice an
// interface?
tr.release()
tr.Release()
break
}
// Continue looping to retry the files that were not selected.
@@ -1038,37 +1323,6 @@ func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
return nil
}
// CheckConsistency checks that all of the files listed in the version exist
// and their on-disk sizes match the sizes listed in the version.
func (v *Version) CheckConsistency(dirname string, fs vfs.FS) error {
var buf bytes.Buffer
var args []interface{}
for level, files := range v.Levels {
iter := files.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
path := base.MakeFilepath(fs, dirname, base.FileTypeTable, f.FileNum)
info, err := fs.Stat(path)
if err != nil {
buf.WriteString("L%d: %s: %v\n")
args = append(args, errors.Safe(level), errors.Safe(f.FileNum), err)
continue
}
if info.Size() != int64(f.Size) {
buf.WriteString("L%d: %s: file size mismatch (%s): %d (disk) != %d (MANIFEST)\n")
args = append(args, errors.Safe(level), errors.Safe(f.FileNum), path,
errors.Safe(info.Size()), errors.Safe(f.Size))
continue
}
}
}
if buf.Len() == 0 {
return nil
}
return errors.Errorf(buf.String(), args...)
}
// VersionList holds a list of versions. The versions are ordered from oldest
// to newest.
type VersionList struct {

324
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored
View File

@@ -8,11 +8,14 @@ import (
"bufio"
"bytes"
"encoding/binary"
"fmt"
"io"
"sync/atomic"
"sort"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// TODO(peter): describe the MANIFEST file format, independently of the C++
@@ -107,9 +110,40 @@ type VersionEdit struct {
// found that there was no overlapping file at the higher level).
DeletedFiles map[DeletedFileEntry]*FileMetadata
NewFiles []NewFileEntry
// CreatedBackingTables can be used to preserve the FileBacking associated
// with a physical sstable. This is useful when virtual sstables in the
// latest version are reconstructed during manifest replay, and we also need
// to reconstruct the FileBacking which is required by these virtual
// sstables.
//
// INVARIANT: The FileBacking associated with a physical sstable must only
// be added as a backing file in the same version edit where the physical
// sstable is first virtualized. This means that the physical sstable must
// be present in DeletedFiles and that there must be at least one virtual
// sstable with the same FileBacking as the physical sstable in NewFiles. A
// file must be present in CreatedBackingTables in exactly one version edit.
// The physical sstable associated with the FileBacking must also not be
// present in NewFiles.
CreatedBackingTables []*FileBacking
// RemovedBackingTables is used to remove the FileBacking associated with a
// virtual sstable. Note that a backing sstable can be removed as soon as
// there are no virtual sstables in the latest version which are using the
// backing sstable, but the backing sstable doesn't necessarily have to be
// removed atomically with the version edit which removes the last virtual
// sstable associated with the backing sstable. The removal can happen in a
// future version edit.
//
// INVARIANT: A file must only be added to RemovedBackingTables if it was
// added to CreateBackingTables in a prior version edit. The same version
// edit also cannot have the same file present in both CreateBackingTables
// and RemovedBackingTables. A file must be present in RemovedBackingTables
// in exactly one version edit.
RemovedBackingTables []base.DiskFileNum
}
// Decode decodes an edit from the specified reader.
//
// TODO(bananabrick): Support decoding of virtual sstable state.
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(byteReader)
if !ok {
@@ -342,6 +376,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
}
m.boundsSet = true
m.InitPhysicalBacking()
v.NewFiles = append(v.NewFiles, NewFileEntry{
Level: level,
Meta: m,
@@ -364,7 +399,51 @@ func (v *VersionEdit) Decode(r io.Reader) error {
return nil
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
var buf bytes.Buffer
if v.ComparerName != "" {
fmt.Fprintf(&buf, " comparer: %s", v.ComparerName)
}
if v.MinUnflushedLogNum != 0 {
fmt.Fprintf(&buf, " log-num: %d\n", v.MinUnflushedLogNum)
}
if v.ObsoletePrevLogNum != 0 {
fmt.Fprintf(&buf, " prev-log-num: %d\n", v.ObsoletePrevLogNum)
}
if v.NextFileNum != 0 {
fmt.Fprintf(&buf, " next-file-num: %d\n", v.NextFileNum)
}
if v.LastSeqNum != 0 {
fmt.Fprintf(&buf, " last-seq-num: %d\n", v.LastSeqNum)
}
entries := make([]DeletedFileEntry, 0, len(v.DeletedFiles))
for df := range v.DeletedFiles {
entries = append(entries, df)
}
sort.Slice(entries, func(i, j int) bool {
if entries[i].Level != entries[j].Level {
return entries[i].Level < entries[j].Level
}
return entries[i].FileNum < entries[j].FileNum
})
for _, df := range entries {
fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum)
}
for _, nf := range v.NewFiles {
fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String())
if nf.Meta.CreationTime != 0 {
fmt.Fprintf(&buf, " (%s)",
time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
}
fmt.Fprintln(&buf)
}
return buf.String()
}
// Encode encodes an edit to the specified writer.
//
// TODO(bananabrick): Support encoding of virtual sstable state.
func (v *VersionEdit) Encode(w io.Writer) error {
e := versionEditEncoder{new(bytes.Buffer)}
@@ -536,10 +615,28 @@ func (e versionEditEncoder) writeUvarint(u uint64) {
// BulkVersionEdit summarizes the files added and deleted from a set of version
// edits.
//
// INVARIANTS:
// No file can be added to a level more than once. This is true globally, and
// also true for all of the calls to Accumulate for a single bulk version edit.
//
// No file can be removed from a level more than once. This is true globally,
// and also true for all of the calls to Accumulate for a single bulk version
// edit.
//
// A file must not be added and removed from a given level in the same version
// edit.
//
// A file that is being removed from a level must have been added to that level
// before (in a prior version edit). Note that a given file can be deleted from
// a level and added to another level in a single version edit
type BulkVersionEdit struct {
Added [NumLevels][]*FileMetadata
Added [NumLevels]map[base.FileNum]*FileMetadata
Deleted [NumLevels]map[base.FileNum]*FileMetadata
AddedFileBacking []*FileBacking
RemovedFileBacking []base.DiskFileNum
// AddedByFileNum maps file number to file metadata for all added files
// from accumulated version edits. AddedByFileNum is only populated if set
// to non-nil by a caller. It must be set to non-nil when replaying
@@ -558,6 +655,19 @@ type BulkVersionEdit struct {
// Accumulate adds the file addition and deletions in the specified version
// edit to the bulk edit's internal state.
//
// INVARIANTS:
// If a file is added to a given level in a call to Accumulate and then removed
// from that level in a subsequent call, the file will not be present in the
// resulting BulkVersionEdit.Deleted for that level.
//
// After accumulation of version edits, the bulk version edit may have
// information about a file which has been deleted from a level, but it may
// not have information about the same file added to the same level. The add
// could've occurred as part of a previous bulk version edit. In this case,
// the deleted file must be present in BulkVersionEdit.Deleted, at the end
// of the accumulation, because we need to decrease the refcount of the
// deleted file in Apply.
func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
for df, m := range ve.DeletedFiles {
dmap := b.Deleted[df.Level]
@@ -579,7 +689,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
if m.MarkedForCompaction {
b.MarkedForCompactionCountDiff--
}
dmap[df.FileNum] = m
if _, ok := b.Added[df.Level][df.FileNum]; !ok {
dmap[df.FileNum] = m
} else {
// Present in b.Added for the same level.
delete(b.Added[df.Level], df.FileNum)
}
}
for _, nf := range ve.NewFiles {
@@ -590,7 +705,10 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
}
}
b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta)
if b.Added[nf.Level] == nil {
b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
}
b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta
if b.AddedByFileNum != nil {
b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta
}
@@ -598,36 +716,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
b.MarkedForCompactionCountDiff++
}
}
// Generate state for the backing files.
b.AddedFileBacking = append(b.AddedFileBacking, ve.CreatedBackingTables...)
// Since a file can be removed from backing files in exactly one version
// edit it is safe to just append without any de-duplication.
b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)
return nil
}
// AccumulateIncompleteAndApplySingleVE should be called if a single version edit
// is to be applied to the provided curr Version and if the caller needs to
// update the versionSet.zombieTables map. This function exists separately from
// BulkVersionEdit.Apply because it is easier to reason about properties
// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we
// know that exactly one version edit is being accumulated.
//
// Note that the version edit passed into this function may be incomplete
// because compactions don't have the ref counting information necessary to
// populate VersionEdit.RemovedBackingTables. This function will complete such a
// version edit by populating RemovedBackingTables.
//
// Invariant: Any file being deleted through ve must belong to the curr Version.
// We can't have a delete for some arbitrary file which does not exist in curr.
func AccumulateIncompleteAndApplySingleVE(
ve *VersionEdit,
curr *Version,
cmp Compare,
formatKey base.FormatKey,
flushSplitBytes int64,
readCompactionRate int64,
backingStateMap map[base.DiskFileNum]*FileBacking,
) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
if len(ve.RemovedBackingTables) != 0 {
panic("pebble: invalid incomplete version edit")
}
var b BulkVersionEdit
err := b.Accumulate(ve)
if err != nil {
return nil, nil, err
}
zombies = make(map[base.DiskFileNum]uint64)
v, err := b.Apply(
curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
)
if err != nil {
return nil, nil, err
}
for _, s := range b.AddedFileBacking {
backingStateMap[s.DiskFileNum] = s
}
for fileNum := range zombies {
if _, ok := backingStateMap[fileNum]; ok {
// This table was backing some virtual sstable in the latest version,
// but is now a zombie. We add RemovedBackingTables entries for
// these, before the version edit is written to disk.
ve.RemovedBackingTables = append(
ve.RemovedBackingTables, fileNum,
)
delete(backingStateMap, fileNum)
}
}
return v, zombies, nil
}
// Apply applies the delta b to the current version to produce a new
// version. The new version is consistent with respect to the comparer cmp.
//
// curr may be nil, which is equivalent to a pointer to a zero version.
//
// On success, a map of zombie files containing the file numbers and sizes of
// deleted files is returned. These files are considered zombies because they
// are no longer referenced by the returned Version, but cannot be deleted from
// disk as they are still in use by the incoming Version.
// On success, if a non-nil zombies map is provided to Apply, the map is updated
// with file numbers and files sizes of deleted files. These files are
// considered zombies because they are no longer referenced by the returned
// Version, but cannot be deleted from disk as they are still in use by the
// incoming Version.
func (b *BulkVersionEdit) Apply(
curr *Version,
cmp Compare,
formatKey base.FormatKey,
flushSplitBytes int64,
readCompactionRate int64,
) (_ *Version, zombies map[base.FileNum]uint64, _ error) {
addZombie := func(fileNum base.FileNum, size uint64) {
if zombies == nil {
zombies = make(map[base.FileNum]uint64)
}
zombies[fileNum] = size
}
// The remove zombie function is used to handle tables that are moved from
// one level to another during a version edit (i.e. a "move" compaction).
removeZombie := func(fileNum base.FileNum) {
zombies map[base.DiskFileNum]uint64,
) (*Version, error) {
addZombie := func(state *FileBacking) {
if zombies != nil {
delete(zombies, fileNum)
zombies[state.DiskFileNum] = state.Size
}
}
removeZombie := func(state *FileBacking) {
if zombies != nil {
delete(zombies, state.DiskFileNum)
}
}
@@ -639,7 +822,7 @@ func (b *BulkVersionEdit) Apply(
}
v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff
if v.Stats.MarkedForCompaction < 0 {
return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
}
for level := range v.Levels {
@@ -660,7 +843,7 @@ func (b *BulkVersionEdit) Apply(
// Initialize L0Sublevels.
if curr == nil || curr.L0Sublevels == nil {
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
} else {
v.L0Sublevels = curr.L0Sublevels
@@ -673,47 +856,70 @@ func (b *BulkVersionEdit) Apply(
// Some edits on this level.
lm := &v.Levels[level]
lmRange := &v.RangeKeyLevels[level]
addedFiles := b.Added[level]
deletedMap := b.Deleted[level]
if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
return nil, nil, base.CorruptionErrorf(
addedFilesMap := b.Added[level]
deletedFilesMap := b.Deleted[level]
if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 {
return nil, base.CorruptionErrorf(
"pebble: internal error: No current or added files but have deleted files: %d",
errors.Safe(len(deletedMap)))
errors.Safe(len(deletedFilesMap)))
}
// NB: addedFiles may be empty and it also is not necessarily
// internally consistent: it does not reflect deletions in deletedMap.
// NB: addedFilesMap may be empty. If a file is present in addedFilesMap
// for a level, it won't be present in deletedFilesMap for the same
// level.
for _, f := range deletedMap {
addZombie(f.FileNum, f.Size)
if obsolete := v.Levels[level].tree.delete(f); obsolete {
for _, f := range deletedFilesMap {
if obsolete := v.Levels[level].tree.Delete(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
// file's reference count dropping to zero.
err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
return nil, nil, err
return nil, err
}
if f.HasRangeKeys {
if obsolete := v.RangeKeyLevels[level].tree.delete(f); obsolete {
if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
// file's reference count dropping to zero.
err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum)
return nil, nil, err
return nil, err
}
}
// Note that a backing sst will only become a zombie if the
// references to it in the latest version is 0. We will remove the
// backing sst from the zombie list in the next loop if one of the
// addedFiles in any of the levels is referencing the backing sst.
// This is possible if a physical sstable is virtualized, or if it
// is moved.
latestRefCount := f.LatestRefs()
if latestRefCount <= 0 {
// If a file is present in deletedFilesMap for a level, then it
// must have already been added to the level previously, which
// means that its latest ref count cannot be 0.
err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum)
return nil, err
} else if f.LatestUnref() == 0 {
addZombie(f.FileBacking)
}
}
addedFiles := make([]*FileMetadata, 0, len(addedFilesMap))
for _, f := range addedFilesMap {
addedFiles = append(addedFiles, f)
}
// Sort addedFiles by file number. This isn't necessary, but tests which
// replay invalid manifests check the error output, and the error output
// depends on the order in which files are added to the btree.
sort.Slice(addedFiles, func(i, j int) bool {
return addedFiles[i].FileNum < addedFiles[j].FileNum
})
var sm, la *FileMetadata
for _, f := range addedFiles {
if _, ok := deletedMap[f.FileNum]; ok {
// Already called addZombie on this file in the preceding
// loop, so we don't need to do it here.
continue
}
// NB: allowedSeeks is used for read triggered compactions. It is set using
// Options.Experimental.ReadCompactionRate which defaults to 32KB.
var allowedSeeks int64
@@ -723,20 +929,23 @@ func (b *BulkVersionEdit) Apply(
if allowedSeeks < 100 {
allowedSeeks = 100
}
atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks)
f.AllowedSeeks.Store(allowedSeeks)
f.InitAllowedSeeks = allowedSeeks
err := lm.tree.insert(f)
err := lm.tree.Insert(f)
// We're adding this file to the new version, so increment the
// latest refs count.
f.LatestRef()
if err != nil {
return nil, nil, errors.Wrap(err, "pebble")
return nil, errors.Wrap(err, "pebble")
}
if f.HasRangeKeys {
err = lmRange.tree.insert(f)
err = lmRange.tree.Insert(f)
if err != nil {
return nil, nil, errors.Wrap(err, "pebble")
return nil, errors.Wrap(err, "pebble")
}
}
removeZombie(f.FileNum)
removeZombie(f.FileBacking)
// Track the keys with the smallest and largest keys, so that we can
// check consistency of the modified span.
if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 {
@@ -748,27 +957,36 @@ func (b *BulkVersionEdit) Apply(
}
if level == 0 {
if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 {
if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 {
// Flushes and ingestions that do not delete any L0 files do not require
// a regeneration of L0Sublevels from scratch. We can instead generate
// it incrementally.
var err error
// AddL0Files requires addedFiles to be sorted in seqnum order.
addedFiles = append([]*FileMetadata(nil), addedFiles...)
SortBySeqNum(addedFiles)
v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0])
if errors.Is(err, errInvalidL0SublevelsOpt) {
err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes)
} else if invariants.Enabled && err == nil {
copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes)
if err != nil {
panic(fmt.Sprintf("error when regenerating sublevels: %s", err))
}
s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels)
s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels)
if s1 != s2 {
panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2))
}
}
if err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
v.L0SublevelFiles = v.L0Sublevels.Levels
} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
continue
}
@@ -789,9 +1007,9 @@ func (b *BulkVersionEdit) Apply(
}
})
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
}
}
return v, zombies, nil
return v, nil
}

4
vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go generated vendored
View File

@@ -2,8 +2,8 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64
package manual

31
vendor/github.com/cockroachdb/pebble/internal/private/flush_external.go generated vendored
View File

@@ -1,31 +0,0 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import "github.com/cockroachdb/pebble/internal/manifest"
// FlushExternalTable is a hook for linking files into L0 without assigning a
// global sequence number, mimicking a flush. FlushExternalTable takes a
// *pebble.DB, the path and the metadata of a sstable to flush directly into
// L0.
//
// Calls to flush a sstable may fail if the file's sequence numbers are not
// greater than the current commit pipeline's sequence number. On success the
// commit pipeline's published sequence number will be moved to the file's
// highest sequence number.
//
// This function is wrapped in a safer, more ergonomic API in the
// internal/replay package. Clients should use the replay package rather than
// calling this private hook directly.
var FlushExternalTable func(interface{}, string, *manifest.FileMetadata) error
// RatchetSeqNum is a hook for allocating and publishing sequence numbers up
// to a specific absolute value. Its first parameter is a *pebble.DB and its
// second is the new next sequence number. RatchetSeqNum does nothing if the
// next sequence is already greater than or equal to nextSeqNum.
//
// This function is used by the internal/replay package to ensure replayed
// operations receive the same absolute sequence number.
var RatchetSeqNum func(interface{}, uint64)

11
vendor/github.com/cockroachdb/pebble/internal/private/sstable.go generated vendored
View File

@@ -8,7 +8,7 @@ import "github.com/cockroachdb/pebble/internal/base"
// SSTableCacheOpts is a hook for specifying cache options to
// sstable.NewReader.
var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{}
var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{}
// SSTableRawTombstonesOpt is a sstable.Reader option for disabling
// fragmentation of the range tombstones returned by
@@ -22,7 +22,8 @@ var SSTableRawTombstonesOpt interface{}
// tool/make_test_sstables.go.
var SSTableWriterDisableKeyOrderChecks func(interface{})
// SSTableInternalTableOpt is an sstable.Writer option that sets properties for
// sstables being created by the db itself (i.e. through flushes and
// compactions), as opposed to those meant for ingestion.
var SSTableInternalTableOpt interface{}
// SSTableInternalProperties is a func(*sstable.Writer) *sstable.Properties
// function that allows Pebble-internal code to mutate properties that external
// sstable writers are not permitted to edit. It's an untyped interface{} to
// avoid a cyclic dependency.
var SSTableInternalProperties interface{}

268
vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go generated vendored
View File

@@ -6,6 +6,7 @@ package rangekey
import (
"bytes"
"math"
"sort"
"github.com/cockroachdb/pebble/internal/base"
@@ -15,7 +16,41 @@ import (
)
// UserIteratorConfig holds state for constructing the range key iterator stack
// for user iteration.
// for user iteration. The range key iterator must merge range key spans across
// the levels of the LSM. This merging is performed by a keyspan.MergingIter
// on-the-fly. The UserIteratorConfig implements keyspan.Transformer, evaluating
// range-key semantics and shadowing, so the spans returned by a MergingIter are
// fully resolved.
//
// The MergingIter is wrapped by a BoundedIter, which elides spans that are
// outside the iterator bounds (or the current prefix's bounds, during prefix
// iteration mode).
//
// To provide determinisim during iteration, the BoundedIter is wrapped by a
// DefragmentingIter that defragments abutting spans with identical
// user-observable state.
//
// At the top-level an InterleavingIter interleaves range keys with point keys
// and performs truncation to iterator bounds.
//
// Below is an abbreviated diagram illustrating the mechanics of a SeekGE.
//
// InterleavingIter.SeekGE
// │
// DefragmentingIter.SeekGE
// │
// BoundedIter.SeekGE
// │
// ╭────────────────┴───────────────╮
// │ ├── defragmentBwd*
// MergingIter.SeekGE │
// │ ╰── defragmentFwd
// ╰─╶╶ per level╶╶ ─╮
// │
// │
// ├── <?>.SeekLT
// │
// ╰── <?>.Next
type UserIteratorConfig struct {
snapshot uint64
comparer *base.Comparer
@@ -24,14 +59,29 @@ type UserIteratorConfig struct {
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
sortBuf keysBySuffix
onlySets bool
bufs *Buffers
}
// Buffers holds various buffers used for range key iteration. They're exposed
// so that they may be pooled and reused between iterators.
type Buffers struct {
merging keyspan.MergingBuffers
defragmenting keyspan.DefragmentingBuffers
sortBuf keyspan.KeysBySuffix
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *Buffers) PrepareForReuse() {
bufs.merging.PrepareForReuse()
bufs.defragmenting.PrepareForReuse()
}
// Init initializes the range key iterator stack for user iteration. The
// resulting fragment iterator applies range key semantics, defragments spans
// according to their user-observable state and removes all Keys other than
// RangeKeySets describing the current state of range keys. The resulting spans
// contain Keys sorted by Suffix.
// according to their user-observable state and, if onlySets = true, removes all
// Keys other than RangeKeySets describing the current state of range keys. The
// resulting spans contain Keys sorted by Suffix.
//
// The snapshot sequence number parameter determines which keys are visible. Any
// keys not visible at the provided snapshot are ignored.
@@ -41,14 +91,18 @@ func (ui *UserIteratorConfig) Init(
lower, upper []byte,
hasPrefix *bool,
prefix *[]byte,
onlySets bool,
bufs *Buffers,
iters ...keyspan.FragmentIterator,
) keyspan.FragmentIterator {
ui.snapshot = snapshot
ui.comparer = comparer
ui.miter.Init(comparer.Compare, ui, iters...)
ui.onlySets = onlySets
ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer)
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
ui.litersUsed = 0
ui.bufs = bufs
return &ui.diter
}
@@ -87,17 +141,17 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
// Apply shadowing of keys.
dst.Start = s.Start
dst.End = s.End
ui.sortBuf = keysBySuffix{
cmp: cmp,
keys: dst.Keys[:0],
ui.bufs.sortBuf = keyspan.KeysBySuffix{
Cmp: cmp,
Keys: ui.bufs.sortBuf.Keys[:0],
}
if err := coalesce(&ui.sortBuf, s.Visible(ui.snapshot).Keys, &dst.Keys); err != nil {
if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
return err
}
// During user iteration over range keys, unsets and deletes don't
// matter. Remove them. This step helps logical defragmentation during
// iteration.
keys := dst.Keys
// During user iteration over range keys, unsets and deletes don't matter.
// Remove them if onlySets = true. This step helps logical defragmentation
// during iteration.
keys := ui.bufs.sortBuf.Keys
dst.Keys = dst.Keys[:0]
for i := range keys {
switch keys[i].Kind() {
@@ -110,11 +164,17 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
// Skip.
continue
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
case base.InternalKeyKindRangeKeyDelete:
// Skip.
continue
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
default:
return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
}
@@ -148,8 +208,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
ret := true
for i := range a.Keys {
if invariants.Enabled {
if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
panic("pebble: unexpected non-RangeKeySet during defragmentation")
}
if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||
@@ -193,8 +253,9 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
//
// Additionally, internal range keys at the same sequence number have subtle
// mechanics:
// * RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix.
// * RANGEKEYDELs only apply to keys at lower sequence numbers.
// - RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix.
// - RANGEKEYDELs only apply to keys at lower sequence numbers.
//
// This is required for ingestion. Ingested sstables are assigned a single
// sequence number for the file, at which all of the file's keys are visible.
// The RANGEKEYSET, RANGEKEYUNSET and RANGEKEYDEL key kinds are ordered such
@@ -204,98 +265,105 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
// set or unset but not both.
//
// The resulting dst Keys slice is sorted by Trailer.
func Coalesce(cmp base.Compare, keys []keyspan.Key, dst *[]keyspan.Key) error {
func Coalesce(cmp base.Compare, eq base.Equal, keys []keyspan.Key, dst *[]keyspan.Key) error {
// TODO(jackson): Currently, Coalesce doesn't actually perform the sequence
// number promotion described in the comment above.
keysBySuffix := keysBySuffix{
cmp: cmp,
keys: (*dst)[:0],
keysBySuffix := keyspan.KeysBySuffix{
Cmp: cmp,
Keys: (*dst)[:0],
}
if err := coalesce(&keysBySuffix, keys, dst); err != nil {
if err := coalesce(eq, &keysBySuffix, math.MaxUint64, keys); err != nil {
return err
}
// coalesce left the keys in *dst sorted by suffix. Re-sort them by trailer.
// Update the span with the (potentially reduced) keys slice. coalesce left
// the keys in *dst sorted by suffix. Re-sort them by trailer.
*dst = keysBySuffix.Keys
keyspan.SortKeysByTrailer(dst)
return nil
}
func coalesce(keysBySuffix *keysBySuffix, keys []keyspan.Key, dst *[]keyspan.Key) error {
var deleted bool
for i := 0; i < len(keys) && !deleted; i++ {
k := keys[i]
if invariants.Enabled && i > 0 && k.Trailer > keys[i-1].Trailer {
func coalesce(
equal base.Equal, keysBySuffix *keyspan.KeysBySuffix, snapshot uint64, keys []keyspan.Key,
) error {
// First, enforce visibility and RangeKeyDelete mechanics. We only need to
// consider the prefix of keys before and including the first
// RangeKeyDelete. We also must skip any keys that aren't visible at the
// provided snapshot sequence number.
//
// NB: Within a given sequence number, keys are ordered as:
// RangeKeySet > RangeKeyUnset > RangeKeyDelete
// This is significant, because this ensures that a Set or Unset sharing a
// sequence number with a Delete do not shadow each other.
deleteIdx := -1
for i := range keys {
if invariants.Enabled && i > 0 && keys[i].Trailer > keys[i-1].Trailer {
panic("pebble: invariant violation: span keys unordered")
}
if !keys[i].VisibleAt(snapshot) {
continue
}
// Once a RangeKeyDelete is observed, we know it shadows all subsequent
// keys and we can break early. We don't add the RangeKeyDelete key to
// keysBySuffix.keys yet, because we don't want a suffix-less key
// that appeared earlier in the slice to elide it. It'll be added back
// in at the end.
if keys[i].Kind() == base.InternalKeyKindRangeKeyDelete {
deleteIdx = i
break
}
keysBySuffix.Keys = append(keysBySuffix.Keys, keys[i])
}
// NB: Within a given sequence number, keys are ordered as:
// RangeKeySet > RangeKeyUnset > RangeKeyDelete
// This is significant, because this ensures that none of the range keys
// sharing a sequence number shadow each other.
switch k.Kind() {
case base.InternalKeyKindRangeKeySet:
n := len(keysBySuffix.keys)
// Sort the accumulated keys by suffix. There may be duplicates within a
// suffix, in which case the one with a larger trailer survives.
//
// We use a stable sort so that the first key with a given suffix is the one
// that with the highest Trailer (because the input `keys` was sorted by
// trailer descending).
sort.Stable(keysBySuffix)
if keysBySuffix.get(n, k.Suffix) < n {
// This suffix is already set or unset at a higher sequence
// number. Skip.
continue
// Grab a handle of the full sorted slice, before reslicing
// keysBySuffix.keys to accumulate the final coalesced keys.
sorted := keysBySuffix.Keys
keysBySuffix.Keys = keysBySuffix.Keys[:0]
var (
// prevSuffix is updated on each iteration of the below loop, and
// compared by the subsequent iteration to determine whether adjacent
// keys are defined at the same suffix.
prevSuffix []byte
// shadowing is set to true once any Key is shadowed by another key.
// When it's set to true—or after the loop if no keys are shadowed—the
// keysBySuffix.keys slice is resliced to contain the prefix of
// unshadowed keys. This avoids copying them incrementally in the common
// case of no shadowing.
shadowing bool
)
for i := range sorted {
if i > 0 && equal(prevSuffix, sorted[i].Suffix) {
// Skip; this key is shadowed by the predecessor that had a larger
// Trailer. If this is the first shadowed key, set shadowing=true
// and reslice keysBySuffix.keys to hold the entire unshadowed
// prefix.
if !shadowing {
keysBySuffix.Keys = keysBySuffix.Keys[:i]
shadowing = true
}
keysBySuffix.keys = append(keysBySuffix.keys, k)
sort.Sort(keysBySuffix)
case base.InternalKeyKindRangeKeyUnset:
n := len(keysBySuffix.keys)
if keysBySuffix.get(n, k.Suffix) < n {
// This suffix is already set or unset at a higher sequence
// number. Skip.
continue
}
keysBySuffix.keys = append(keysBySuffix.keys, k)
sort.Sort(keysBySuffix)
case base.InternalKeyKindRangeKeyDelete:
// All remaining range keys in this span have been deleted by this
// RangeKeyDelete. There's no need to continue looping, because all
// the remaining keys are shadowed by this one. The for loop
// condition will terminate when it sees the last key is a
// range key deletion.
keysBySuffix.keys = append(keysBySuffix.keys, k)
deleted = true
default:
return base.CorruptionErrorf("pebble: unexpected range key kind %s", k.Kind())
continue
}
prevSuffix = sorted[i].Suffix
if shadowing {
keysBySuffix.Keys = append(keysBySuffix.Keys, sorted[i])
}
}
// Update the span with the (potentially reduced) keys slice.
// NB: We don't re-sort by Trailer. The exported Coalesce function however
// will.
*dst = keysBySuffix.keys
// If there was no shadowing, keysBySuffix.keys is untouched. We can simply
// set it to the existing `sorted` slice (also backed by keysBySuffix.keys).
if !shadowing {
keysBySuffix.Keys = sorted
}
// If the original input `keys` slice contained a RangeKeyDelete, add it.
if deleteIdx >= 0 {
keysBySuffix.Keys = append(keysBySuffix.Keys, keys[deleteIdx])
}
return nil
}
type keysBySuffix struct {
cmp base.Compare
keys []keyspan.Key
}
// get searches for suffix among the first n keys in keys. If the suffix is
// found, it returns the index of the item with the suffix. If the suffix is not
// found, it returns n.
func (s *keysBySuffix) get(n int, suffix []byte) (i int) {
// Binary search for the suffix to see if there's an existing key with the
// suffix. Only binary search among the first n items. get is called while
// appending new keys with suffixes that may sort before existing keys.
// The n parameter indicates what portion of the keys slice is sorted and
// may contain relevant keys.
i = sort.Search(n, func(i int) bool {
return s.cmp(s.keys[i].Suffix, suffix) >= 0
})
if i < n && s.cmp(s.keys[i].Suffix, suffix) == 0 {
return i
}
return n
}
func (s *keysBySuffix) Len() int { return len(s.keys) }
func (s *keysBySuffix) Less(i, j int) bool { return s.cmp(s.keys[i].Suffix, s.keys[j].Suffix) < 0 }
func (s *keysBySuffix) Swap(i, j int) { s.keys[i], s.keys[j] = s.keys[j], s.keys[i] }

16
vendor/github.com/cockroachdb/pebble/internal/rate/rate.go generated vendored
View File

@@ -196,13 +196,15 @@ func (lim *Limiter) Reserve() Reservation {
// The Limiter takes this Reservation into account when allowing future events.
// ReserveN returns false if n exceeds the Limiter's burst size.
// Usage example:
// r := lim.ReserveN(time.Now(), 1)
// if !r.OK() {
// // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
// return
// }
// time.Sleep(r.Delay())
// Act()
//
// r := lim.ReserveN(time.Now(), 1)
// if !r.OK() {
// // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
// return
// }
// time.Sleep(r.Delay())
// Act()
//
// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
// If you need to respect a deadline or cancel the delay, use Wait instead.
// To drop or skip events exceeding rate limit, use Allow instead.

4
vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_64bit.go generated vendored
View File

@@ -12,8 +12,8 @@
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64
package rawalloc

423
vendor/github.com/cockroachdb/pebble/iterator.go generated vendored
View File

@@ -6,13 +6,14 @@ package pebble
import (
"bytes"
"context"
"io"
"sync"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/fastrand"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/invariants"
@@ -34,9 +35,9 @@ import (
// always a single internalIterator position corresponding to the position
// returned to the user. Consider the example:
//
// a.MERGE.9 a.MERGE.8 a.MERGE.7 a.SET.6 b.DELETE.9 b.DELETE.5 b.SET.4
// \ /
// \ Iterator.Key() = 'a' /
// a.MERGE.9 a.MERGE.8 a.MERGE.7 a.SET.6 b.DELETE.9 b.DELETE.5 b.SET.4
// \ /
// \ Iterator.Key() = 'a' /
//
// The Iterator exposes one valid position at user key 'a' and the two exhausted
// positions at the beginning and end of iteration. The underlying
@@ -118,6 +119,7 @@ type IteratorStats struct {
// ReverseStepCount includes Prev.
ReverseStepCount [NumStatsKind]int
InternalStats InternalIteratorStats
RangeKeyStats RangeKeyIteratorStats
}
var _ redact.SafeFormatter = &IteratorStats{}
@@ -126,6 +128,32 @@ var _ redact.SafeFormatter = &IteratorStats{}
// iterators.
type InternalIteratorStats = base.InternalIteratorStats
// RangeKeyIteratorStats contains miscellaneous stats about range keys
// encountered by the iterator.
type RangeKeyIteratorStats struct {
// Count records the number of range keys encountered during
// iteration. Range keys may be counted multiple times if the iterator
// leaves a range key's bounds and then returns.
Count int
// ContainedPoints records the number of point keys encountered within the
// bounds of a range key. Note that this includes point keys with suffixes
// that sort both above and below the covering range key's suffix.
ContainedPoints int
// SkippedPoints records the count of the subset of ContainedPoints point
// keys that were skipped during iteration due to range-key masking. It does
// not include point keys that were never loaded because a
// RangeKeyMasking.Filter excluded the entire containing block.
SkippedPoints int
}
// Merge adds all of the argument's statistics to the receiver. It may be used
// to accumulate stats across multiple iterators.
func (s *RangeKeyIteratorStats) Merge(o RangeKeyIteratorStats) {
s.Count += o.Count
s.ContainedPoints += o.ContainedPoints
s.SkippedPoints += o.SkippedPoints
}
// LazyValue is a lazy value. See the long comment in base.LazyValue.
type LazyValue = base.LazyValue
@@ -149,6 +177,11 @@ type LazyValue = base.LazyValue
// Next, Prev) return without advancing if the iterator has an accumulated
// error.
type Iterator struct {
// The context is stored here since (a) Iterators are expected to be
// short-lived (since they pin memtables and sstables), (b) plumbing a
// context into every method is very painful, (c) they do not (yet) respect
// context cancellation and are only used for tracing.
ctx context.Context
opts IterOptions
merge Merge
comparer base.Comparer
@@ -172,6 +205,7 @@ type Iterator struct {
value LazyValue
// For use in LazyValue.Clone.
valueBuf []byte
fetcher base.LazyFetcher
// For use in LazyValue.Value.
lazyValueBuf []byte
valueCloser io.Closer
@@ -278,6 +312,9 @@ type Iterator struct {
closePointIterOnce bool
// Used in some tests to disable the random disabling of seek optimizations.
forceEnableSeekOpt bool
// Set to true if NextPrefix is not currently permitted. Defaults to false
// in case an iterator never had any bounds.
nextPrefixNotPermittedByUpperBound bool
}
// cmp is a convenience shorthand for the i.comparer.Compare function.
@@ -343,11 +380,8 @@ type iteratorRangeKeyState struct {
// start and end are the [start, end) boundaries of the current range keys.
start []byte
end []byte
// keys is sorted by Suffix ascending.
keys []RangeKeyData
// buf is used to save range-key data before moving the range-key iterator.
// Start and end boundaries, suffixes and values are all copied into buf.
buf []byte
rangeKeyBuffers
// iterConfig holds fields that are used for the construction of the
// iterator stack, but do not need to be directly accessed during iteration.
@@ -356,6 +390,31 @@ type iteratorRangeKeyState struct {
iterConfig rangekey.UserIteratorConfig
}
type rangeKeyBuffers struct {
// keys is sorted by Suffix ascending.
keys []RangeKeyData
// buf is used to save range-key data before moving the range-key iterator.
// Start and end boundaries, suffixes and values are all copied into buf.
buf bytealloc.A
// internal holds buffers used by the range key internal iterators.
internal rangekey.Buffers
}
func (b *rangeKeyBuffers) PrepareForReuse() {
const maxKeysReuse = 100
if len(b.keys) > maxKeysReuse {
b.keys = nil
}
// Avoid caching the key buf if it is overly large. The constant is
// fairly arbitrary.
if cap(b.buf) >= maxKeyBufCacheSize {
b.buf = nil
} else {
b.buf = b.buf[:0]
}
b.internal.PrepareForReuse()
}
func (i *iteratorRangeKeyState) init(cmp base.Compare, split base.Split, opts *IterOptions) {
i.cmp = cmp
i.split = split
@@ -717,16 +776,16 @@ func (i *Iterator) sampleRead() {
if len(mi.levels) > 1 {
mi.ForEachLevelIter(func(li *levelIter) bool {
l := manifest.LevelToInt(li.level)
if file := li.files.Current(); file != nil {
if f := li.iterFile; f != nil {
var containsKey bool
if i.pos == iterPosNext || i.pos == iterPosCurForward ||
i.pos == iterPosCurForwardPaused {
containsKey = i.cmp(file.SmallestPointKey.UserKey, i.key) <= 0
containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
i.pos == iterPosCurReversePaused {
containsKey = i.cmp(file.LargestPointKey.UserKey, i.key) >= 0
containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
}
// Do nothing if the current key is not contained in file's
// Do nothing if the current key is not contained in f's
// bounds. We could seek the LevelIterator at this level
// to find the right file, but the performance impacts of
// doing that are significant enough to negate the benefits
@@ -740,7 +799,7 @@ func (i *Iterator) sampleRead() {
return true
}
topLevel = l
topFile = file
topFile = f
}
}
return false
@@ -751,14 +810,14 @@ func (i *Iterator) sampleRead() {
return
}
if numOverlappingLevels >= 2 {
allowedSeeks := atomic.AddInt64(&topFile.Atomic.AllowedSeeks, -1)
allowedSeeks := topFile.AllowedSeeks.Add(-1)
if allowedSeeks == 0 {
// Since the compaction queue can handle duplicates, we can keep
// adding to the queue even once allowedSeeks hits 0.
// In fact, we NEED to keep adding to the queue, because the queue
// is small and evicts older and possibly useful compactions.
atomic.AddInt64(&topFile.Atomic.AllowedSeeks, topFile.InitAllowedSeeks)
topFile.AllowedSeeks.Add(topFile.InitAllowedSeeks)
read := readCompaction{
start: topFile.SmallestPointKey.UserKey,
@@ -899,7 +958,7 @@ func (i *Iterator) findPrevEntry(limit []byte) {
// call, so use valueBuf instead. Note that valueBuf is only used
// in this one instance; everywhere else (eg. in findNextEntry),
// we just point i.value to the unsafe i.iter-owned value buffer.
i.value, i.valueBuf = i.iterValue.Clone(i.valueBuf[:0])
i.value, i.valueBuf = i.iterValue.Clone(i.valueBuf[:0], &i.fetcher)
i.saveRangeKey()
i.iterValidityState = IterValid
i.iterKey, i.iterValue = i.iter.Prev()
@@ -1226,7 +1285,7 @@ func (i *Iterator) SeekGEWithLimit(key []byte, limit []byte) IterValidityState {
// An example Split function may separate a timestamp suffix from the prefix of
// the key.
//
// Split(<key>@<timestamp>) -> <key>
// Split(<key>@<timestamp>) -> <key>
//
// Consider the keys "a@1", "a@2", "aa@3", "aa@4". The prefixes for these keys
// are "a", and "aa". Note that despite "a" and "aa" sharing a prefix by the
@@ -1234,20 +1293,20 @@ func (i *Iterator) SeekGEWithLimit(key []byte, limit []byte) IterValidityState {
// function. To see how this works, consider the following set of calls on this
// data set:
//
// SeekPrefixGE("a@0") -> "a@1"
// Next() -> "a@2"
// Next() -> EOF
// SeekPrefixGE("a@0") -> "a@1"
// Next() -> "a@2"
// Next() -> EOF
//
// If you're just looking to iterate over keys with a shared prefix, as
// defined by the configured comparer, set iterator bounds instead:
//
// iter := db.NewIter(&pebble.IterOptions{
// LowerBound: []byte("prefix"),
// UpperBound: []byte("prefiy"),
// })
// for iter.First(); iter.Valid(); iter.Next() {
// // Only keys beginning with "prefix" will be visited.
// }
// iter := db.NewIter(&pebble.IterOptions{
// LowerBound: []byte("prefix"),
// UpperBound: []byte("prefiy"),
// })
// for iter.First(); iter.Valid(); iter.Next() {
// // Only keys beginning with "prefix" will be visited.
// }
//
// See ExampleIterator_SeekPrefixGE for a working example.
//
@@ -1487,13 +1546,7 @@ func (i *Iterator) First() bool {
i.requiresReposition = false
i.stats.ForwardSeekCount[InterfaceCall]++
if lowerBound := i.opts.GetLowerBound(); lowerBound != nil {
i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone)
i.stats.ForwardSeekCount[InternalIterCall]++
} else {
i.iterKey, i.iterValue = i.iter.First()
i.stats.ForwardSeekCount[InternalIterCall]++
}
i.iterFirstWithinBounds()
i.findNextEntry(nil)
i.maybeSampleRead()
return i.iterValidityState == IterValid
@@ -1525,13 +1578,7 @@ func (i *Iterator) Last() bool {
i.requiresReposition = false
i.stats.ReverseSeekCount[InterfaceCall]++
if upperBound := i.opts.GetUpperBound(); upperBound != nil {
i.iterKey, i.iterValue = i.iter.SeekLT(upperBound, base.SeekLTFlagsNone)
i.stats.ReverseSeekCount[InternalIterCall]++
} else {
i.iterKey, i.iterValue = i.iter.Last()
i.stats.ReverseSeekCount[InternalIterCall]++
}
i.iterLastWithinBounds()
i.findPrevEntry(nil)
i.maybeSampleRead()
return i.iterValidityState == IterValid
@@ -1540,7 +1587,7 @@ func (i *Iterator) Last() bool {
// Next moves the iterator to the next key/value pair. Returns true if the
// iterator is pointing at a valid entry and false otherwise.
func (i *Iterator) Next() bool {
return i.NextWithLimit(nil) == IterValid
return i.nextWithLimit(nil) == IterValid
}
// NextWithLimit moves the iterator to the next key/value pair.
@@ -1554,6 +1601,161 @@ func (i *Iterator) Next() bool {
// guarantees it will surface any range keys with bounds overlapping the
// keyspace up to limit.
func (i *Iterator) NextWithLimit(limit []byte) IterValidityState {
return i.nextWithLimit(limit)
}
// NextPrefix moves the iterator to the next key/value pair with a key
// containing a different prefix than the current key. Prefixes are determined
// by Comparer.Split. Exhausts the iterator if invoked while in prefix-iteration
// mode.
//
// It is not permitted to invoke NextPrefix while at a IterAtLimit position.
// When called in this condition, NextPrefix has non-deterministic behavior.
//
// It is not permitted to invoke NextPrefix when the Iterator has an
// upper-bound that is a versioned MVCC key (see the comment for
// Comparer.Split). It returns an error in this case.
func (i *Iterator) NextPrefix() bool {
if i.nextPrefixNotPermittedByUpperBound {
i.lastPositioningOp = unknownLastPositionOp
i.requiresReposition = false
i.err = errors.Errorf("NextPrefix not permitted with upper bound %s",
i.comparer.FormatKey(i.opts.UpperBound))
i.iterValidityState = IterExhausted
return false
}
if i.hasPrefix {
i.iterValidityState = IterExhausted
return false
}
return i.nextPrefix() == IterValid
}
func (i *Iterator) nextPrefix() IterValidityState {
if i.rangeKey != nil {
// NB: Check Valid() before clearing requiresReposition.
i.rangeKey.prevPosHadRangeKey = i.rangeKey.hasRangeKey && i.Valid()
// If we have a range key but did not expose it at the previous iterator
// position (because the iterator was not at a valid position), updated
// must be true. This ensures that after an iterator op sequence like:
// - Next() → (IterValid, RangeBounds() = [a,b))
// - NextWithLimit(...) → (IterAtLimit, RangeBounds() = -)
// - NextWithLimit(...) → (IterValid, RangeBounds() = [a,b))
// the iterator returns RangeKeyChanged()=true.
//
// The remainder of this function will only update i.rangeKey.updated if
// the iterator moves into a new range key, or out of the current range
// key.
i.rangeKey.updated = i.rangeKey.hasRangeKey && !i.Valid() && i.opts.rangeKeys()
}
// Although NextPrefix documents that behavior at IterAtLimit is undefined,
// this function handles these cases as a simple prefix-agnostic Next. This
// is done for deterministic behavior in the metamorphic tests.
//
// TODO(jackson): If the metamorphic test operation generator is adjusted to
// make generation of some operations conditional on the previous
// operations, then we can remove this behavior and explicitly error.
i.lastPositioningOp = unknownLastPositionOp
i.requiresReposition = false
switch i.pos {
case iterPosCurForward:
// Positioned on the current key. Advance to the next prefix.
i.internalNextPrefix(i.split(i.key))
case iterPosCurForwardPaused:
// Positioned at a limit. Implement as a prefix-agnostic Next. See TODO
// up above. The iterator is already positioned at the next key.
case iterPosCurReverse:
// Switching directions.
// Unless the iterator was exhausted, reverse iteration needs to
// position the iterator at iterPosPrev.
if i.iterKey != nil {
i.err = errors.New("switching from reverse to forward but iter is not at prev")
i.iterValidityState = IterExhausted
return i.iterValidityState
}
// The Iterator is exhausted and i.iter is positioned before the first
// key. Reposition to point to the first internal key.
i.iterFirstWithinBounds()
case iterPosCurReversePaused:
// Positioned at a limit. Implement as a prefix-agnostic Next. See TODO
// up above.
//
// Switching directions; The iterator must not be exhausted since it
// paused.
if i.iterKey == nil {
i.err = errors.New("switching paused from reverse to forward but iter is exhausted")
i.iterValidityState = IterExhausted
return i.iterValidityState
}
i.nextUserKey()
case iterPosPrev:
// The underlying iterator is pointed to the previous key (this can
// only happen when switching iteration directions).
if i.iterKey == nil {
// We're positioned before the first key. Need to reposition to point to
// the first key.
i.iterFirstWithinBounds()
} else {
// Move the internal iterator back onto the user key stored in
// i.key. iterPosPrev guarantees that it's positioned at the last
// key with the user key less than i.key, so we're guaranteed to
// land on the correct key with a single Next.
i.iterKey, i.iterValue = i.iter.Next()
if invariants.Enabled && !i.equal(i.iterKey.UserKey, i.key) {
i.opts.logger.Fatalf("pebble: invariant violation: Nexting internal iterator from iterPosPrev landed on %q, not %q",
i.iterKey.UserKey, i.key)
}
}
// The internal iterator is now positioned at i.key. Advance to the next
// prefix.
i.internalNextPrefix(i.split(i.key))
case iterPosNext:
// Already positioned on the next key. Only call nextPrefixKey if the
// next key shares the same prefix.
if i.iterKey != nil {
currKeyPrefixLen := i.split(i.key)
iterKeyPrefixLen := i.split(i.iterKey.UserKey)
if bytes.Equal(i.iterKey.UserKey[:iterKeyPrefixLen], i.key[:currKeyPrefixLen]) {
i.internalNextPrefix(currKeyPrefixLen)
}
}
}
i.stats.ForwardStepCount[InterfaceCall]++
i.findNextEntry(nil /* limit */)
i.maybeSampleRead()
return i.iterValidityState
}
func (i *Iterator) internalNextPrefix(currKeyPrefixLen int) {
if i.iterKey == nil {
return
}
// The Next "fast-path" is not really a fast-path when there is more than
// one version. However, even with TableFormatPebblev3, there is a small
// slowdown (~10%) for one version if we remove it and only call NextPrefix.
// When there are two versions, only calling NextPrefix is ~30% faster.
i.stats.ForwardStepCount[InternalIterCall]++
if i.iterKey, i.iterValue = i.iter.Next(); i.iterKey == nil {
return
}
iterKeyPrefixLen := i.split(i.iterKey.UserKey)
if !bytes.Equal(i.iterKey.UserKey[:iterKeyPrefixLen], i.key[:currKeyPrefixLen]) {
return
}
i.stats.ForwardStepCount[InternalIterCall]++
i.prefixOrFullSeekKey = i.comparer.ImmediateSuccessor(i.prefixOrFullSeekKey[:0], i.key[:currKeyPrefixLen])
i.iterKey, i.iterValue = i.iter.NextPrefix(i.prefixOrFullSeekKey)
if invariants.Enabled && i.iterKey != nil {
if iterKeyPrefixLen := i.split(i.iterKey.UserKey); i.cmp(i.iterKey.UserKey[:iterKeyPrefixLen], i.prefixOrFullSeekKey) < 0 {
panic("pebble: iter.NextPrefix did not advance beyond the current prefix")
}
}
}
func (i *Iterator) nextWithLimit(limit []byte) IterValidityState {
i.stats.ForwardStepCount[InterfaceCall]++
if i.hasPrefix {
if limit != nil {
@@ -1607,13 +1809,7 @@ func (i *Iterator) NextWithLimit(limit []byte) IterValidityState {
}
// We're positioned before the first key. Need to reposition to point to
// the first key.
if lowerBound := i.opts.GetLowerBound(); lowerBound != nil {
i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone)
i.stats.ForwardSeekCount[InternalIterCall]++
} else {
i.iterKey, i.iterValue = i.iter.First()
i.stats.ForwardSeekCount[InternalIterCall]++
}
i.iterFirstWithinBounds()
case iterPosCurReversePaused:
// Switching directions.
// The iterator must not be exhausted since it paused.
@@ -1633,13 +1829,7 @@ func (i *Iterator) NextWithLimit(limit []byte) IterValidityState {
if i.iterKey == nil {
// We're positioned before the first key. Need to reposition to point to
// the first key.
if lowerBound := i.opts.GetLowerBound(); lowerBound != nil {
i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone)
i.stats.ForwardSeekCount[InternalIterCall]++
} else {
i.iterKey, i.iterValue = i.iter.First()
i.stats.ForwardSeekCount[InternalIterCall]++
}
i.iterFirstWithinBounds()
} else {
i.nextUserKey()
}
@@ -1737,13 +1927,7 @@ func (i *Iterator) PrevWithLimit(limit []byte) IterValidityState {
if i.iterKey == nil {
// We're positioned after the last key. Need to reposition to point to
// the last key.
if upperBound := i.opts.GetUpperBound(); upperBound != nil {
i.iterKey, i.iterValue = i.iter.SeekLT(upperBound, base.SeekLTFlagsNone)
i.stats.ReverseSeekCount[InternalIterCall]++
} else {
i.iterKey, i.iterValue = i.iter.Last()
i.stats.ReverseSeekCount[InternalIterCall]++
}
i.iterLastWithinBounds()
} else {
i.prevUserKey()
}
@@ -1756,6 +1940,28 @@ func (i *Iterator) PrevWithLimit(limit []byte) IterValidityState {
return i.iterValidityState
}
// iterFirstWithinBounds moves the internal iterator to the first key,
// respecting bounds.
func (i *Iterator) iterFirstWithinBounds() {
i.stats.ForwardSeekCount[InternalIterCall]++
if lowerBound := i.opts.GetLowerBound(); lowerBound != nil {
i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone)
} else {
i.iterKey, i.iterValue = i.iter.First()
}
}
// iterLastWithinBounds moves the internal iterator to the last key, respecting
// bounds.
func (i *Iterator) iterLastWithinBounds() {
i.stats.ReverseSeekCount[InternalIterCall]++
if upperBound := i.opts.GetUpperBound(); upperBound != nil {
i.iterKey, i.iterValue = i.iter.SeekLT(upperBound, base.SeekLTFlagsNone)
} else {
i.iterKey, i.iterValue = i.iter.Last()
}
}
// RangeKeyData describes a range key's data, set through RangeKeySet. The key
// boundaries of the range key is provided by Iterator.RangeBounds.
type RangeKeyData struct {
@@ -1827,14 +2033,13 @@ func (i *Iterator) saveRangeKey() {
i.rangeKey.hasRangeKey = true
return
}
i.stats.RangeKeyStats.Count += len(s.Keys)
i.rangeKey.buf.Reset()
i.rangeKey.hasRangeKey = true
i.rangeKey.updated = true
i.rangeKey.stale = false
i.rangeKey.buf = append(i.rangeKey.buf[:0], s.Start...)
i.rangeKey.start = i.rangeKey.buf
i.rangeKey.buf = append(i.rangeKey.buf, s.End...)
i.rangeKey.end = i.rangeKey.buf[len(i.rangeKey.buf)-len(s.End):]
i.rangeKey.buf, i.rangeKey.start = i.rangeKey.buf.Copy(s.Start)
i.rangeKey.buf, i.rangeKey.end = i.rangeKey.buf.Copy(s.End)
i.rangeKey.keys = i.rangeKey.keys[:0]
for j := 0; j < len(s.Keys); j++ {
if invariants.Enabled {
@@ -1844,14 +2049,10 @@ func (i *Iterator) saveRangeKey() {
panic("pebble: user iteration encountered range keys not in suffix order")
}
}
i.rangeKey.buf = append(i.rangeKey.buf, s.Keys[j].Suffix...)
suffix := i.rangeKey.buf[len(i.rangeKey.buf)-len(s.Keys[j].Suffix):]
i.rangeKey.buf = append(i.rangeKey.buf, s.Keys[j].Value...)
value := i.rangeKey.buf[len(i.rangeKey.buf)-len(s.Keys[j].Value):]
i.rangeKey.keys = append(i.rangeKey.keys, RangeKeyData{
Suffix: suffix,
Value: value,
})
var rkd RangeKeyData
i.rangeKey.buf, rkd.Suffix = i.rangeKey.buf.Copy(s.Keys[j].Suffix)
i.rangeKey.buf, rkd.Value = i.rangeKey.buf.Copy(s.Keys[j].Value)
i.rangeKey.keys = append(i.rangeKey.keys, rkd)
}
}
@@ -1959,6 +2160,8 @@ func (i *Iterator) Error() error {
return i.err
}
const maxKeyBufCacheSize = 4 << 10 // 4 KB
// Close closes the iterator and returns any accumulated error. Exhausting
// all the key/value pairs in a table is not considered to be an error.
// It is not valid to call any method, including Close, after the iterator
@@ -2023,15 +2226,12 @@ func (i *Iterator) Close() error {
i.valueCloser = nil
}
const maxKeyBufCacheSize = 4 << 10 // 4 KB
if i.rangeKey != nil {
// Avoid caching the key buf if it is overly large. The constant is
// fairly arbitrary.
if cap(i.rangeKey.buf) >= maxKeyBufCacheSize {
i.rangeKey.buf = nil
i.rangeKey.rangeKeyBuffers.PrepareForReuse()
*i.rangeKey = iteratorRangeKeyState{
rangeKeyBuffers: i.rangeKey.rangeKeyBuffers,
}
*i.rangeKey = iteratorRangeKeyState{buf: i.rangeKey.buf}
iterRangeKeyStateAllocPool.Put(i.rangeKey)
i.rangeKey = nil
}
@@ -2098,7 +2298,7 @@ func (i *Iterator) SetBounds(lower, upper []byte) {
// Copy the user-provided bounds into an Iterator-owned buffer, and set them
// on i.opts.{Lower,Upper}Bound.
i.saveBounds(lower, upper)
i.processBounds(lower, upper)
i.iter.SetBounds(i.opts.LowerBound, i.opts.UpperBound)
// If the iterator has an open point iterator that's not currently being
@@ -2120,7 +2320,10 @@ func (i *Iterator) SetBounds(lower, upper []byte) {
i.invalidate()
}
func (i *Iterator) saveBounds(lower, upper []byte) {
// Initialization and changing of the bounds must call processBounds.
// processBounds saves the bounds and computes derived state from those
// bounds.
func (i *Iterator) processBounds(lower, upper []byte) {
// Copy the user-provided bounds into an Iterator-owned buffer. We can't
// overwrite the current bounds, because some internal iterators compare old
// and new bounds for optimizations.
@@ -2132,9 +2335,19 @@ func (i *Iterator) saveBounds(lower, upper []byte) {
} else {
i.opts.LowerBound = nil
}
i.nextPrefixNotPermittedByUpperBound = false
if upper != nil {
buf = append(buf, upper...)
i.opts.UpperBound = buf[len(buf)-len(upper):]
if i.comparer.Split != nil {
if i.comparer.Split(i.opts.UpperBound) != len(i.opts.UpperBound) {
// Setting an upper bound that is a versioned MVCC key. This means
// that a key can have some MVCC versions before the upper bound and
// some after. This causes significant complications for NextPrefix,
// so we bar the user of NextPrefix.
i.nextPrefixNotPermittedByUpperBound = true
}
}
} else {
i.opts.UpperBound = nil
}
@@ -2319,7 +2532,7 @@ func (i *Iterator) SetOptions(o *IterOptions) {
i.opts.LowerBound, i.opts.UpperBound = lower, upper
} else {
i.opts = *o
i.saveBounds(o.LowerBound, o.UpperBound)
i.processBounds(o.LowerBound, o.UpperBound)
// Propagate the changed bounds to the existing point iterator.
// NB: We propagate i.opts.{Lower,Upper}Bound, not o.{Lower,Upper}Bound
// because i.opts now point to buffers owned by Pebble.
@@ -2338,10 +2551,10 @@ func (i *Iterator) SetOptions(o *IterOptions) {
// Iterators created through NewExternalIter have a different iterator
// initialization process.
if i.externalReaders != nil {
finishInitializingExternal(i)
finishInitializingExternal(i.ctx, i)
return
}
finishInitializingIter(i.alloc)
finishInitializingIter(i.ctx, i.alloc)
}
func (i *Iterator) invalidate() {
@@ -2421,6 +2634,12 @@ type CloneOptions struct {
// will cause an increase in memory and disk usage (use NewSnapshot for that
// purpose).
func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) {
return i.CloneWithContext(context.Background(), opts)
}
// CloneWithContext is like Clone, and additionally accepts a context for
// tracing.
func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*Iterator, error) {
if opts.IterOptions == nil {
opts.IterOptions = &i.opts
}
@@ -2436,6 +2655,7 @@ func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) {
buf := iterAllocPool.Get().(*iterAlloc)
dbi := &buf.dbi
*dbi = Iterator{
ctx: ctx,
opts: *opts.IterOptions,
alloc: buf,
merge: i.merge,
@@ -2450,7 +2670,7 @@ func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) {
newIterRangeKey: i.newIterRangeKey,
seqNum: i.seqNum,
}
dbi.saveBounds(dbi.opts.LowerBound, dbi.opts.UpperBound)
dbi.processBounds(dbi.opts.LowerBound, dbi.opts.UpperBound)
// If the caller requested the clone have a current view of the indexed
// batch, set the clone's batch sequence number appropriately.
@@ -2458,7 +2678,7 @@ func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) {
dbi.batchSeqNum = (uint64(len(i.batch.data)) | base.InternalKeySeqNumBatch)
}
return finishInitializingIter(buf), nil
return finishInitializingIter(ctx, buf), nil
}
// Merge adds all of the argument's statistics to the receiver. It may be used
@@ -2471,6 +2691,7 @@ func (stats *IteratorStats) Merge(o IteratorStats) {
stats.ReverseStepCount[i] += o.ReverseStepCount[i]
}
stats.InternalStats.Merge(o.InternalStats)
stats.RangeKeyStats.Merge(o.RangeKeyStats)
}
func (stats *IteratorStats) String() string {
@@ -2492,14 +2713,30 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
}
if stats.InternalStats != (InternalIteratorStats{}) {
s.SafeString(",\n(internal-stats: ")
s.Printf("(block-bytes: (total %s, cached %s)), "+
"(points: (count %s, key-bytes %s, value-bytes %s, tombstoned: %s))",
s.Printf("(block-bytes: (total %s, cached %s, read-time %s)), "+
"(points: (count %s, key-bytes %s, value-bytes %s, tombstoned %s))",
humanize.IEC.Uint64(stats.InternalStats.BlockBytes),
humanize.IEC.Uint64(stats.InternalStats.BlockBytesInCache),
humanize.FormattedString(stats.InternalStats.BlockReadDuration.String()),
humanize.SI.Uint64(stats.InternalStats.PointCount),
humanize.SI.Uint64(stats.InternalStats.KeyBytes),
humanize.SI.Uint64(stats.InternalStats.ValueBytes),
humanize.SI.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones),
)
if stats.InternalStats.SeparatedPointValue.Count != 0 {
s.Printf(", (separated: (count %s, bytes %s, fetched %s)))",
humanize.SI.Uint64(stats.InternalStats.SeparatedPointValue.Count),
humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes),
humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched))
} else {
s.Printf(")")
}
}
if stats.RangeKeyStats != (RangeKeyIteratorStats{}) {
s.SafeString(",\n(range-key-stats: ")
s.Printf("(count %d), (contained points: (count %d, skipped %d)))",
stats.RangeKeyStats.Count,
stats.RangeKeyStats.ContainedPoints,
stats.RangeKeyStats.SkippedPoints)
}
}

118
vendor/github.com/cockroachdb/pebble/level_checker.go generated vendored
View File

@@ -5,10 +5,10 @@
package pebble
import (
"context"
"fmt"
"io"
"sort"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
@@ -59,7 +59,7 @@ type simpleMergingIterLevel struct {
type simpleMergingIter struct {
levels []simpleMergingIterLevel
snapshot uint64
heap mergingIterHeap
heap simpleMergingIterHeap
// The last point's key and level. For validation.
lastKey InternalKey
lastLevel int
@@ -86,12 +86,12 @@ func (m *simpleMergingIter) init(
m.snapshot = snapshot
m.lastLevel = -1
m.heap.cmp = cmp
m.heap.items = make([]mergingIterItem, 0, len(levels))
m.heap.items = make([]simpleMergingIterItem, 0, len(levels))
for i := range m.levels {
l := &m.levels[i]
l.iterKey, l.iterValue = l.iter.First()
if l.iterKey != nil {
item := mergingIterItem{
item := simpleMergingIterItem{
index: i,
value: l.iterValue,
}
@@ -117,7 +117,7 @@ func (m *simpleMergingIter) positionRangeDels() {
if l.rangeDelIter == nil {
continue
}
l.tombstone = keyspan.SeekGE(m.heap.cmp, l.rangeDelIter, item.key.UserKey)
l.tombstone = l.rangeDelIter.SeekGE(item.key.UserKey)
}
}
@@ -378,7 +378,8 @@ func checkRangeTombstones(c *checkConfig) error {
lf := files.Take()
atomicUnit, _ := expandToAtomicUnit(c.cmp, lf.Slice(), true /* disableIsCompacting */)
lower, upper := manifest.KeyRange(c.cmp, atomicUnit.Iter())
iterToClose, iter, err := c.newIters(lf.FileMetadata, nil, internalIterOpts{})
iterToClose, iter, err := c.newIters(
context.Background(), lf.FileMetadata, nil, internalIterOpts{})
if err != nil {
return err
}
@@ -554,11 +555,11 @@ type CheckLevelsStats struct {
}
// CheckLevels checks:
// - Every entry in the DB is consistent with the level invariant. See the
// comment at the top of the file.
// - Point keys in sstables are ordered.
// - Range delete tombstones in sstables are ordered and fragmented.
// - Successful processing of all MERGE records.
// - Every entry in the DB is consistent with the level invariant. See the
// comment at the top of the file.
// - Point keys in sstables are ordered.
// - Range delete tombstones in sstables are ordered and fragmented.
// - Successful processing of all MERGE records.
func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
// Grab and reference the current readState.
readState := d.loadReadState()
@@ -566,7 +567,7 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
// Determine the seqnum to read at after grabbing the read state (current and
// memtables) above.
seqNum := atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum)
seqNum := d.mu.versions.visibleSeqNum.Load()
checkConfig := &checkConfig{
logger: d.opts.Logger,
@@ -638,7 +639,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
manifestIter := current.L0SublevelFiles[sublevel].Iter()
iterOpts := IterOptions{logger: c.logger}
li := &levelIter{}
li.init(iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
manifest.L0Sublevel(sublevel), internalIterOpts{})
li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)
@@ -652,7 +653,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
iterOpts := IterOptions{logger: c.logger}
li := &levelIter{}
li.init(iterOpts, c.cmp, nil /* split */, c.newIters,
li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters,
current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{})
li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)
@@ -674,3 +675,92 @@ func checkLevelsInternal(c *checkConfig) (err error) {
// Phase 2: Check that the tombstones are mutually consistent.
return checkRangeTombstones(c)
}
type simpleMergingIterItem struct {
index int
key InternalKey
value base.LazyValue
}
type simpleMergingIterHeap struct {
cmp Compare
reverse bool
items []simpleMergingIterItem
}
func (h *simpleMergingIterHeap) len() int {
return len(h.items)
}
func (h *simpleMergingIterHeap) less(i, j int) bool {
ikey, jkey := h.items[i].key, h.items[j].key
if c := h.cmp(ikey.UserKey, jkey.UserKey); c != 0 {
if h.reverse {
return c > 0
}
return c < 0
}
if h.reverse {
return ikey.Trailer < jkey.Trailer
}
return ikey.Trailer > jkey.Trailer
}
func (h *simpleMergingIterHeap) swap(i, j int) {
h.items[i], h.items[j] = h.items[j], h.items[i]
}
// init, fix, up and down are copied from the go stdlib.
func (h *simpleMergingIterHeap) init() {
// heapify
n := h.len()
for i := n/2 - 1; i >= 0; i-- {
h.down(i, n)
}
}
func (h *simpleMergingIterHeap) fix(i int) {
if !h.down(i, h.len()) {
h.up(i)
}
}
func (h *simpleMergingIterHeap) pop() *simpleMergingIterItem {
n := h.len() - 1
h.swap(0, n)
h.down(0, n)
item := &h.items[n]
h.items = h.items[:n]
return item
}
func (h *simpleMergingIterHeap) up(j int) {
for {
i := (j - 1) / 2 // parent
if i == j || !h.less(j, i) {
break
}
h.swap(i, j)
j = i
}
}
func (h *simpleMergingIterHeap) down(i0, n int) bool {
i := i0
for {
j1 := 2*i + 1
if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
break
}
j := j1 // left child
if j2 := j1 + 1; j2 < n && h.less(j2, j1) {
j = j2 // = 2*i + 2 // right child
}
if !h.less(j, i) {
break
}
h.swap(i, j)
i = j
}
return i > i0
}

111
vendor/github.com/cockroachdb/pebble/level_iter.go generated vendored
View File

@@ -5,9 +5,9 @@
package pebble
import (
"context"
"fmt"
"runtime/debug"
"unsafe"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
@@ -17,14 +17,38 @@ import (
)
// tableNewIters creates a new point and range-del iterator for the given file
// number. If bytesIterated is specified, it is incremented as the given file is
// iterated through.
// number.
//
// On success, the internalIterator is not-nil and must be closed; the
// FragmentIterator can be nil.
// TODO(radu): always return a non-nil FragmentIterator.
//
// On error, the iterators are nil.
//
// The only (non-test) implementation of tableNewIters is tableCacheContainer.newIters().
type tableNewIters func(
ctx context.Context,
file *manifest.FileMetadata,
opts *IterOptions,
internalOpts internalIterOpts,
) (internalIterator, keyspan.FragmentIterator, error)
// tableNewRangeDelIter takes a tableNewIters and returns a TableNewSpanIter
// for the rangedel iterator returned by tableNewIters.
func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter {
return func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
iter, rangeDelIter, err := newIters(
ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters}, internalIterOpts{})
if iter != nil {
_ = iter.Close()
}
if rangeDelIter == nil {
rangeDelIter = emptyKeyspanIter
}
return rangeDelIter, err
}
}
type internalIterOpts struct {
bytesIterated *uint64
stats *base.InternalIteratorStats
@@ -54,6 +78,11 @@ type internalIterOpts struct {
// kind InternalKeyKindRangeDeletion which will be used to pause the levelIter
// at the sstable until the mergingIter is ready to advance past it.
type levelIter struct {
// The context is stored here since (a) iterators are expected to be
// short-lived (since they pin sstables), (b) plumbing a context into every
// method is very painful, (c) they do not (yet) respect context
// cancellation and are only used for tracing.
ctx context.Context
logger Logger
cmp Compare
split Split
@@ -91,7 +120,8 @@ type levelIter struct {
// - err != nil
// - some other constraint, like the bounds in opts, caused the file at index to not
// be relevant to the iteration.
iter internalIterator
iter internalIterator
// iterFile holds the current file. It is always equal to l.files.Current().
iterFile *fileMetadata
// filteredIter is an optional interface that may be implemented by internal
// iterators that perform filtering of keys. When a new file's iterator is
@@ -214,11 +244,13 @@ func newLevelIter(
bytesIterated *uint64,
) *levelIter {
l := &levelIter{}
l.init(opts, cmp, split, newIters, files, level, internalIterOpts{bytesIterated: bytesIterated})
l.init(context.Background(), opts, cmp, split, newIters, files, level,
internalIterOpts{bytesIterated: bytesIterated})
return l
}
func (l *levelIter) init(
ctx context.Context,
opts IterOptions,
cmp Compare,
split Split,
@@ -227,6 +259,7 @@ func (l *levelIter) init(
level manifest.Level,
internalOpts internalIterOpts,
) {
l.ctx = ctx
l.err = nil
l.level = level
l.logger = opts.getLogger()
@@ -313,9 +346,10 @@ func (l *levelIter) maybeTriggerCombinedIteration(file *fileMetadata, dir int) {
func (l *levelIter) findFileGE(key []byte, flags base.SeekGEFlags) *fileMetadata {
// Find the earliest file whose largest key is >= key.
if invariants.Enabled && flags.TrySeekUsingNext() && disableSeekOpt(key, uintptr(unsafe.Pointer(l))) {
flags = flags.DisableTrySeekUsingNext()
}
// NB: if flags.TrySeekUsingNext()=true, the levelIter must respect it. If
// the levelIter is positioned at the key P, it must return a key ≥ P. If
// used within a merging iterator, the merging iterator will depend on the
// levelIter only moving forward to maintain heap invariants.
// Ordinarily we seek the LevelIterator using SeekGE. In some instances, we
// Next instead. In other instances, we try Next-ing first, falling back to
@@ -632,7 +666,7 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) loadFileReturnIndicato
var rangeDelIter keyspan.FragmentIterator
var iter internalIterator
iter, rangeDelIter, l.err = l.newIters(l.files.Current(), &l.tableOpts, l.internalOpts)
iter, rangeDelIter, l.err = l.newIters(l.ctx, l.iterFile, &l.tableOpts, l.internalOpts)
l.iter = iter
if l.err != nil {
return noFileLoaded
@@ -655,7 +689,7 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) loadFileReturnIndicato
if l.boundaryContext != nil {
l.boundaryContext.smallestUserKey = file.Smallest.UserKey
l.boundaryContext.largestUserKey = file.Largest.UserKey
l.boundaryContext.isLargestUserKeyRangeDelSentinel = file.Largest.IsExclusiveSentinel()
l.boundaryContext.isLargestUserKeyExclusive = file.Largest.IsExclusiveSentinel()
}
return newFileLoaded
}
@@ -687,7 +721,6 @@ func (l *levelIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
l.boundaryContext.isSyntheticIterBoundsKey = false
l.boundaryContext.isIgnorableBoundaryKey = false
}
// NB: the top-level Iterator has already adjusted key based on
// IterOptions.LowerBound.
loadFileIndicator := l.loadFile(l.findFileGE(key, flags), +1)
@@ -864,6 +897,56 @@ func (l *levelIter) Next() (*InternalKey, base.LazyValue) {
return l.verify(l.skipEmptyFileForward())
}
func (l *levelIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
if l.err != nil || l.iter == nil {
return nil, base.LazyValue{}
}
if l.boundaryContext != nil {
l.boundaryContext.isSyntheticIterBoundsKey = false
l.boundaryContext.isIgnorableBoundaryKey = false
}
switch {
case l.largestBoundary != nil:
if l.tableOpts.UpperBound != nil {
// The UpperBound was within this file, so don't load the next
// file. We leave the largestBoundary unchanged so that subsequent
// calls to Next() stay at this file. If a Seek/First/Last call is
// made and this file continues to be relevant, loadFile() will
// set the largestBoundary to nil.
if l.rangeDelIterPtr != nil {
*l.rangeDelIterPtr = nil
}
return nil, base.LazyValue{}
}
// We're stepping past the boundary key, so we need to load a later
// file.
default:
// Reset the smallest boundary since we're moving away from it.
l.smallestBoundary = nil
if key, val := l.iter.NextPrefix(succKey); key != nil {
return l.verify(key, val)
}
// Fall through to seeking.
}
// Seek the manifest level iterator using TrySeekUsingNext=true and
// RelativeSeek=true so that we take advantage of the knowledge that
// `succKey` can only be contained in later files.
metadataSeekFlags := base.SeekGEFlagsNone.EnableTrySeekUsingNext().EnableRelativeSeek()
if l.loadFile(l.findFileGE(succKey, metadataSeekFlags), +1) != noFileLoaded {
// NB: The SeekGE on the file's iterator must not set TrySeekUsingNext,
// because l.iter is unpositioned.
if key, val := l.iter.SeekGE(succKey, base.SeekGEFlagsNone); key != nil {
return l.verify(key, val)
}
return l.verify(l.skipEmptyFileForward())
}
return nil, base.LazyValue{}
}
func (l *levelIter) Prev() (*InternalKey, base.LazyValue) {
if l.err != nil || l.iter == nil {
return nil, base.LazyValue{}
@@ -955,6 +1038,9 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, base.LazyValue) {
// If the boundary is a range deletion tombstone, return that key.
if l.iterFile.LargestPointKey.Kind() == InternalKeyKindRangeDelete {
l.largestBoundary = &l.iterFile.LargestPointKey
if l.boundaryContext != nil {
l.boundaryContext.isIgnorableBoundaryKey = true
}
return l.largestBoundary, base.LazyValue{}
}
// If the last point iterator positioning op might've skipped keys,
@@ -1042,6 +1128,9 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, base.LazyValue) {
// If the boundary is a range deletion tombstone, return that key.
if l.iterFile.SmallestPointKey.Kind() == InternalKeyKindRangeDelete {
l.smallestBoundary = &l.iterFile.SmallestPointKey
if l.boundaryContext != nil {
l.boundaryContext.isIgnorableBoundaryKey = true
}
return l.smallestBoundary, base.LazyValue{}
}
// If the last point iterator positioning op skipped keys, it's

10
vendor/github.com/cockroachdb/pebble/log_recycler.go generated vendored
View File

@@ -32,14 +32,14 @@ type logRecycler struct {
// the log file should not be deleted (i.e. the log is being recycled), and
// false otherwise.
func (r *logRecycler) add(logInfo fileInfo) bool {
if logInfo.fileNum < r.minRecycleLogNum {
if logInfo.fileNum.FileNum() < r.minRecycleLogNum {
return false
}
r.mu.Lock()
defer r.mu.Unlock()
if logInfo.fileNum <= r.mu.maxLogNum {
if logInfo.fileNum.FileNum() <= r.mu.maxLogNum {
// The log file number was already considered for recycling. Don't consider
// it again. This avoids a race between adding the same log file for
// recycling multiple times, and removing the log file for actual
@@ -49,7 +49,7 @@ func (r *logRecycler) add(logInfo fileInfo) bool {
// shouldn't be deleted.
return true
}
r.mu.maxLogNum = logInfo.fileNum
r.mu.maxLogNum = logInfo.fileNum.FileNum()
if len(r.mu.logs) >= r.limit {
return false
}
@@ -89,7 +89,7 @@ func (r *logRecycler) pop(logNum FileNum) error {
if len(r.mu.logs) == 0 {
return errors.New("pebble: log recycler empty")
}
if r.mu.logs[0].fileNum != logNum {
if r.mu.logs[0].fileNum.FileNum() != logNum {
return errors.Errorf("pebble: log recycler invalid %d vs %d", errors.Safe(logNum), errors.Safe(fileInfoNums(r.mu.logs)))
}
r.mu.logs = r.mu.logs[1:]
@@ -102,7 +102,7 @@ func fileInfoNums(finfos []fileInfo) []FileNum {
}
nums := make([]FileNum, len(finfos))
for i := range finfos {
nums[i] = finfos[i].fileNum
nums[i] = finfos[i].fileNum.FileNum()
}
return nums
}

27
vendor/github.com/cockroachdb/pebble/logger.go generated vendored
View File

@@ -4,30 +4,13 @@
package pebble
import (
"fmt"
"log"
"os"
)
import "github.com/cockroachdb/pebble/internal/base"
// Logger defines an interface for writing log messages.
type Logger interface {
Infof(format string, args ...interface{})
Fatalf(format string, args ...interface{})
}
type defaultLogger struct{}
type Logger = base.Logger
// DefaultLogger logs to the Go stdlib logs.
var DefaultLogger defaultLogger
var DefaultLogger = base.DefaultLogger
// Infof implements the Logger.Infof interface.
func (defaultLogger) Infof(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
}
// Fatalf implements the Logger.Fatalf interface.
func (defaultLogger) Fatalf(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
os.Exit(1)
}
// LoggerAndTracer defines an interface for logging and tracing.
type LoggerAndTracer = base.LoggerAndTracer

25
vendor/github.com/cockroachdb/pebble/mem_table.go generated vendored
View File

@@ -80,7 +80,7 @@ type memTable struct {
// inflight mutations that have reserved space in the memtable but not yet
// applied. The memtable cannot be flushed to disk until the writer refs
// drops to zero.
writerRefs int32
writerRefs atomic.Int32
tombstones keySpanCache
rangeKeys keySpanCache
// The current logSeqNum at the time the memtable was created. This is
@@ -115,13 +115,13 @@ func newMemTable(opts memTableOptions) *memTable {
}
m := &memTable{
cmp: opts.Comparer.Compare,
formatKey: opts.Comparer.FormatKey,
equal: opts.Comparer.Equal,
arenaBuf: opts.arenaBuf,
writerRefs: 1,
logSeqNum: opts.logSeqNum,
cmp: opts.Comparer.Compare,
formatKey: opts.Comparer.FormatKey,
equal: opts.Comparer.Equal,
arenaBuf: opts.arenaBuf,
logSeqNum: opts.logSeqNum,
}
m.writerRefs.Store(1)
m.tombstones = keySpanCache{
cmp: m.cmp,
formatKey: m.formatKey,
@@ -143,18 +143,19 @@ func newMemTable(opts memTableOptions) *memTable {
m.skl.Reset(arena, m.cmp)
m.rangeDelSkl.Reset(arena, m.cmp)
m.rangeKeySkl.Reset(arena, m.cmp)
m.reserved = arena.Size()
return m
}
func (m *memTable) writerRef() {
switch v := atomic.AddInt32(&m.writerRefs, 1); {
switch v := m.writerRefs.Add(1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
}
func (m *memTable) writerUnref() bool {
switch v := atomic.AddInt32(&m.writerRefs, -1); {
switch v := m.writerRefs.Add(-1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
@@ -165,7 +166,7 @@ func (m *memTable) writerUnref() bool {
}
func (m *memTable) readyForFlush() bool {
return atomic.LoadInt32(&m.writerRefs) == 0
return m.writerRefs.Load() == 0
}
// Prepare reserves space for the batch in the memtable and references the
@@ -210,6 +211,8 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
// Don't increment seqNum for LogData, since these are not applied
// to the memtable.
seqNum--
case InternalKeyKindIngestSST:
panic("pebble: cannot apply ingested sstable key kind to memtable")
default:
err = ins.Add(&m.skl, ikey, value)
}
@@ -263,7 +266,7 @@ func (m *memTable) containsRangeKeys() bool {
func (m *memTable) availBytes() uint32 {
a := m.skl.Arena()
if atomic.LoadInt32(&m.writerRefs) == 1 {
if m.writerRefs.Load() == 1 {
// If there are no other concurrent apply operations, we can update the
// reserved bytes setting to accurately reflect how many bytes of been
// allocated vs the over-estimation present in memTableEntrySize.

493
vendor/github.com/cockroachdb/pebble/merging_iter.go generated vendored
View File

@@ -8,6 +8,7 @@ import (
"bytes"
"fmt"
"runtime/debug"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
@@ -16,7 +17,8 @@ import (
)
type mergingIterLevel struct {
iter internalIterator
index int
iter internalIterator
// rangeDelIter is set to the range-deletion iterator for the level. When
// configured with a levelIter, this pointer changes as sstable boundaries
// are crossed. See levelIter.initRangeDel and the Range Deletions comment
@@ -44,20 +46,25 @@ type levelIterBoundaryContext struct {
// smallestUserKey and largestUserKey are populated with the smallest and
// largest boundaries of the current file.
smallestUserKey, largestUserKey []byte
// isLargestUserKeyRangeDelSentinel is set to true when a file's largest
// boundary is an exclusive range deletion sentinel. If true, the file does
// not contain any keys with the provided user key, and the largestUserKey
// bound is exclusive.
isLargestUserKeyRangeDelSentinel bool
// isLargestUserKeyExclusive is set to true when a file's largest boundary
// is an exclusive key, (eg, a range deletion sentinel). If true, the file
// does not contain any keys with the provided user key, and the
// largestUserKey bound is exclusive.
isLargestUserKeyExclusive bool
// isSyntheticIterBoundsKey is set to true iff the key returned by the level
// iterator is a synthetic key derived from the iterator bounds. This is
// used to prevent the mergingIter from being stuck at such a synthetic key
// if it becomes the top element of the heap.
// iterator is a synthetic key derived from the iterator bounds. This is used
// to prevent the mergingIter from being stuck at such a synthetic key if it
// becomes the top element of the heap. When used with a user-facing Iterator,
// the only range deletions exposed by this mergingIter should be those with
// `isSyntheticIterBoundsKey || isIgnorableBoundaryKey`.
isSyntheticIterBoundsKey bool
// isIgnorableBoundaryKey is set to true iff the key returned by the level
// iterator is a file boundary key that should be ignored. This is used to
// iterator is a file boundary key that should be ignored when returning to
// the parent iterator. File boundary keys are used by the level iter to
// keep a levelIter file's range deletion iterator open as long as other
// levels within the merging iterator require it.
// levels within the merging iterator require it. When used with a user-facing
// Iterator, the only range deletions exposed by this mergingIter should be
// those with `isSyntheticIterBoundsKey || isIgnorableBoundaryKey`.
isIgnorableBoundaryKey bool
}
@@ -73,7 +80,7 @@ type levelIterBoundaryContext struct {
// property. When one of the child iterators is exhausted during Next/Prev
// iteration, it is removed from the heap.
//
// Range Deletions
// # Range Deletions
//
// A mergingIter can optionally be configured with a slice of range deletion
// iterators. The range deletion iterator slice must exactly parallel the point
@@ -143,11 +150,11 @@ type levelIterBoundaryContext struct {
// to skip processing entries in that shadow. For example, consider the
// scenario:
//
// r0: a---e
// r1: d---h
// r2: g---k
// r3: j---n
// r4: m---q
// r0: a---e
// r1: d---h
// r2: g---k
// r3: j---n
// r4: m---q
//
// This is showing 5 levels of range deletions. Consider what happens upon
// SeekGE("b"). We first seek the point iterator for level 0 (the point values
@@ -189,17 +196,17 @@ type levelIterBoundaryContext struct {
//
// For a full example, consider the following setup:
//
// p0: o
// r0: m---q
// p0: o
// r0: m---q
//
// p1: n p
// r1: g---k
// p1: n p
// r1: g---k
//
// p2: b d i
// r2: a---e q----v
// p2: b d i
// r2: a---e q----v
//
// p3: e
// r3:
// p3: e
// r3:
//
// If we start iterating from the beginning, the first key we encounter is "b"
// in p2. When the mergingIter is pointing at a valid entry, the range deletion
@@ -243,12 +250,15 @@ type mergingIter struct {
upper []byte
stats *InternalIteratorStats
// levelsPositioned, if non-nil, is a slice of the same length as levels.
// It's used by NextPrefix to record which levels have already been
// repositioned. It's created lazily by the first call to NextPrefix.
levelsPositioned []bool
combinedIterState *combinedIterState
// Elide range tombstones from being returned during iteration. Set to true
// when mergingIter is a child of Iterator and the mergingIter is processing
// range tombstones.
elideRangeTombstones bool
// Used in some tests to disable the random disabling of seek optimizations.
forceEnableSeekOpt bool
}
// mergingIter implements the base.InternalIterator interface.
@@ -300,21 +310,20 @@ func (m *mergingIter) init(
m.split = split
m.stats = stats
if cap(m.heap.items) < len(levels) {
m.heap.items = make([]mergingIterItem, 0, len(levels))
m.heap.items = make([]*mergingIterLevel, 0, len(levels))
} else {
m.heap.items = m.heap.items[:0]
}
for l := range m.levels {
m.levels[l].index = l
}
}
func (m *mergingIter) initHeap() {
m.heap.items = m.heap.items[:0]
for i := range m.levels {
if l := &m.levels[i]; l.iterKey != nil {
m.heap.items = append(m.heap.items, mergingIterItem{
index: i,
key: *l.iterKey,
value: l.iterValue,
})
m.heap.items = append(m.heap.items, l)
} else {
m.err = firstError(m.err, l.iter.Error())
if m.err != nil {
@@ -343,13 +352,13 @@ func (m *mergingIter) initMinRangeDelIters(oldTopLevel int) {
}
// Position the range-del iterators at levels <= m.heap.items[0].index.
item := &m.heap.items[0]
item := m.heap.items[0]
for level := oldTopLevel + 1; level <= item.index; level++ {
l := &m.levels[level]
if l.rangeDelIter == nil {
continue
}
l.tombstone = keyspan.SeekGE(m.heap.cmp, l.rangeDelIter, item.key.UserKey)
l.tombstone = l.rangeDelIter.SeekGE(item.iterKey.UserKey)
}
}
@@ -370,13 +379,13 @@ func (m *mergingIter) initMaxRangeDelIters(oldTopLevel int) {
return
}
// Position the range-del iterators at levels <= m.heap.items[0].index.
item := &m.heap.items[0]
item := m.heap.items[0]
for level := oldTopLevel + 1; level <= item.index; level++ {
l := &m.levels[level]
if l.rangeDelIter == nil {
continue
}
l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.key.UserKey)
l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.iterKey.UserKey)
}
}
@@ -400,8 +409,8 @@ func (m *mergingIter) switchToMinHeap() {
// The current key is a:2 and i2 is pointed at a:1. When we switch to forward
// iteration, we want to return a key that is greater than a:2.
key := m.heap.items[0].key
cur := &m.levels[m.heap.items[0].index]
key := m.heap.items[0].iterKey
cur := m.heap.items[0]
for i := range m.levels {
l := &m.levels[i]
@@ -442,7 +451,7 @@ func (m *mergingIter) switchToMinHeap() {
}
}
for ; l.iterKey != nil; l.iterKey, l.iterValue = l.iter.Next() {
if base.InternalCompare(m.heap.cmp, key, *l.iterKey) < 0 {
if base.InternalCompare(m.heap.cmp, *key, *l.iterKey) < 0 {
// key < iter-key
break
}
@@ -483,8 +492,8 @@ func (m *mergingIter) switchToMaxHeap() {
//
// The current key is b:2 and i2 is pointing at b:1. When we switch to
// reverse iteration, we want to return a key that is less than b:2.
key := m.heap.items[0].key
cur := &m.levels[m.heap.items[0].index]
key := m.heap.items[0].iterKey
cur := m.heap.items[0]
for i := range m.levels {
l := &m.levels[i]
@@ -524,7 +533,7 @@ func (m *mergingIter) switchToMaxHeap() {
}
}
for ; l.iterKey != nil; l.iterKey, l.iterValue = l.iter.Prev() {
if base.InternalCompare(m.heap.cmp, key, *l.iterKey) > 0 {
if base.InternalCompare(m.heap.cmp, *key, *l.iterKey) > 0 {
// key > iter-key
break
}
@@ -547,13 +556,69 @@ func (m *mergingIter) switchToMaxHeap() {
m.initMaxHeap()
}
// Steps to the next entry. item is the current top item in the heap.
func (m *mergingIter) nextEntry(item *mergingIterItem) {
l := &m.levels[item.index]
oldTopLevel := item.index
// maybeNextEntryWithinPrefix steps to the next entry, as long as the iteration
// prefix has not already been exceeded. If it has, it exhausts the iterator by
// resetting the heap to empty.
func (m *mergingIter) maybeNextEntryWithinPrefix(l *mergingIterLevel) {
if s := m.split(l.iterKey.UserKey); !bytes.Equal(m.prefix, l.iterKey.UserKey[:s]) {
// The item at the root of the heap already exceeds the iteration
// prefix. We should not advance any more. Clear the heap to reflect
// that the iterator is now exhausted (within this prefix, at
// least).
m.heap.items = m.heap.items[:0]
return
}
m.nextEntry(l, nil /* succKey */)
}
// nextEntry unconditionally steps to the next entry. item is the current top
// item in the heap.
//
// nextEntry should be called directly when not in prefix-iteration mode, or by
// Next. During prefix iteration mode, all other callers should use
// maybeNextEntryWithinPrefix which will avoid advancing the iterator if the
// current iteration prefix has been exhausted. See the comment within
// nextEntry's body for an explanation of why other callers should call
// maybeNextEntryWithinPrefix, which will ensure the documented invariant is
// preserved.
func (m *mergingIter) nextEntry(l *mergingIterLevel, succKey []byte) {
// INVARIANT: If in prefix iteration mode, item.iterKey must have a prefix equal
// to m.prefix. This invariant is important for ensuring TrySeekUsingNext
// optimizations behave correctly.
//
// During prefix iteration, the iterator does not have a full view of the
// LSM. Some level iterators may omit keys that are known to fall outside
// the seek prefix (eg, due to sstable bloom filter exclusion). It's
// important that in such cases we don't position any iterators beyond
// m.prefix, because doing so may interfere with future seeks.
//
// Let prefixes P1 < P2 < P3. Imagine a SeekPrefixGE to prefix P1, followed
// by a SeekPrefixGE to prefix P2. Imagine there exist live keys at prefix
// P2, but they're not visible to the SeekPrefixGE(P1) (because of
// bloom-filter exclusion or a range tombstone that deletes prefix P1 but
// not P2). If the SeekPrefixGE(P1) is allowed to move any level iterators
// to P3, the SeekPrefixGE(P2, TrySeekUsingNext=true) may mistakenly think
// the level contains no point keys or range tombstones within the prefix
// P2. Care is taken to avoid ever advancing the iterator beyond the current
// prefix. If nextEntry is ever invoked while we're already beyond the
// current prefix, we're violating the invariant.
if invariants.Enabled && m.prefix != nil {
if s := m.split(l.iterKey.UserKey); !bytes.Equal(m.prefix, l.iterKey.UserKey[:s]) {
m.logger.Fatalf("mergingIter: prefix violation: nexting beyond prefix %q; existing heap root %q\n%s",
m.prefix, l.iterKey, debug.Stack())
}
}
oldTopLevel := l.index
oldRangeDelIter := l.rangeDelIter
if l.iterKey, l.iterValue = l.iter.Next(); l.iterKey != nil {
item.key, item.value = *l.iterKey, l.iterValue
if succKey == nil {
l.iterKey, l.iterValue = l.iter.Next()
} else {
l.iterKey, l.iterValue = l.iter.NextPrefix(succKey)
}
if l.iterKey != nil {
if m.heap.len() > 1 {
m.heap.fix(0)
}
@@ -575,11 +640,15 @@ func (m *mergingIter) nextEntry(item *mergingIterItem) {
m.initMinRangeDelIters(oldTopLevel)
}
// isNextEntryDeleted() starts from the current entry (as the next entry) and if it is deleted,
// moves the iterators forward as needed and returns true, else it returns false. item is the top
// item in the heap.
func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool {
// Look for a range deletion tombstone containing item.key at higher
// isNextEntryDeleted starts from the current entry (as the next entry) and if
// it is deleted, moves the iterators forward as needed and returns true, else
// it returns false. item is the top item in the heap.
//
// During prefix iteration mode, isNextEntryDeleted will exhaust the iterator by
// clearing the heap if the deleted key(s) extend beyond the iteration prefix
// during prefix-iteration mode.
func (m *mergingIter) isNextEntryDeleted(item *mergingIterLevel) bool {
// Look for a range deletion tombstone containing item.iterKey at higher
// levels (level < item.index). If we find such a range tombstone we know
// it deletes the key in the current level. Also look for a range
// deletion at the current level (level == item.index). If we find such a
@@ -593,16 +662,16 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool {
// direction.
continue
}
if m.heap.cmp(l.tombstone.End, item.key.UserKey) <= 0 {
if m.heap.cmp(l.tombstone.End, item.iterKey.UserKey) <= 0 {
// The current key is at or past the tombstone end key.
//
// NB: for the case that this l.rangeDelIter is provided by a levelIter we know that
// the levelIter must be positioned at a key >= item.key. So it is sufficient to seek the
// the levelIter must be positioned at a key >= item.iterKey. So it is sufficient to seek the
// current l.rangeDelIter (since any range del iterators that will be provided by the
// levelIter in the future cannot contain item.key). Also, it is possible that we
// levelIter in the future cannot contain item.iterKey). Also, it is possible that we
// will encounter parts of the range delete that should be ignored -- we handle that
// below.
l.tombstone = keyspan.SeekGE(m.heap.cmp, l.rangeDelIter, item.key.UserKey)
l.tombstone = l.rangeDelIter.SeekGE(item.iterKey.UserKey)
}
if l.tombstone == nil {
continue
@@ -611,9 +680,9 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool {
// Reasoning for correctness of untruncated tombstone handling when the untruncated
// tombstone is at a higher level:
// The iterator corresponding to this tombstone is still in the heap so it must be
// positioned >= item.key. Which means the Largest key bound of the sstable containing this
// tombstone is >= item.key. So the upper limit of this tombstone cannot be file-bounds-constrained
// to < item.key. But it is possible that item.key < smallestUserKey, in which
// positioned >= item.iterKey. Which means the Largest key bound of the sstable containing this
// tombstone is >= item.iterKey. So the upper limit of this tombstone cannot be file-bounds-constrained
// to < item.iterKey. But it is possible that item.key < smallestUserKey, in which
// case this tombstone should be ignored.
//
// Example 1:
@@ -626,11 +695,11 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool {
// does not cover this key.
//
// For a tombstone at the same level as the key, the file bounds are trivially satisfied.
if (l.smallestUserKey == nil || m.heap.cmp(l.smallestUserKey, item.key.UserKey) <= 0) &&
l.tombstone.VisibleAt(m.snapshot) && l.tombstone.Contains(m.heap.cmp, item.key.UserKey) {
if (l.smallestUserKey == nil || m.heap.cmp(l.smallestUserKey, item.iterKey.UserKey) <= 0) &&
l.tombstone.VisibleAt(m.snapshot) && l.tombstone.Contains(m.heap.cmp, item.iterKey.UserKey) {
if level < item.index {
// We could also do m.seekGE(..., level + 1). The levels from
// [level + 1, item.index) are already after item.key so seeking them may be
// [level + 1, item.index) are already after item.iterKey so seeking them may be
// wasteful.
// We can seek up to the min of largestUserKey and tombstone.End.
@@ -642,33 +711,70 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool {
// the exclusive bound of the tombstone. We do not need to look at
// isLargestKeyRangeDelSentinel.
//
// Progress argument: Since this file is at a higher level than item.key we know
// Progress argument: Since this file is at a higher level than item.iterKey we know
// that the iterator in this file must be positioned within its bounds and at a key
// X > item.key (otherwise it would be the min of the heap). It is not
// possible for X.UserKey == item.key.UserKey, since it is incompatible with
// X > item.key (a lower version cannot be in a higher sstable), so it must be that
// X.UserKey > item.key.UserKey. Which means l.largestUserKey > item.key.UserKey.
// We also know that l.tombstone.End > item.key.UserKey. So the min of these,
// seekKey, computed below, is > item.key.UserKey, so the call to seekGE() will
// X > item.iterKey (otherwise it would be the min of the heap). It is not
// possible for X.UserKey == item.iterKey.UserKey, since it is incompatible with
// X > item.iterKey (a lower version cannot be in a higher sstable), so it must be that
// X.UserKey > item.iterKey.UserKey. Which means l.largestUserKey > item.key.UserKey.
// We also know that l.tombstone.End > item.iterKey.UserKey. So the min of these,
// seekKey, computed below, is > item.iterKey.UserKey, so the call to seekGE() will
// make forward progress.
seekKey := l.tombstone.End
if l.largestUserKey != nil && m.heap.cmp(l.largestUserKey, seekKey) < 0 {
seekKey = l.largestUserKey
}
// This seek is not directly due to a SeekGE call, so we don't
// know enough about the underlying iterator positions, and so
// we keep the try-seek-using-next optimization disabled.
// This seek is not directly due to a SeekGE call, so we don't know
// enough about the underlying iterator positions, and so we keep the
// try-seek-using-next optimization disabled. Additionally, if we're in
// prefix-seek mode and a re-seek would have moved us past the original
// prefix, we can remove all merging iter levels below the rangedel
// tombstone's level and return immediately instead of re-seeking. This
// is correct since those levels cannot provide a key that matches the
// prefix, and is also visible. Additionally, this is important to make
// subsequent `TrySeekUsingNext` work correctly, as a re-seek on a
// different prefix could have resulted in this iterator skipping visible
// keys at prefixes in between m.prefix and seekKey, that are currently
// not in the heap due to a bloom filter mismatch.
//
// Additionally, we set the relative-seek flag. This is
// important when iterating with lazy combined iteration. If
// there's a range key between this level's current file and the
// file the seek will land on, we need to detect it in order to
// trigger construction of the combined iterator.
if m.prefix != nil {
if n := m.split(seekKey); !bytes.Equal(m.prefix, seekKey[:n]) {
for i := item.index; i < len(m.levels); i++ {
// Remove this level from the heap. Setting iterKey and iterValue
// to their zero values should be sufficient for initMinHeap to not
// re-initialize the heap with them in it. Other fields in
// mergingIterLevel can remain as-is; the iter/rangeDelIter needs
// to stay intact for future trySeekUsingNexts to work, the level
// iter boundary context is owned by the levelIter which is not
// being repositioned, and any tombstones in these levels will be
// irrelevant for us anyway.
m.levels[i].iterKey = nil
m.levels[i].iterValue = base.LazyValue{}
}
// TODO(bilal): Consider a more efficient way of removing levels from
// the heap without reinitializing all of it. This would likely
// necessitate tracking the heap positions of each mergingIterHeap
// item in the mergingIterLevel, and then swapping that item in the
// heap with the last-positioned heap item, and shrinking the heap by
// one.
m.initMinHeap()
return true
}
}
m.seekGE(seekKey, item.index, base.SeekGEFlagsNone.EnableRelativeSeek())
return true
}
if l.tombstone.CoversAt(m.snapshot, item.key.SeqNum()) {
m.nextEntry(item)
if l.tombstone.CoversAt(m.snapshot, item.iterKey.SeqNum()) {
if m.prefix == nil {
m.nextEntry(item, nil /* succKey */)
} else {
m.maybeNextEntryWithinPrefix(item)
}
return true
}
}
@@ -678,73 +784,56 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool {
// Starting from the current entry, finds the first (next) entry that can be returned.
func (m *mergingIter) findNextEntry() (*InternalKey, base.LazyValue) {
var reseeked bool
for m.heap.len() > 0 && m.err == nil {
item := &m.heap.items[0]
item := m.heap.items[0]
if m.levels[item.index].isSyntheticIterBoundsKey {
break
}
// For prefix iteration, stop if we've already exceeded the iterator's
// current prefix. There are two cases where we perform this check.
//
// 1. We already re-seeked the iterator due to a range tombstone. We
// could amortize the cost of this comparison, by doing it only after
// we have iterated in this for loop a few times. But unless we find
// a performance benefit to that, we do the simple thing and compare
// each time. Note that isNextEntryDeleted already did at least 4 key
// comparisons in order to return true, and additionally at least one
// heap comparison to step to the next entry.
//
// Note that we cannot move this comparison into the
// isNextEntryDeleted branch. Once isNextEntryDeleted determines a
// key is deleted and seeks the level's iterator, item.key's memory
// is potentially invalid. If the iterator is now exhausted, item.key
// may be garbage.
//
// 2. If the heap root is an ignorable boundary key, Next-ing the
// iterator will close the associated file and open the next file in
// the level. We don't want to do this unnecessarily because the
// 'exhausted' file may only appear exhausted because it failed a
// bloom filter check. Avoiding Next-ing the file has a performance
// benefit of unnecessarily advancing to the next file, but it's also
// necessary for correctness of some TrySeekUsingNext optimizations
// deeper in the iterator stack:
//
// There may exist valid keys between our current prefix (m.prefix)
// and the ignorable boundary key (item.key) that were not returned
// due to the bloom filter test. A subsequent SeekPrefixGE call with
// a seek key in the range (m.prefix - item.key) will have the
// TrySeekUsingNext flag enabled. The levelIter relies on having not
// skipped these keys in order to avoid more expensive full seeks.
if m.prefix != nil && (reseeked || m.levels[item.index].isIgnorableBoundaryKey) {
if n := m.split(item.key.UserKey); !bytes.Equal(m.prefix, item.key.UserKey[:n]) {
return nil, base.LazyValue{}
}
}
m.addItemStats(item)
if m.isNextEntryDeleted(item) {
m.stats.PointsCoveredByRangeTombstones++
reseeked = true
// Skip ignorable boundary keys. These are not real keys and exist to
// keep sstables open until we've surpassed their end boundaries so that
// their range deletions are visible.
if m.levels[item.index].isIgnorableBoundaryKey {
if m.prefix == nil {
m.nextEntry(item, nil /* succKey */)
} else {
m.maybeNextEntryWithinPrefix(item)
}
continue
}
if item.key.Visible(m.snapshot, m.batchSnapshot) &&
(!m.levels[item.index].isIgnorableBoundaryKey) &&
(item.key.Kind() != InternalKeyKindRangeDelete || !m.elideRangeTombstones) {
return &item.key, item.value
// Check if the heap root key is deleted by a range tombstone in a
// higher level. If it is, isNextEntryDeleted will advance the iterator
// to a later key (through seeking or nexting).
if m.isNextEntryDeleted(item) {
m.stats.PointsCoveredByRangeTombstones++
continue
}
m.nextEntry(item)
// Check if the key is visible at the iterator sequence numbers.
if !item.iterKey.Visible(m.snapshot, m.batchSnapshot) {
if m.prefix == nil {
m.nextEntry(item, nil /* succKey */)
} else {
m.maybeNextEntryWithinPrefix(item)
}
continue
}
// The heap root is visible and not deleted by any range tombstones.
// Return it.
return item.iterKey, item.iterValue
}
return nil, base.LazyValue{}
}
// Steps to the prev entry. item is the current top item in the heap.
func (m *mergingIter) prevEntry(item *mergingIterItem) {
l := &m.levels[item.index]
oldTopLevel := item.index
func (m *mergingIter) prevEntry(l *mergingIterLevel) {
oldTopLevel := l.index
oldRangeDelIter := l.rangeDelIter
if l.iterKey, l.iterValue = l.iter.Prev(); l.iterKey != nil {
item.key, item.value = *l.iterKey, l.iterValue
if m.heap.len() > 1 {
m.heap.fix(0)
}
@@ -770,8 +859,8 @@ func (m *mergingIter) prevEntry(item *mergingIterItem) {
// isPrevEntryDeleted() starts from the current entry (as the prev entry) and if it is deleted,
// moves the iterators backward as needed and returns true, else it returns false. item is the top
// item in the heap.
func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool {
// Look for a range deletion tombstone containing item.key at higher
func (m *mergingIter) isPrevEntryDeleted(item *mergingIterLevel) bool {
// Look for a range deletion tombstone containing item.iterKey at higher
// levels (level < item.index). If we find such a range tombstone we know
// it deletes the key in the current level. Also look for a range
// deletion at the current level (level == item.index). If we find such a
@@ -785,16 +874,16 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool {
// direction.
continue
}
if m.heap.cmp(item.key.UserKey, l.tombstone.Start) < 0 {
if m.heap.cmp(item.iterKey.UserKey, l.tombstone.Start) < 0 {
// The current key is before the tombstone start key.
//
// NB: for the case that this l.rangeDelIter is provided by a levelIter we know that
// the levelIter must be positioned at a key < item.key. So it is sufficient to seek the
// the levelIter must be positioned at a key < item.iterKey. So it is sufficient to seek the
// current l.rangeDelIter (since any range del iterators that will be provided by the
// levelIter in the future cannot contain item.key). Also, it is it is possible that we
// levelIter in the future cannot contain item.iterKey). Also, it is it is possible that we
// will encounter parts of the range delete that should be ignored -- we handle that
// below.
l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.key.UserKey)
l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.iterKey.UserKey)
}
if l.tombstone == nil {
continue
@@ -804,9 +893,9 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool {
// tombstone is at a higher level:
//
// The iterator corresponding to this tombstone is still in the heap so it must be
// positioned <= item.key. Which means the Smallest key bound of the sstable containing this
// tombstone is <= item.key. So the lower limit of this tombstone cannot have been
// file-bounds-constrained to > item.key. But it is possible that item.key >= Largest
// positioned <= item.iterKey. Which means the Smallest key bound of the sstable containing this
// tombstone is <= item.iterKey. So the lower limit of this tombstone cannot have been
// file-bounds-constrained to > item.iterKey. But it is possible that item.key >= Largest
// key bound of this sstable, in which case this tombstone should be ignored.
//
// Example 1:
@@ -823,20 +912,20 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool {
// Example 3:
// sstable bounds [c#8, g#RangeDelSentinel] containing [b, i)#7 and the key is g#10.
// This key is not deleted by this tombstone. We need to look at
// isLargestUserKeyRangeDelSentinel.
// isLargestUserKeyExclusive.
//
// For a tombstone at the same level as the key, the file bounds are trivially satisfied.
// Default to within bounds.
withinLargestSSTableBound := true
if l.largestUserKey != nil {
cmpResult := m.heap.cmp(l.largestUserKey, item.key.UserKey)
withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyRangeDelSentinel)
cmpResult := m.heap.cmp(l.largestUserKey, item.iterKey.UserKey)
withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyExclusive)
}
if withinLargestSSTableBound && l.tombstone.Contains(m.heap.cmp, item.key.UserKey) && l.tombstone.VisibleAt(m.snapshot) {
if withinLargestSSTableBound && l.tombstone.Contains(m.heap.cmp, item.iterKey.UserKey) && l.tombstone.VisibleAt(m.snapshot) {
if level < item.index {
// We could also do m.seekLT(..., level + 1). The levels from
// [level + 1, item.index) are already before item.key so seeking them may be
// [level + 1, item.index) are already before item.iterKey so seeking them may be
// wasteful.
// We can seek up to the max of smallestUserKey and tombstone.Start.UserKey.
@@ -844,10 +933,10 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool {
// Using example 1 above, we can seek to the larger of c and b, which is c.
//
// Progress argument: We know that the iterator in this file is positioned within
// its bounds and at a key X < item.key (otherwise it would be the max of the heap).
// So smallestUserKey <= item.key.UserKey and we already know that
// l.tombstone.Start.UserKey <= item.key.UserKey. So the seekKey computed below
// is <= item.key.UserKey, and since we do a seekLT() we will make backwards
// its bounds and at a key X < item.iterKey (otherwise it would be the max of the heap).
// So smallestUserKey <= item.iterKey.UserKey and we already know that
// l.tombstone.Start.UserKey <= item.iterKey.UserKey. So the seekKey computed below
// is <= item.iterKey.UserKey, and since we do a seekLT() we will make backwards
// progress.
seekKey := l.tombstone.Start
if l.smallestUserKey != nil && m.heap.cmp(l.smallestUserKey, seekKey) > 0 {
@@ -861,7 +950,7 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool {
m.seekLT(seekKey, item.index, base.SeekLTFlagsNone.EnableRelativeSeek())
return true
}
if l.tombstone.CoversAt(m.snapshot, item.key.SeqNum()) {
if l.tombstone.CoversAt(m.snapshot, item.iterKey.SeqNum()) {
m.prevEntry(item)
return true
}
@@ -873,7 +962,7 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool {
// Starting from the current entry, finds the first (prev) entry that can be returned.
func (m *mergingIter) findPrevEntry() (*InternalKey, base.LazyValue) {
for m.heap.len() > 0 && m.err == nil {
item := &m.heap.items[0]
item := m.heap.items[0]
if m.levels[item.index].isSyntheticIterBoundsKey {
break
}
@@ -882,10 +971,9 @@ func (m *mergingIter) findPrevEntry() (*InternalKey, base.LazyValue) {
m.stats.PointsCoveredByRangeTombstones++
continue
}
if item.key.Visible(m.snapshot, m.batchSnapshot) &&
(!m.levels[item.index].isIgnorableBoundaryKey) &&
(item.key.Kind() != InternalKeyKindRangeDelete || !m.elideRangeTombstones) {
return &item.key, item.value
if item.iterKey.Visible(m.snapshot, m.batchSnapshot) &&
(!m.levels[item.index].isIgnorableBoundaryKey) {
return item.iterKey, item.iterValue
}
m.prevEntry(item)
}
@@ -915,6 +1003,37 @@ func (m *mergingIter) seekGE(key []byte, level int, flags base.SeekGEFlags) {
// level (and any lower levels) when the current iterator position is
// contained within a range tombstone at a higher level.
// Deterministically disable the TrySeekUsingNext optimizations sometimes in
// invariant builds to encourage the metamorphic tests to surface bugs. Note
// that we cannot disable the optimization within individual levels. It must
// be disabled for all levels or none. If one lower-level iterator performs
// a fresh seek whereas another takes advantage of its current iterator
// position, the heap can become inconsistent. Consider the following
// example:
//
// L5: [ [b-c) ] [ d ]*
// L6: [ b ] [e]*
//
// Imagine a SeekGE(a). The [b-c) range tombstone deletes the L6 point key
// 'b', resulting in the iterator positioned at d with the heap:
//
// {L5: d, L6: e}
//
// A subsequent SeekGE(b) is seeking to a larger key, so the caller may set
// TrySeekUsingNext()=true. If the L5 iterator used the TrySeekUsingNext
// optimization but the L6 iterator did not, the iterator would have the
// heap:
//
// {L6: b, L5: d}
//
// Because the L5 iterator has already advanced to the next sstable, the
// merging iterator cannot observe the [b-c) range tombstone and will
// mistakenly return L6's deleted point key 'b'.
if invariants.Enabled && flags.TrySeekUsingNext() && !m.forceEnableSeekOpt &&
disableSeekOpt(key, uintptr(unsafe.Pointer(m))) {
flags = flags.DisableTrySeekUsingNext()
}
for ; level < len(m.levels); level++ {
if invariants.Enabled && m.lower != nil && m.heap.cmp(key, m.lower) < 0 {
m.logger.Fatalf("mergingIter: lower bound violation: %s < %s\n%s", key, m.lower, debug.Stack())
@@ -950,7 +1069,7 @@ func (m *mergingIter) seekGE(key []byte, level int, flags base.SeekGEFlags) {
// so we can have a sstable with bounds [c#8, i#InternalRangeDelSentinel], and the
// tombstone is [b, k)#8 and the seek key is i: levelIter.SeekGE(i) will move past
// this sstable since it realizes the largest key is a InternalRangeDelSentinel.
l.tombstone = keyspan.SeekGE(m.heap.cmp, rangeDelIter, key)
l.tombstone = rangeDelIter.SeekGE(key)
if l.tombstone != nil && l.tombstone.VisibleAt(m.snapshot) && l.tombstone.Contains(m.heap.cmp, key) &&
(l.smallestUserKey == nil || m.heap.cmp(l.smallestUserKey, key) <= 0) {
// NB: Based on the comment above l.largestUserKey >= key, and based on the
@@ -1040,7 +1159,7 @@ func (m *mergingIter) seekLT(key []byte, level int, flags base.SeekLTFlags) {
withinLargestSSTableBound := true
if l.largestUserKey != nil {
cmpResult := m.heap.cmp(l.largestUserKey, key)
withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyRangeDelSentinel)
withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyExclusive)
}
l.tombstone = keyspan.SeekLE(m.heap.cmp, rangeDelIter, key)
@@ -1123,7 +1242,59 @@ func (m *mergingIter) Next() (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
m.nextEntry(&m.heap.items[0])
// NB: It's okay to call nextEntry directly even during prefix iteration
// mode (as opposed to indirectly through maybeNextEntryWithinPrefix).
// During prefix iteration mode, we rely on the caller to not call Next if
// the iterator has already advanced beyond the iteration prefix. See the
// comment above the base.InternalIterator interface.
m.nextEntry(m.heap.items[0], nil /* succKey */)
return m.findNextEntry()
}
func (m *mergingIter) NextPrefix(succKey []byte) (*InternalKey, LazyValue) {
if m.dir != 1 {
panic("pebble: cannot switch directions with NextPrefix")
}
if m.err != nil || m.heap.len() == 0 {
return nil, LazyValue{}
}
if m.levelsPositioned == nil {
m.levelsPositioned = make([]bool, len(m.levels))
} else {
for i := range m.levelsPositioned {
m.levelsPositioned[i] = false
}
}
// The heap root necessarily must be positioned at a key < succKey, because
// NextPrefix was invoked.
root := &m.heap.items[0]
m.levelsPositioned[(*root).index] = true
if invariants.Enabled && m.heap.cmp((*root).iterKey.UserKey, succKey) >= 0 {
m.logger.Fatalf("pebble: invariant violation: NextPrefix(%q) called on merging iterator already positioned at %q",
succKey, (*root).iterKey)
}
m.nextEntry(*root, succKey)
// NB: root is a pointer to the heap root. nextEntry may have changed
// the heap root, so we must not expect root to still point to the same
// level (or to even be valid, if the heap is now exhaused).
for m.heap.len() > 0 {
if m.levelsPositioned[(*root).index] {
// A level we've previously positioned is at the top of the heap, so
// there are no other levels positioned at keys < succKey. We've
// advanced as far as we need to.
break
}
// Since this level was not the original heap root when NextPrefix was
// called, we don't know whether this level's current key has the
// previous prefix or a new one.
if m.heap.cmp((*root).iterKey.UserKey, succKey) >= 0 {
break
}
m.levelsPositioned[(*root).index] = true
m.nextEntry(*root, succKey)
}
return m.findNextEntry()
}
@@ -1145,7 +1316,7 @@ func (m *mergingIter) Prev() (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
m.prevEntry(&m.heap.items[0])
m.prevEntry(m.heap.items[0])
return m.findPrevEntry()
}
@@ -1188,7 +1359,7 @@ func (m *mergingIter) DebugString() string {
sep := ""
for m.heap.len() > 0 {
item := m.heap.pop()
fmt.Fprintf(&buf, "%s%s", sep, item.key)
fmt.Fprintf(&buf, "%s%s", sep, item.iterKey)
sep = " "
}
if m.dir == 1 {
@@ -1209,10 +1380,10 @@ func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) {
}
}
func (m *mergingIter) addItemStats(item *mergingIterItem) {
func (m *mergingIter) addItemStats(l *mergingIterLevel) {
m.stats.PointCount++
m.stats.KeyBytes += uint64(len(item.key.UserKey))
m.stats.ValueBytes += uint64(len(item.value.ValueOrHandle))
m.stats.KeyBytes += uint64(len(l.iterKey.UserKey))
m.stats.ValueBytes += uint64(len(l.iterValue.ValueOrHandle))
}
var _ internalIterator = &mergingIter{}

16
vendor/github.com/cockroachdb/pebble/merging_iter_heap.go generated vendored
View File

@@ -4,18 +4,10 @@
package pebble
import "github.com/cockroachdb/pebble/internal/base"
type mergingIterItem struct {
index int
key InternalKey
value base.LazyValue
}
type mergingIterHeap struct {
cmp Compare
reverse bool
items []mergingIterItem
items []*mergingIterLevel
}
func (h *mergingIterHeap) len() int {
@@ -27,7 +19,7 @@ func (h *mergingIterHeap) clear() {
}
func (h *mergingIterHeap) less(i, j int) bool {
ikey, jkey := h.items[i].key, h.items[j].key
ikey, jkey := h.items[i].iterKey, h.items[j].iterKey
if c := h.cmp(ikey.UserKey, jkey.UserKey); c != 0 {
if h.reverse {
return c > 0
@@ -59,11 +51,11 @@ func (h *mergingIterHeap) fix(i int) {
}
}
func (h *mergingIterHeap) pop() *mergingIterItem {
func (h *mergingIterHeap) pop() *mergingIterLevel {
n := h.len() - 1
h.swap(0, n)
h.down(0, n)
item := &h.items[n]
item := h.items[n]
h.items = h.items[:n]
return item
}

131
vendor/github.com/cockroachdb/pebble/metrics.go generated vendored
View File

@@ -6,6 +6,7 @@ package pebble
import (
"fmt"
"strings"
"time"
"github.com/cockroachdb/pebble/internal/base"
@@ -78,6 +79,18 @@ type LevelMetrics struct {
TablesIngested uint64
// The number of sstables moved to this level by a "move" compaction.
TablesMoved uint64
// Additional contains misc additional metrics that are not always printed.
Additional struct {
// The sum of Properties.ValueBlocksSize for all the sstables in this
// level. Printed by LevelMetrics.format iff there is at least one level
// with a non-zero value.
ValueBlocksSize uint64
// Cumulative metrics about bytes written to data blocks and value blocks,
// via compactions (except move compactions) or flushes. Not printed by
// LevelMetrics.format, but are available to sophisticated clients.
BytesWrittenDataBlocks uint64
BytesWrittenValueBlocks uint64
}
}
// Add updates the counter metrics for the level.
@@ -94,6 +107,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) {
m.TablesFlushed += u.TablesFlushed
m.TablesIngested += u.TablesIngested
m.TablesMoved += u.TablesMoved
m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks
m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks
m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize
}
// WriteAmp computes the write amplification for compactions at this
@@ -107,8 +123,10 @@ func (m *LevelMetrics) WriteAmp() float64 {
// format generates a string of the receiver's metrics, formatting it into the
// supplied buffer.
func (m *LevelMetrics) format(w redact.SafePrinter, score redact.SafeValue) {
w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f\n",
func (m *LevelMetrics) format(
w redact.SafePrinter, score redact.SafeValue, includeValueBlocksSize bool,
) {
w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f",
redact.Safe(m.NumFiles),
humanize.IEC.Int64(m.Size),
score,
@@ -122,6 +140,11 @@ func (m *LevelMetrics) format(w redact.SafePrinter, score redact.SafeValue) {
humanize.IEC.Uint64(m.BytesRead),
redact.Safe(m.Sublevels),
redact.Safe(m.WriteAmp()))
if includeValueBlocksSize {
w.Printf(" %7s\n", humanize.IEC.Uint64(m.Additional.ValueBlocksSize))
} else {
w.SafeString("\n")
}
}
// Metrics holds metrics for various subsystems of the DB such as the Cache,
@@ -156,12 +179,27 @@ type Metrics struct {
// compaction. Such files are compacted in a rewrite compaction
// when no other compactions are picked.
MarkedFiles int
// Duration records the cumulative duration of all compactions since the
// database was opened.
Duration time.Duration
}
Flush struct {
// The total number of flushes.
Count int64
WriteThroughput ThroughputMetric
// Number of flushes that are in-progress. In the current implementation
// this will always be zero or one.
NumInProgress int64
// AsIngestCount is a monotonically increasing counter of flush operations
// handling ingested tables.
AsIngestCount uint64
// AsIngestCount is a monotonically increasing counter of tables ingested as
// flushables.
AsIngestTableCount uint64
// AsIngestBytes is a monotonically increasing counter of the bytes flushed
// for flushables that originated as ingestion operations.
AsIngestBytes uint64
}
Filter FilterMetrics
@@ -184,6 +222,9 @@ type Metrics struct {
Keys struct {
// The approximate count of internal range key set keys in the database.
RangeKeySetsCount uint64
// The approximate count of internal tombstones (DEL, SINGLEDEL and
// RANGEDEL key kinds) within the database.
TombstoneCount uint64
}
Snapshots struct {
@@ -191,6 +232,14 @@ type Metrics struct {
Count int
// The sequence number of the earliest, currently open snapshot.
EarliestSeqNum uint64
// A running tally of keys written to sstables during flushes or
// compactions that would've been elided if it weren't for open
// snapshots.
PinnedKeys uint64
// A running cumulative sum of the size of keys and values written to
// sstables during flushes or compactions that would've been elided if
// it weren't for open snapshots.
PinnedSize uint64
}
Table struct {
@@ -210,6 +259,8 @@ type Metrics struct {
// Count of the number of open sstable iterators.
TableIters int64
// Uptime is the total time since this DB was opened.
Uptime time.Duration
WAL struct {
// Number of live WAL files.
@@ -329,27 +380,27 @@ func (m *Metrics) formatWAL(w redact.SafePrinter) {
// String pretty-prints the metrics, showing a line for the WAL, a line per-level, and
// a total:
//
// __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___w-amp
// WAL 1 27 B - 48 B - - - - 108 B - - 2.2
// 0 2 1.6 K 0.50 81 B 825 B 1 0 B 0 2.4 K 3 0 B 30.6
// 1 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0
// 2 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0
// 3 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0
// 4 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0
// 5 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0
// 6 1 825 B 0.00 1.6 K 0 B 0 0 B 0 825 B 1 1.6 K 0.5
// total 3 2.4 K - 933 B 825 B 1 0 B 0 4.1 K 4 1.6 K 4.5
// flush 3
// compact 1 1.6 K 0 B 1 (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)
// ctype 0 0 0 0 0 (default, delete, elision, move, read)
// memtbl 1 4.0 M
// zmemtbl 0 0 B
// ztbl 0 0 B
// bcache 4 752 B 7.7% (score == hit-rate)
// tcache 0 0 B 0.0% (score == hit-rate)
// snapshots 0 0 (score == earliest seq num)
// titers 0
// filter - - 0.0% (score == utility)
// __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp
// WAL 1 28 B - 17 B - - - - 56 B - - - 3.3
// 0 1 770 B 0.25 28 B 0 B 0 0 B 0 770 B 1 0 B 1 27.5
// 1 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 2 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 3 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 4 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 5 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 6 0 0 B - 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// total 1 770 B - 56 B 0 B 0 0 B 0 826 B 1 0 B 1 14.8
// flush 1 0 B 0 0 (ingest = ingested-as-flushable, move = tables-ingested)
// compact 0 0 B 0 B 0 (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)
// ctype 0 0 0 0 0 0 0 (default, delete, elision, move, read, rewrite, multi-level)
// memtbl 1 256 K
// zmemtbl 1 256 K
// ztbl 0 0 B
// bcache 4 697 B 0.0% (score == hit-rate)
// tcache 1 696 B 0.0% (score == hit-rate)
// snaps 0 - 0 (score == earliest seq num)
// titers 1
// filter - - 0.0% (score == utility)
//
// The WAL "in" metric is the size of the batches written to the WAL. The WAL
// "write" metric is the size of the physical data written to the WAL which
@@ -376,9 +427,18 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
// width specifiers. When the issue is fixed, we can convert these to
// RedactableStrings. https://github.com/cockroachdb/redact/issues/17
haveValueBlocks := false
var valueBlocksHeading redact.SafeString
for level := 0; level < numLevels; level++ {
if m.Levels[level].Additional.ValueBlocksSize > 0 {
haveValueBlocks = true
valueBlocksHeading = "__val-bl"
break
}
}
var total LevelMetrics
w.SafeString("__level_____count____size___score______in__ingest(sz_cnt)" +
"____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp\n")
w.Printf("__level_____count____size___score______in__ingest(sz_cnt)"+
"____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp%s\n", valueBlocksHeading)
m.formatWAL(w)
for level := 0; level < numLevels; level++ {
l := &m.Levels[level]
@@ -389,7 +449,7 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
if level < numLevels-1 {
score = redact.Safe(fmt.Sprintf("%0.2f", l.Score))
}
l.format(w, score)
l.format(w, score, haveValueBlocks)
total.Add(l)
total.Sublevels += l.Sublevels
}
@@ -400,23 +460,30 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
// ingested.
total.BytesFlushed += total.BytesIn
w.SafeString(" total ")
total.format(w, notApplicable)
total.format(w, notApplicable, haveValueBlocks)
w.Printf(" flush %9d\n", redact.Safe(m.Flush.Count))
w.Printf("compact %9d %7s %7s %7d %7s (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)\n",
w.Printf(" flush %9d %31s %7d %7d %s\n",
redact.Safe(m.Flush.Count),
humanize.IEC.Uint64(m.Flush.AsIngestBytes),
redact.Safe(m.Flush.AsIngestTableCount),
redact.Safe(m.Flush.AsIngestCount),
redact.SafeString(`(ingest = tables-ingested, move = ingested-as-flushable)`))
w.Printf("compact %9d %7s %7s %7d %s %s\n",
redact.Safe(m.Compact.Count),
humanize.IEC.Uint64(m.Compact.EstimatedDebt),
humanize.IEC.Int64(m.Compact.InProgressBytes),
redact.Safe(m.Compact.NumInProgress),
redact.SafeString(""))
w.Printf(" ctype %9d %7d %7d %7d %7d %7d %7d (default, delete, elision, move, read, rewrite, multi-level)\n",
redact.SafeString(strings.Repeat(" ", 24)),
redact.SafeString(`(size == estimated-debt, score = in-progress-bytes, in = num-in-progress)`))
w.Printf(" ctype %9d %7d %7d %7d %7d %7d %7d %s\n",
redact.Safe(m.Compact.DefaultCount),
redact.Safe(m.Compact.DeleteOnlyCount),
redact.Safe(m.Compact.ElisionOnlyCount),
redact.Safe(m.Compact.MoveCount),
redact.Safe(m.Compact.ReadCount),
redact.Safe(m.Compact.RewriteCount),
redact.Safe(m.Compact.MultiLevelCount))
redact.Safe(m.Compact.MultiLevelCount),
redact.SafeString(`(default, delete, elision, move, read, rewrite, multi-level)`))
w.Printf(" memtbl %9d %7s\n",
redact.Safe(m.MemTable.Count),
humanize.IEC.Uint64(m.MemTable.Size))

34
vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go generated vendored Normal file
View File

@@ -0,0 +1,34 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import "context"
// NoopReadHandle can be used by Readable implementations that don't
// support read-ahead.
type NoopReadHandle struct {
readable Readable
}
// MakeNoopReadHandle initializes a NoopReadHandle.
func MakeNoopReadHandle(r Readable) NoopReadHandle {
return NoopReadHandle{readable: r}
}
var _ ReadHandle = (*NoopReadHandle)(nil)
// ReadAt is part of the ReadHandle interface.
func (h *NoopReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error {
return h.readable.ReadAt(ctx, p, off)
}
// Close is part of the ReadHandle interface.
func (*NoopReadHandle) Close() error { return nil }
// MaxReadahead is part of the ReadHandle interface.
func (*NoopReadHandle) MaxReadahead() {}
// RecordCacheHit is part of the ReadHandle interface.
func (*NoopReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {}

252
vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go generated vendored Normal file
View File

@@ -0,0 +1,252 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import (
"context"
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/vfs"
)
// Readable is the handle for an object that is open for reading.
type Readable interface {
// ReadAt reads len(p) bytes into p starting at offset off.
//
// Clients of ReadAt can execute parallel ReadAt calls on the
// same Readable.
ReadAt(ctx context.Context, p []byte, off int64) error
Close() error
// Size returns the size of the object.
Size() int64
// NewReadHandle creates a read handle for ReadAt requests that are related
// and can benefit from optimizations like read-ahead.
//
// The ReadHandle must be closed before the Readable is closed.
//
// Multiple separate ReadHandles can be used.
NewReadHandle(ctx context.Context) ReadHandle
}
// ReadHandle is used to perform reads that are related and might benefit from
// optimizations like read-ahead.
type ReadHandle interface {
// ReadAt reads len(p) bytes into p starting at offset off.
//
// Parallel ReadAt calls on the same ReadHandle are not allowed.
ReadAt(ctx context.Context, p []byte, off int64) error
Close() error
// MaxReadahead configures the implementation to expect large sequential
// reads. Used to skip any initial read-ahead ramp-up.
MaxReadahead()
// RecordCacheHit informs the implementation that we were able to retrieve a
// block from cache. This is useful for example when the implementation is
// trying to detect a sequential reading pattern.
RecordCacheHit(ctx context.Context, offset, size int64)
}
// Writable is the handle for an object that is open for writing.
// Either Finish or Abort must be called.
type Writable interface {
// Write writes len(p) bytes from p to the underlying object. The data is not
// guaranteed to be durable until Finish is called.
//
// Note that Write *is* allowed to modify the slice passed in, whether
// temporarily or permanently. Callers of Write need to take this into
// account.
Write(p []byte) error
// Finish completes the object and makes the data durable.
// No further calls are allowed after calling Finish.
Finish() error
// Abort gives up on finishing the object. There is no guarantee about whether
// the object exists after calling Abort.
// No further calls are allowed after calling Abort.
Abort()
}
// ObjectMetadata contains the metadata required to be able to access an object.
type ObjectMetadata struct {
DiskFileNum base.DiskFileNum
FileType base.FileType
// The fields below are only set if the object is on shared storage.
Shared struct {
// CreatorID identifies the DB instance that originally created the object.
CreatorID CreatorID
// CreatorFileNum is the identifier for the object within the context of the
// DB instance that originally created the object.
CreatorFileNum base.DiskFileNum
CleanupMethod SharedCleanupMethod
}
}
// IsShared returns true if the object is on shared storage.
func (meta *ObjectMetadata) IsShared() bool {
return meta.Shared.CreatorID.IsSet()
}
// CreatorID identifies the DB instance that originally created a shared object.
// This ID is incorporated in backing object names.
// Must be non-zero.
type CreatorID uint64
// IsSet returns true if the CreatorID is not zero.
func (c CreatorID) IsSet() bool { return c != 0 }
func (c CreatorID) String() string { return fmt.Sprintf("%d", c) }
// SharedCleanupMethod indicates the method for cleaning up unused shared objects.
type SharedCleanupMethod uint8
const (
// SharedRefTracking is used for shared objects for which objstorage providers
// keep track of references via reference marker objects.
SharedRefTracking SharedCleanupMethod = iota
// SharedNoCleanup is used for shared objects that are managed externally; the
// objstorage provider never deletes such objects.
SharedNoCleanup
)
// OpenOptions contains optional arguments for OpenForReading.
type OpenOptions struct {
// MustExist triggers a fatal error if the file does not exist. The fatal
// error message contains extra information helpful for debugging.
MustExist bool
}
// CreateOptions contains optional arguments for Create.
type CreateOptions struct {
// PreferSharedStorage causes the object to be created on shared storage if
// the provider has shared storage configured.
PreferSharedStorage bool
// SharedCleanupMethod is used for the object when it is created on shared storage.
// The default (zero) value is SharedRefTracking.
SharedCleanupMethod SharedCleanupMethod
}
// Provider is a singleton object used to access and manage objects.
//
// An object is conceptually like a large immutable file. The main use of
// objects is for storing sstables; in the future it could also be used for blob
// storage.
//
// The Provider can only manage objects that it knows about - either objects
// created by the provider, or existing objects the Provider was informed about
// via AddObjects.
//
// Objects are currently backed by a vfs.File or a shared.Storage object.
type Provider interface {
// OpenForReading opens an existing object.
OpenForReading(
ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts OpenOptions,
) (Readable, error)
// Create creates a new object and opens it for writing.
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
Create(
ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts CreateOptions,
) (w Writable, meta ObjectMetadata, err error)
// Remove removes an object.
//
// The object is not guaranteed to be durably removed until Sync is called.
Remove(fileType base.FileType, FileNum base.DiskFileNum) error
// Sync flushes the metadata from creation or removal of objects since the last Sync.
// This includes objects that have been Created but for which
// Writable.Finish() has not yet been called.
Sync() error
// LinkOrCopyFromLocal creates a new object that is either a copy of a given
// local file or a hard link (if the new object is created on the same FS, and
// if the FS supports it).
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
LinkOrCopyFromLocal(
ctx context.Context,
srcFS vfs.FS,
srcFilePath string,
dstFileType base.FileType,
dstFileNum base.DiskFileNum,
opts CreateOptions,
) (ObjectMetadata, error)
// Lookup returns the metadata of an object that is already known to the Provider.
// Does not perform any I/O.
Lookup(fileType base.FileType, FileNum base.DiskFileNum) (ObjectMetadata, error)
// Path returns an internal, implementation-dependent path for the object. It is
// meant to be used for informational purposes (like logging).
Path(meta ObjectMetadata) string
// Size returns the size of the object.
Size(meta ObjectMetadata) (int64, error)
// List returns the objects currently known to the provider. Does not perform any I/O.
List() []ObjectMetadata
// SetCreatorID sets the CreatorID which is needed in order to use shared
// objects. Shared object usage is disabled until this method is called the
// first time. Once set, the Creator ID is persisted and cannot change.
//
// Cannot be called if shared storage is not configured for the provider.
SetCreatorID(creatorID CreatorID) error
// SharedObjectBacking encodes the shared object metadata.
SharedObjectBacking(meta *ObjectMetadata) (SharedObjectBackingHandle, error)
// AttachSharedObjects registers existing shared objects with this provider.
AttachSharedObjects(objs []SharedObjectToAttach) ([]ObjectMetadata, error)
Close() error
// IsNotExistError indicates whether the error is known to report that a file or
// directory does not exist.
IsNotExistError(err error) bool
}
// SharedObjectBacking encodes the metadata necessary to incorporate a shared
// object into a different Pebble instance. The encoding is specific to a given
// Provider implementation.
type SharedObjectBacking []byte
// SharedObjectBackingHandle is a container for a SharedObjectBacking which
// ensures that the backing stays valid. A backing can otherwise become invalid
// if this provider unrefs the shared object. The SharedObjectBackingHandle
// delays any unref until Close.
type SharedObjectBackingHandle interface {
// Get returns the backing. The backing is only guaranteed to be valid until
// Close is called (or until the Provider is closed). If Close was already
// called, returns an error.
Get() (SharedObjectBacking, error)
Close()
}
// SharedObjectToAttach contains the arguments needed to attach an existing shared object.
type SharedObjectToAttach struct {
// FileNum is the file number that will be used to refer to this object (in
// the context of this instance).
FileNum base.DiskFileNum
FileType base.FileType
// Backing contains the metadata for the shared object backing (normally
// generated from a different instance, but using the same Provider
// implementation).
Backing SharedObjectBacking
}

68
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,68 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objiotracing
import "github.com/cockroachdb/pebble/internal/base"
// OpType indicates the type of operation.
type OpType uint8
// OpType values.
const (
ReadOp OpType = iota
WriteOp
// RecordCacheHitOp happens when a read is satisfied from the block cache. See
// objstorage.ReadHandle.RecordCacheHit().
RecordCacheHitOp
// MaxReadaheadOp is a "meta operation" that configures a read handle for
// large sequential reads. See objstorage.ReadHandle.MaxReadahead().
MaxReadaheadOp
)
// Reason indicates the higher-level context of the operation.
type Reason uint8
// Reason values.
const (
UnknownReason Reason = iota
ForFlush
ForCompaction
ForIngestion
// TODO(radu): add ForUserFacing.
)
// BlockType indicates the type of data block relevant to an operation.
type BlockType uint8
// BlockType values.
const (
UnknownBlock BlockType = iota
DataBlock
ValueBlock
FilterBlock
MetadataBlock
)
// Event is the on-disk format of a tracing event. It is exported here so that
// trace processing tools can use it by importing this package.
type Event struct {
// Event start time as a Unix time (see time.Time.StartUnixNano()).
// Note that recorded events are not necessarily ordered by time - this is
// because separate event "streams" use local buffers (for performance).
StartUnixNano int64
Op OpType
Reason Reason
BlockType BlockType
// LSM level plus one (with 0 indicating unknown level).
LevelPlusOne uint8
// Hardcoded padding so that struct layout doesn't depend on architecture.
_ uint32
FileNum base.FileNum
// HandleID is a unique identifier corresponding to an objstorage.ReadHandle;
// only set for read operations performed through a ReadHandle.
HandleID uint64
Offset int64
Size int64
}

59
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go generated vendored Normal file
View File

@@ -0,0 +1,59 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !pebble_obj_io_tracing
// +build !pebble_obj_io_tracing
package objiotracing
import (
"context"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// Enabled is used to short circuit tracing-related code in regular builds.
const Enabled = false
// Tracer manages the writing of object IO traces to files.
type Tracer struct{}
// Open creates a Tracer which generates trace files in the given directory.
// Each trace file contains a series of Events (as they are in memory).
func Open(fs vfs.FS, fsDir string) *Tracer {
return nil
}
// Close the tracer, flushing any remaining events.
func (*Tracer) Close() {}
// WrapReadable wraps an objstorage.Readable with one that generates tracing
// events.
func (*Tracer) WrapReadable(
ctx context.Context, r objstorage.Readable, fileNum base.DiskFileNum,
) objstorage.Readable {
return r
}
// WrapWritable wraps an objstorage.Writable with one that generates tracing
// events.
func (t *Tracer) WrapWritable(
ctx context.Context, w objstorage.Writable, fileNum base.DiskFileNum,
) objstorage.Writable {
return w
}
// WithReason creates a context that has an associated Reason (which ends up in
// traces created under that context).
func WithReason(ctx context.Context, reason Reason) context.Context { return ctx }
// WithBlockType creates a context that has an associated BlockType (which ends up in
// traces created under that context).
func WithBlockType(ctx context.Context, blockType BlockType) context.Context { return ctx }
// WithLevel creates a context that has an associated level (which ends up in
// traces created under that context).
func WithLevel(ctx context.Context, level int) context.Context { return ctx }

410
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go generated vendored Normal file
View File

@@ -0,0 +1,410 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build pebble_obj_io_tracing
// +build pebble_obj_io_tracing
package objiotracing
import (
"bufio"
"context"
"fmt"
"math/rand"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// Enabled is used to short circuit tracing-related code in regular builds.
const Enabled = true
// Tracer manages the writing of object IO traces to files.
//
// The tracer runs a background worker goroutine which receives trace event
// buffers over a channel and dumps them to IOTRACES- files. Wrapper
// implementations for Readable, ReadHandle, Writable are producers of traces;
// they maintain internal buffers of events which get flushed to the buffered
// channel when they get full. This allows for minimal synchronization per IO
// (as for most of these structures, an instance only allows a single IO at a
// time).
type Tracer struct {
fs vfs.FS
fsDir string
handleID atomic.Uint64
workerStopCh chan struct{}
workerDataCh chan eventBuf
workerWait sync.WaitGroup
}
// Open creates a Tracer which generates trace files in the given directory.
// Each trace file contains a series of Events (as they are in memory).
func Open(fs vfs.FS, fsDir string) *Tracer {
t := &Tracer{
fs: fs,
fsDir: fsDir,
workerStopCh: make(chan struct{}),
workerDataCh: make(chan eventBuf, channelBufSize),
}
t.handleID.Store(uint64(rand.NewSource(time.Now().UnixNano()).Int63()))
t.workerWait.Add(1)
go t.workerLoop()
return t
}
// Close the tracer, flushing any remaining events.
func (t *Tracer) Close() {
if t.workerStopCh == nil {
return
}
// Tell the worker to stop and wait for it to finish up.
close(t.workerStopCh)
t.workerWait.Wait()
t.workerStopCh = nil
}
// WrapWritable wraps an objstorage.Writable with one that generates tracing
// events.
func (t *Tracer) WrapWritable(
ctx context.Context, w objstorage.Writable, fileNum base.FileNum,
) objstorage.Writable {
return &writable{
w: w,
fileNum: fileNum,
g: makeEventGenerator(ctx, t),
}
}
type writable struct {
w objstorage.Writable
fileNum base.FileNum
curOffset int64
g eventGenerator
}
var _ objstorage.Writable = (*writable)(nil)
// Write is part of the objstorage.Writable interface.
func (w *writable) Write(p []byte) error {
w.g.add(context.Background(), Event{
Op: WriteOp,
FileNum: w.fileNum,
Offset: w.curOffset,
Size: int64(len(p)),
})
// If w.w.Write(p) returns an error, a new writable
// will be used, so even tho all of p may not have
// been written to the underlying "file", it is okay
// to add len(p) to curOffset.
w.curOffset += int64(len(p))
return w.w.Write(p)
}
// Finish is part of the objstorage.Writable interface.
func (w *writable) Finish() error {
w.g.flush()
return w.w.Finish()
}
// Abort is part of the objstorage.Writable interface.
func (w *writable) Abort() {
w.g.flush()
w.w.Abort()
}
// WrapReadable wraps an objstorage.Readable with one that generates tracing
// events.
func (t *Tracer) WrapReadable(
ctx context.Context, r objstorage.Readable, fileNum base.FileNum,
) objstorage.Readable {
res := &readable{
r: r,
fileNum: fileNum,
}
res.mu.g = makeEventGenerator(ctx, t)
return res
}
type readable struct {
r objstorage.Readable
fileNum base.FileNum
mu struct {
sync.Mutex
g eventGenerator
}
}
var _ objstorage.Readable = (*readable)(nil)
// ReadAt is part of the objstorage.Readable interface.
func (r *readable) ReadAt(ctx context.Context, v []byte, off int64) (n int, err error) {
r.mu.Lock()
r.mu.g.add(ctx, Event{
Op: ReadOp,
FileNum: r.fileNum,
Offset: off,
Size: int64(len(v)),
})
r.mu.Unlock()
return r.r.ReadAt(ctx, v, off)
}
// Close is part of the objstorage.Readable interface.
func (r *readable) Close() error {
r.mu.g.flush()
return r.r.Close()
}
// Size is part of the objstorage.Readable interface.
func (r *readable) Size() int64 {
return r.r.Size()
}
// NewReadHandle is part of the objstorage.Readable interface.
func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle {
// It's safe to get the tracer from the generator without the mutex since it never changes.
t := r.mu.g.t
return &readHandle{
rh: r.r.NewReadHandle(ctx),
fileNum: r.fileNum,
handleID: t.handleID.Add(1),
g: makeEventGenerator(ctx, t),
}
}
type readHandle struct {
rh objstorage.ReadHandle
fileNum base.FileNum
handleID uint64
g eventGenerator
}
var _ objstorage.ReadHandle = (*readHandle)(nil)
// ReadAt is part of the objstorage.ReadHandle interface.
func (rh *readHandle) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) {
rh.g.add(ctx, Event{
Op: ReadOp,
FileNum: rh.fileNum,
HandleID: rh.handleID,
Offset: off,
Size: int64(len(p)),
})
return rh.rh.ReadAt(ctx, p, off)
}
// Close is part of the objstorage.ReadHandle interface.
func (rh *readHandle) Close() error {
rh.g.flush()
return rh.rh.Close()
}
// MaxReadahead is part of the objstorage.ReadHandle interface.
func (rh *readHandle) MaxReadahead() {
rh.g.add(context.Background(), Event{
Op: MaxReadaheadOp,
FileNum: rh.fileNum,
HandleID: rh.handleID,
})
rh.rh.MaxReadahead()
}
// RecordCacheHit is part of the objstorage.ReadHandle interface.
func (rh *readHandle) RecordCacheHit(ctx context.Context, offset, size int64) {
rh.g.add(ctx, Event{
Op: RecordCacheHitOp,
FileNum: rh.fileNum,
HandleID: rh.handleID,
Offset: offset,
Size: size,
})
rh.rh.RecordCacheHit(ctx, offset, size)
}
type ctxInfo struct {
reason Reason
blockType BlockType
levelPlusOne uint8
}
func mergeCtxInfo(base, other ctxInfo) ctxInfo {
res := other
if res.reason == 0 {
res.reason = base.reason
}
if res.blockType == 0 {
res.blockType = base.blockType
}
if res.levelPlusOne == 0 {
res.levelPlusOne = base.levelPlusOne
}
return res
}
type ctxInfoKey struct{}
func withInfo(ctx context.Context, info ctxInfo) context.Context {
return context.WithValue(ctx, ctxInfoKey{}, info)
}
func infoFromCtx(ctx context.Context) ctxInfo {
res := ctx.Value(ctxInfoKey{})
if res == nil {
return ctxInfo{}
}
return res.(ctxInfo)
}
// WithReason creates a context that has an associated Reason (which ends up in
// traces created under that context).
func WithReason(ctx context.Context, reason Reason) context.Context {
info := infoFromCtx(ctx)
info.reason = reason
return withInfo(ctx, info)
}
// WithBlockType creates a context that has an associated BlockType (which ends up in
// traces created under that context).
func WithBlockType(ctx context.Context, blockType BlockType) context.Context {
info := infoFromCtx(ctx)
info.blockType = blockType
return withInfo(ctx, info)
}
// WithLevel creates a context that has an associated level (which ends up in
// traces created under that context).
func WithLevel(ctx context.Context, level int) context.Context {
info := infoFromCtx(ctx)
info.levelPlusOne = uint8(level) + 1
return withInfo(ctx, info)
}
const (
eventSize = int(unsafe.Sizeof(Event{}))
targetEntriesPerFile = 256 * 1024 * 1024 / eventSize // 256MB files
eventsPerBuf = 16
channelBufSize = 512 * 1024 / eventsPerBuf // 512K events.
bytesPerFileSync = 128 * 1024
)
type eventBuf struct {
events [eventsPerBuf]Event
num int
}
type eventGenerator struct {
t *Tracer
baseCtxInfo ctxInfo
buf eventBuf
}
func makeEventGenerator(ctx context.Context, t *Tracer) eventGenerator {
return eventGenerator{
t: t,
baseCtxInfo: infoFromCtx(ctx),
}
}
func (g *eventGenerator) flush() {
if g.buf.num > 0 {
g.t.workerDataCh <- g.buf
g.buf.num = 0
}
}
func (g *eventGenerator) add(ctx context.Context, e Event) {
e.StartUnixNano = time.Now().UnixNano()
info := infoFromCtx(ctx)
info = mergeCtxInfo(g.baseCtxInfo, info)
e.Reason = info.reason
e.BlockType = info.blockType
e.LevelPlusOne = info.levelPlusOne
if g.buf.num == eventsPerBuf {
g.flush()
}
g.buf.events[g.buf.num] = e
g.buf.num++
}
type workerState struct {
curFile vfs.File
curBW *bufio.Writer
numEntriesInFile int
}
func (t *Tracer) workerLoop() {
defer t.workerWait.Done()
stopCh := t.workerStopCh
dataCh := t.workerDataCh
var state workerState
t.workerNewFile(&state)
for {
select {
case <-stopCh:
close(dataCh)
// Flush any remaining traces.
for data := range dataCh {
t.workerWriteTraces(&state, data)
}
t.workerCloseFile(&state)
return
case data := <-dataCh:
t.workerWriteTraces(&state, data)
}
}
}
func (t *Tracer) workerWriteTraces(state *workerState, data eventBuf) {
if state.numEntriesInFile >= targetEntriesPerFile {
t.workerCloseFile(state)
t.workerNewFile(state)
}
state.numEntriesInFile += data.num
p := unsafe.Pointer(&data.events[0])
b := unsafe.Slice((*byte)(p), eventSize*data.num)
if _, err := state.curBW.Write(b); err != nil {
panic(err)
}
}
func (t *Tracer) workerNewFile(state *workerState) {
filename := fmt.Sprintf("IOTRACES-%s", time.Now().UTC().Format(time.RFC3339Nano))
file, err := t.fs.Create(t.fs.PathJoin(t.fsDir, filename))
if err != nil {
panic(err)
}
file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
BytesPerSync: bytesPerFileSync,
})
state.curFile = file
state.curBW = bufio.NewWriter(file)
state.numEntriesInFile = 0
}
func (t *Tracer) workerCloseFile(state *workerState) {
if state.curFile != nil {
if err := state.curBW.Flush(); err != nil {
panic(err)
}
if err := state.curFile.Sync(); err != nil {
panic(err)
}
if err := state.curFile.Close(); err != nil {
panic(err)
}
state.curFile = nil
state.curBW = nil
}
}

460
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go generated vendored Normal file
View File

@@ -0,0 +1,460 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"io"
"os"
"sort"
"sync"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/vfs"
)
// provider is the implementation of objstorage.Provider.
type provider struct {
st Settings
fsDir vfs.File
tracer *objiotracing.Tracer
shared sharedSubsystem
mu struct {
sync.RWMutex
shared struct {
// catalogBatch accumulates shared object creations and deletions until
// Sync is called.
catalogBatch sharedobjcat.Batch
}
// localObjectsChanged is set if non-shared objects were created or deleted
// but Sync was not yet called.
localObjectsChanged bool
// knownObjects maintains information about objects that are known to the provider.
// It is initialized with the list of files in the manifest when we open a DB.
knownObjects map[base.DiskFileNum]objstorage.ObjectMetadata
// protectedObjects are objects that cannot be unreferenced because they
// have outstanding SharedObjectBackingHandles. The value is a count of outstanding handles
protectedObjects map[base.DiskFileNum]int
}
}
var _ objstorage.Provider = (*provider)(nil)
// Settings that must be specified when creating the provider.
type Settings struct {
Logger base.Logger
// Local filesystem configuration.
FS vfs.FS
FSDirName string
// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
//
// This is an optional optimization to avoid double listing on Open when the
// higher layer already has a listing. When nil, we obtain the listing on
// Open.
FSDirInitialListing []string
// Cleaner cleans obsolete files from the local filesystem.
//
// The default cleaner uses the DeleteCleaner.
FSCleaner base.Cleaner
// NoSyncOnClose decides whether the implementation will enforce a
// close-time synchronization (e.g., fdatasync() or sync_file_range())
// on files it writes to. Setting this to true removes the guarantee for a
// sync on close. Some implementations can still issue a non-blocking sync.
NoSyncOnClose bool
// BytesPerSync enables periodic syncing of files in order to smooth out
// writes to disk. This option does not provide any persistence guarantee, but
// is used to avoid latency spikes if the OS automatically decides to write
// out a large chunk of dirty filesystem buffers.
BytesPerSync int
// Fields here are set only if the provider is to support shared objects
// (experimental).
Shared struct {
Storage shared.Storage
}
}
// DefaultSettings initializes default settings (with no shared storage),
// suitable for tests and tools.
func DefaultSettings(fs vfs.FS, dirName string) Settings {
return Settings{
Logger: base.DefaultLogger,
FS: fs,
FSDirName: dirName,
FSCleaner: base.DeleteCleaner{},
NoSyncOnClose: false,
BytesPerSync: 512 * 1024, // 512KB
}
}
// Open creates the provider.
func Open(settings Settings) (objstorage.Provider, error) {
return open(settings)
}
func open(settings Settings) (p *provider, _ error) {
fsDir, err := settings.FS.OpenDir(settings.FSDirName)
if err != nil {
return nil, err
}
defer func() {
if p == nil {
fsDir.Close()
}
}()
p = &provider{
st: settings,
fsDir: fsDir,
}
p.mu.knownObjects = make(map[base.DiskFileNum]objstorage.ObjectMetadata)
p.mu.protectedObjects = make(map[base.DiskFileNum]int)
if objiotracing.Enabled {
p.tracer = objiotracing.Open(settings.FS, settings.FSDirName)
}
// Add local FS objects.
if err := p.vfsInit(); err != nil {
return nil, err
}
// Initialize shared subsystem (if configured) and add shared objects.
if err := p.sharedInit(); err != nil {
return nil, err
}
return p, nil
}
// Close is part of the objstorage.Provider interface.
func (p *provider) Close() error {
var err error
if p.fsDir != nil {
err = p.fsDir.Close()
p.fsDir = nil
}
if objiotracing.Enabled {
if p.tracer != nil {
p.tracer.Close()
p.tracer = nil
}
}
return err
}
// OpenForReading opens an existing object.
func (p *provider) OpenForReading(
ctx context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
meta, err := p.Lookup(fileType, fileNum)
if err != nil {
if opts.MustExist {
p.st.Logger.Fatalf("%v", err)
}
return nil, err
}
var r objstorage.Readable
if !meta.IsShared() {
r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts)
} else {
r, err = p.sharedOpenForReading(ctx, meta, opts)
}
if err != nil {
return nil, err
}
if objiotracing.Enabled {
r = p.tracer.WrapReadable(ctx, r, fileNum)
}
return r, nil
}
// Create creates a new object and opens it for writing.
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
func (p *provider) Create(
ctx context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
if opts.PreferSharedStorage && p.st.Shared.Storage != nil {
w, meta, err = p.sharedCreate(ctx, fileType, fileNum, opts)
} else {
w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
}
if err != nil {
err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum))
return nil, objstorage.ObjectMetadata{}, err
}
p.addMetadata(meta)
if objiotracing.Enabled {
w = p.tracer.WrapWritable(ctx, w, fileNum)
}
return w, meta, nil
}
// Remove removes an object.
//
// Note that if the object is shared, the object is only (conceptually) removed
// from this provider. If other providers have references on the shared object,
// it will not be removed.
//
// The object is not guaranteed to be durably removed until Sync is called.
func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) error {
meta, err := p.Lookup(fileType, fileNum)
if err != nil {
return err
}
if !meta.IsShared() {
err = p.vfsRemove(fileType, fileNum)
} else {
// TODO(radu): implement shared object removal (i.e. deref).
err = p.sharedUnref(meta)
}
if err != nil && !p.IsNotExistError(err) {
// We want to be able to retry a Remove, so we keep the object in our list.
// TODO(radu): we should mark the object as "zombie" and not allow any other
// operations.
return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum))
}
p.removeMetadata(fileNum)
return err
}
// IsNotExistError is part of the objstorage.Provider interface.
func (p *provider) IsNotExistError(err error) bool {
if oserror.IsNotExist(err) {
return true
}
if p.sharedStorage() != nil && p.sharedStorage().IsNotExistError(err) {
return true
}
return false
}
// Sync flushes the metadata from creation or removal of objects since the last Sync.
func (p *provider) Sync() error {
if err := p.vfsSync(); err != nil {
return err
}
if err := p.sharedSync(); err != nil {
return err
}
return nil
}
// LinkOrCopyFromLocal creates a new object that is either a copy of a given
// local file or a hard link (if the new object is created on the same FS, and
// if the FS supports it).
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
func (p *provider) LinkOrCopyFromLocal(
ctx context.Context,
srcFS vfs.FS,
srcFilePath string,
dstFileType base.FileType,
dstFileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (objstorage.ObjectMetadata, error) {
shared := opts.PreferSharedStorage && p.st.Shared.Storage != nil
if !shared && srcFS == p.st.FS {
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{
NoSyncOnClose: p.st.NoSyncOnClose,
BytesPerSync: p.st.BytesPerSync,
})
dstPath := p.vfsPath(dstFileType, dstFileNum)
if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
return objstorage.ObjectMetadata{}, err
}
meta := objstorage.ObjectMetadata{
DiskFileNum: dstFileNum,
FileType: dstFileType,
}
p.addMetadata(meta)
return meta, nil
}
// Create the object and copy the data.
w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts)
if err != nil {
return objstorage.ObjectMetadata{}, err
}
f, err := srcFS.Open(srcFilePath, vfs.SequentialReadsOption)
if err != nil {
return objstorage.ObjectMetadata{}, err
}
defer f.Close()
buf := make([]byte, 64*1024)
for {
n, readErr := f.Read(buf)
if readErr != nil && readErr != io.EOF {
w.Abort()
return objstorage.ObjectMetadata{}, readErr
}
if n > 0 {
if err := w.Write(buf[:n]); err != nil {
w.Abort()
return objstorage.ObjectMetadata{}, err
}
}
if readErr == io.EOF {
break
}
}
if err := w.Finish(); err != nil {
return objstorage.ObjectMetadata{}, err
}
return meta, nil
}
// Lookup is part of the objstorage.Provider interface.
func (p *provider) Lookup(
fileType base.FileType, fileNum base.DiskFileNum,
) (objstorage.ObjectMetadata, error) {
p.mu.RLock()
defer p.mu.RUnlock()
meta, ok := p.mu.knownObjects[fileNum]
if !ok {
return objstorage.ObjectMetadata{}, errors.Wrapf(
os.ErrNotExist,
"file %s (type %d) unknown to the objstorage provider",
errors.Safe(fileNum), errors.Safe(fileType),
)
}
if meta.FileType != fileType {
return objstorage.ObjectMetadata{}, errors.AssertionFailedf(
"file %s type mismatch (known type %d, expected type %d)",
errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType),
)
}
return meta, nil
}
// Path is part of the objstorage.Provider interface.
func (p *provider) Path(meta objstorage.ObjectMetadata) string {
if !meta.IsShared() {
return p.vfsPath(meta.FileType, meta.DiskFileNum)
}
return p.sharedPath(meta)
}
// Size returns the size of the object.
func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
if !meta.IsShared() {
return p.vfsSize(meta.FileType, meta.DiskFileNum)
}
return p.sharedSize(meta)
}
// List is part of the objstorage.Provider interface.
func (p *provider) List() []objstorage.ObjectMetadata {
p.mu.RLock()
defer p.mu.RUnlock()
res := make([]objstorage.ObjectMetadata, 0, len(p.mu.knownObjects))
for _, meta := range p.mu.knownObjects {
res = append(res, meta)
}
sort.Slice(res, func(i, j int) bool {
return res[i].DiskFileNum.FileNum() < res[j].DiskFileNum.FileNum()
})
return res
}
func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
p.mu.Lock()
defer p.mu.Unlock()
p.mu.knownObjects[meta.DiskFileNum] = meta
if meta.IsShared() {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
FileNum: meta.DiskFileNum,
FileType: meta.FileType,
CreatorID: meta.Shared.CreatorID,
CreatorFileNum: meta.Shared.CreatorFileNum,
CleanupMethod: meta.Shared.CleanupMethod,
})
} else {
p.mu.localObjectsChanged = true
}
}
func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
p.mu.Lock()
defer p.mu.Unlock()
meta, ok := p.mu.knownObjects[fileNum]
if !ok {
return
}
delete(p.mu.knownObjects, fileNum)
if meta.IsShared() {
p.mu.shared.catalogBatch.DeleteObject(fileNum)
} else {
p.mu.localObjectsChanged = true
}
}
// protectObject prevents the unreferencing of a shared object until
// unprotectObject is called.
func (p *provider) protectObject(fileNum base.DiskFileNum) {
p.mu.Lock()
defer p.mu.Unlock()
p.mu.protectedObjects[fileNum] = p.mu.protectedObjects[fileNum] + 1
}
func (p *provider) unprotectObject(fileNum base.DiskFileNum) {
p.mu.Lock()
defer p.mu.Unlock()
v := p.mu.protectedObjects[fileNum]
if invariants.Enabled && v == 0 {
panic("invalid protection count")
}
if v > 1 {
p.mu.protectedObjects[fileNum] = v - 1
} else {
delete(p.mu.protectedObjects, fileNum)
// TODO(radu): check if the object is still in knownObject; if not, unref it
// now.
}
}
func (p *provider) isProtected(fileNum base.DiskFileNum) bool {
p.mu.Lock()
defer p.mu.Unlock()
return p.mu.protectedObjects[fileNum] > 0
}

200
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go generated vendored Normal file
View File

@@ -0,0 +1,200 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
const (
// Constants for dynamic readahead of data blocks. Note that the size values
// make sense as some multiple of the default block size; and they should
// both be larger than the default block size.
minFileReadsForReadahead = 2
// TODO(bilal): Have the initial size value be a factor of the block size,
// as opposed to a hardcoded value.
initialReadaheadSize = 64 << 10 /* 64KB */
maxReadaheadSize = 256 << 10 /* 256KB */
)
// readaheadState contains state variables related to readahead. Updated on
// file reads.
type readaheadState struct {
// Number of sequential reads.
numReads int64
// Size issued to the next call to Prefetch. Starts at or above
// initialReadaheadSize and grows exponentially until maxReadaheadSize.
size int64
// prevSize is the size used in the last Prefetch call.
prevSize int64
// The byte offset up to which the OS has been asked to read ahead / cached.
// When reading ahead, reads up to this limit should not incur an IO
// operation. Reads after this limit can benefit from a new call to
// Prefetch.
limit int64
}
func makeReadaheadState() readaheadState {
return readaheadState{
size: initialReadaheadSize,
}
}
func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
currentReadEnd := offset + blockLength
if rs.numReads >= minFileReadsForReadahead {
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
// This is a read that would have resulted in a readahead, had it
// not been a cache hit.
rs.limit = currentReadEnd
return
}
if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
// We read too far away from rs.limit to benefit from readahead in
// any scenario. Reset all variables.
rs.numReads = 1
rs.limit = currentReadEnd
rs.size = initialReadaheadSize
rs.prevSize = 0
return
}
// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
// here. This is a read that is potentially benefitting from a past
// readahead.
return
}
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
// Blocks are being read sequentially and would benefit from readahead
// down the line.
rs.numReads++
return
}
// We read too far ahead of the last read, or before it. This indicates
// a random read, where readahead is not desirable. Reset all variables.
rs.numReads = 1
rs.limit = currentReadEnd
rs.size = initialReadaheadSize
rs.prevSize = 0
}
// maybeReadahead updates state and determines whether to issue a readahead /
// prefetch call for a block read at offset for blockLength bytes.
// Returns a size value (greater than 0) that should be prefetched if readahead
// would be beneficial.
func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
currentReadEnd := offset + blockLength
if rs.numReads >= minFileReadsForReadahead {
// The minimum threshold of sequential reads to justify reading ahead
// has been reached.
// There are two intervals: the interval being read:
// [offset, currentReadEnd]
// as well as the interval where a read would benefit from read ahead:
// [rs.limit, rs.limit + rs.size]
// We increase the latter interval to
// [rs.limit, rs.limit + maxReadaheadSize] to account for cases where
// readahead may not be beneficial with a small readahead size, but over
// time the readahead size would increase exponentially to make it
// beneficial.
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
// We are doing a read in the interval ahead of
// the last readahead range. In the diagrams below, ++++ is the last
// readahead range, ==== is the range represented by
// [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range
// being read.
//
// rs.limit rs.limit + maxReadaheadSize
// ++++++++++|===========================|
//
// |-------------|
// offset currentReadEnd
//
// This case is also possible, as are all cases with an overlap
// between [rs.limit, rs.limit + maxReadaheadSize] and [offset,
// currentReadEnd]:
//
// rs.limit rs.limit + maxReadaheadSize
// ++++++++++|===========================|
//
// |-------------|
// offset currentReadEnd
//
//
rs.numReads++
rs.limit = offset + rs.size
rs.prevSize = rs.size
// Increase rs.size for the next read.
rs.size *= 2
if rs.size > maxReadaheadSize {
rs.size = maxReadaheadSize
}
return rs.prevSize
}
if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
// The above conditional has rs.limit > rs.prevSize to confirm that
// rs.limit - rs.prevSize would not underflow.
// We read too far away from rs.limit to benefit from readahead in
// any scenario. Reset all variables.
// The case where we read too far ahead:
//
// (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize)
// |+++++++++++++|=============|
//
// |-------------|
// offset currentReadEnd
//
// Or too far behind:
//
// (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize)
// |+++++++++++++|=============|
//
// |-------------|
// offset currentReadEnd
//
rs.numReads = 1
rs.limit = currentReadEnd
rs.size = initialReadaheadSize
rs.prevSize = 0
return 0
}
// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
// here. This is a read that is potentially benefitting from a past
// readahead, but there's no reason to issue a readahead call at the
// moment.
//
// (rs.limit - rs.prevSize) (rs.limit + maxReadaheadSize)
// |+++++++++++++|===============|
// (rs.limit)
//
// |-------|
// offset currentReadEnd
//
rs.numReads++
return 0
}
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
// Blocks are being read sequentially and would benefit from readahead
// down the line.
//
// (rs.limit) (rs.limit + maxReadaheadSize)
// |=============|
//
// |-------|
// offset currentReadEnd
//
rs.numReads++
return 0
}
// We read too far ahead of the last read, or before it. This indicates
// a random read, where readahead is not desirable. Reset all variables.
//
// (rs.limit - maxReadaheadSize) (rs.limit) (rs.limit + maxReadaheadSize)
// |+++++++++++++|=============|
//
// |-------|
// offset currentReadEnd
//
rs.numReads = 1
rs.limit = currentReadEnd
rs.size = initialReadaheadSize
rs.prevSize = 0
return 0
}

255
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go generated vendored Normal file
View File

@@ -0,0 +1,255 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"sync"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/shared"
)
// sharedSubsystem contains the provider fields related to shared storage.
// All fields remain unset if shared storage is not configured.
type sharedSubsystem struct {
catalog *sharedobjcat.Catalog
// checkRefsOnOpen controls whether we check the ref marker file when opening
// an object. Normally this is true when invariants are enabled (but the provider
// test tweaks this field).
checkRefsOnOpen bool
// initialized guards access to the creatorID field.
initialized atomic.Bool
creatorID objstorage.CreatorID
initOnce sync.Once
}
func (ss *sharedSubsystem) init(creatorID objstorage.CreatorID) {
ss.initOnce.Do(func() {
ss.creatorID = creatorID
ss.initialized.Store(true)
})
}
func (p *provider) sharedStorage() shared.Storage {
return p.st.Shared.Storage
}
// sharedInit initializes the shared object subsystem (if configured) and finds
// any shared objects.
func (p *provider) sharedInit() error {
if p.sharedStorage() == nil {
return nil
}
catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName)
if err != nil {
return errors.Wrapf(err, "pebble: could not open shared object catalog")
}
p.shared.catalog = catalog
p.shared.checkRefsOnOpen = invariants.Enabled
// The creator ID may or may not be initialized yet.
if contents.CreatorID.IsSet() {
p.shared.init(contents.CreatorID)
p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID)
} else {
p.st.Logger.Infof("shared storage configured; no creatorID yet")
}
for _, meta := range contents.Objects {
o := objstorage.ObjectMetadata{
DiskFileNum: meta.FileNum,
FileType: meta.FileType,
}
o.Shared.CreatorID = meta.CreatorID
o.Shared.CreatorFileNum = meta.CreatorFileNum
o.Shared.CleanupMethod = meta.CleanupMethod
p.mu.knownObjects[o.DiskFileNum] = o
}
return nil
}
// SetCreatorID is part of the objstorage.Provider interface.
func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
if p.sharedStorage() == nil {
return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled")
}
// Note: this call is a cheap no-op if the creator ID was already set. This
// call also checks if we are trying to change the ID.
if err := p.shared.catalog.SetCreatorID(creatorID); err != nil {
return err
}
if !p.shared.initialized.Load() {
p.st.Logger.Infof("shared storage creatorID set to %s", creatorID)
p.shared.init(creatorID)
}
return nil
}
func (p *provider) sharedCheckInitialized() error {
if p.sharedStorage() == nil {
return errors.Errorf("shared object support not configured")
}
if !p.shared.initialized.Load() {
return errors.Errorf("shared object support not available: shared creator ID not yet set")
}
return nil
}
func (p *provider) sharedSync() error {
batch := func() sharedobjcat.Batch {
p.mu.Lock()
defer p.mu.Unlock()
res := p.mu.shared.catalogBatch.Copy()
p.mu.shared.catalogBatch.Reset()
return res
}()
if batch.IsEmpty() {
return nil
}
err := p.shared.catalog.ApplyBatch(batch)
if err != nil {
// We have to put back the batch (for the next Sync).
p.mu.Lock()
defer p.mu.Unlock()
batch.Append(p.mu.shared.catalogBatch)
p.mu.shared.catalogBatch = batch
return err
}
return nil
}
func (p *provider) sharedPath(meta objstorage.ObjectMetadata) string {
return "shared://" + sharedObjectName(meta)
}
// sharedCreateRef creates a reference marker object.
func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
return nil
}
refName := p.sharedObjectRefName(meta)
writer, err := p.sharedStorage().CreateObject(refName)
if err == nil {
// The object is empty, just close the writer.
err = writer.Close()
}
if err != nil {
return errors.Wrapf(err, "creating marker object %q", refName)
}
return nil
}
func (p *provider) sharedCreate(
_ context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (objstorage.Writable, objstorage.ObjectMetadata, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, objstorage.ObjectMetadata{}, err
}
meta := objstorage.ObjectMetadata{
DiskFileNum: fileNum,
FileType: fileType,
}
meta.Shared.CreatorID = p.shared.creatorID
meta.Shared.CreatorFileNum = fileNum
meta.Shared.CleanupMethod = opts.SharedCleanupMethod
objName := sharedObjectName(meta)
writer, err := p.sharedStorage().CreateObject(objName)
if err != nil {
return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
}
return &sharedWritable{
p: p,
meta: meta,
storageWriter: writer,
}, meta, nil
}
func (p *provider) sharedOpenForReading(
ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, err
}
// Verify we have a reference on this object; for performance reasons, we only
// do this in testing scenarios.
if p.shared.checkRefsOnOpen && meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
refName := p.sharedObjectRefName(meta)
if _, err := p.sharedStorage().Size(refName); err != nil {
if p.sharedStorage().IsNotExistError(err) {
if opts.MustExist {
p.st.Logger.Fatalf("marker object %q does not exist", refName)
// TODO(radu): maybe list references for the object.
}
return nil, errors.Errorf("marker object %q does not exist", refName)
}
return nil, errors.Wrapf(err, "checking marker object %q", refName)
}
}
objName := sharedObjectName(meta)
reader, size, err := p.sharedStorage().ReadObject(ctx, objName)
if err != nil {
if opts.MustExist && p.sharedStorage().IsNotExistError(err) {
p.st.Logger.Fatalf("object %q does not exist", objName)
// TODO(radu): maybe list references for the object.
}
return nil, err
}
return newSharedReadable(reader, size), nil
}
func (p *provider) sharedSize(meta objstorage.ObjectMetadata) (int64, error) {
if err := p.sharedCheckInitialized(); err != nil {
return 0, err
}
objName := sharedObjectName(meta)
return p.sharedStorage().Size(objName)
}
// sharedUnref implements object "removal" with the shared backend. The ref
// marker object is removed and the backing object is removed only if there are
// no other ref markers.
func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
if meta.Shared.CleanupMethod == objstorage.SharedNoCleanup {
// Never delete objects in this mode.
return nil
}
if p.isProtected(meta.DiskFileNum) {
// TODO(radu): we need a mechanism to unref the object when it becomes
// unprotected.
return nil
}
refName := p.sharedObjectRefName(meta)
// Tolerate a not-exists error.
if err := p.sharedStorage().Delete(refName); err != nil && !p.sharedStorage().IsNotExistError(err) {
return err
}
otherRefs, err := p.sharedStorage().List(sharedObjectRefPrefix(meta), "" /* delimiter */)
if err != nil {
return err
}
if len(otherRefs) == 0 {
objName := sharedObjectName(meta)
if err := p.sharedStorage().Delete(objName); err != nil && !p.sharedStorage().IsNotExistError(err) {
return err
}
}
return nil
}

240
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_backing.go generated vendored Normal file
View File

@@ -0,0 +1,240 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"bytes"
"encoding/binary"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
)
const (
tagCreatorID = 1
tagCreatorFileNum = 2
tagCleanupMethod = 3
// tagRefCheckID encodes the information for a ref marker that needs to be
// checked when attaching this object to another provider. This is set to the
// creator ID and FileNum for the provider that encodes the backing, and
// allows the "target" provider to check that the "source" provider kept its
// reference on the object alive.
tagRefCheckID = 4
// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
// followed by the length of the data (so they can be skipped).
// Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if
// they are encountered by earlier code that doesn't know the tag.
tagNotSafeToIgnoreMask = 64
)
func (p *provider) encodeSharedObjectBacking(
meta *objstorage.ObjectMetadata,
) (objstorage.SharedObjectBacking, error) {
if !meta.IsShared() {
return nil, errors.AssertionFailedf("object %s not on shared storage", meta.DiskFileNum)
}
buf := make([]byte, 0, binary.MaxVarintLen64*4)
buf = binary.AppendUvarint(buf, tagCreatorID)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID))
// TODO(radu): encode file type as well?
buf = binary.AppendUvarint(buf, tagCreatorFileNum)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, tagCleanupMethod)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CleanupMethod))
if meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
buf = binary.AppendUvarint(buf, tagRefCheckID)
buf = binary.AppendUvarint(buf, uint64(p.shared.creatorID))
buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum()))
}
return buf, nil
}
type sharedObjectBackingHandle struct {
backing objstorage.SharedObjectBacking
fileNum base.DiskFileNum
p *provider
}
func (s *sharedObjectBackingHandle) Get() (objstorage.SharedObjectBacking, error) {
if s.backing == nil {
return nil, errors.Errorf("SharedObjectBackingHandle.Get() called after Close()")
}
return s.backing, nil
}
func (s *sharedObjectBackingHandle) Close() {
if s.backing != nil {
s.backing = nil
s.p.unprotectObject(s.fileNum)
}
}
var _ objstorage.SharedObjectBackingHandle = (*sharedObjectBackingHandle)(nil)
// SharedObjectBacking is part of the objstorage.Provider interface.
func (p *provider) SharedObjectBacking(
meta *objstorage.ObjectMetadata,
) (objstorage.SharedObjectBackingHandle, error) {
backing, err := p.encodeSharedObjectBacking(meta)
if err != nil {
return nil, err
}
p.protectObject(meta.DiskFileNum)
return &sharedObjectBackingHandle{
backing: backing,
fileNum: meta.DiskFileNum,
p: p,
}, nil
}
type decodedBacking struct {
meta objstorage.ObjectMetadata
// refToCheck is set only when meta.Shared.CleanupMethod is RefTracking
refToCheck struct {
creatorID objstorage.CreatorID
fileNum base.DiskFileNum
}
}
// decodeSharedObjectBacking decodes the shared object metadata.
// Returns the object metadata and (optionally) the creator ID of the provider
// that encoded the backing whose ref marker needs to be checked.
func decodeSharedObjectBacking(
fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.SharedObjectBacking,
) (decodedBacking, error) {
var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64
br := bytes.NewReader(buf)
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return decodedBacking{}, err
}
switch tag {
case tagCreatorID:
creatorID, err = binary.ReadUvarint(br)
case tagCreatorFileNum:
creatorFileNum, err = binary.ReadUvarint(br)
case tagCleanupMethod:
cleanupMethod, err = binary.ReadUvarint(br)
case tagRefCheckID:
refCheckCreatorID, err = binary.ReadUvarint(br)
if err == nil {
refCheckFileNum, err = binary.ReadUvarint(br)
}
default:
// Ignore unknown tags, unless they're not safe to ignore.
if tag&tagNotSafeToIgnoreMask != 0 {
return decodedBacking{}, errors.Newf("unknown tag %d", tag)
}
var dataLen uint64
dataLen, err = binary.ReadUvarint(br)
if err == nil {
_, err = br.Seek(int64(dataLen), io.SeekCurrent)
}
}
if err != nil {
return decodedBacking{}, err
}
}
if creatorID == 0 {
return decodedBacking{}, errors.Newf("shared object backing missing creator ID")
}
if creatorFileNum == 0 {
return decodedBacking{}, errors.Newf("shared object backing missing creator file num")
}
var res decodedBacking
res.meta.DiskFileNum = fileNum
res.meta.FileType = fileType
res.meta.Shared.CreatorID = objstorage.CreatorID(creatorID)
res.meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
res.meta.Shared.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
if res.meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
if refCheckCreatorID == 0 || refCheckFileNum == 0 {
return decodedBacking{}, errors.Newf("shared object backing missing ref to check")
}
res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID)
res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum()
}
return res, nil
}
// AttachSharedObjects is part of the objstorage.Provider interface.
func (p *provider) AttachSharedObjects(
objs []objstorage.SharedObjectToAttach,
) ([]objstorage.ObjectMetadata, error) {
decoded := make([]decodedBacking, len(objs))
for i, o := range objs {
var err error
decoded[i], err = decodeSharedObjectBacking(o.FileType, o.FileNum, o.Backing)
if err != nil {
return nil, err
}
}
// Create the reference marker objects.
// TODO(radu): parallelize this.
for _, d := range decoded {
if d.meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
continue
}
if err := p.sharedCreateRef(d.meta); err != nil {
// TODO(radu): clean up references previously created in this loop.
return nil, err
}
// Check the "origin"'s reference.
refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum)
if _, err := p.sharedStorage().Size(refName); err != nil {
_ = p.sharedUnref(d.meta)
// TODO(radu): clean up references previously created in this loop.
if p.sharedStorage().IsNotExistError(err) {
return nil, errors.Errorf("origin marker object %q does not exist;"+
" object probably removed from the provider which created the backing", refName)
}
return nil, errors.Wrapf(err, "checking origin's marker object %s", refName)
}
}
func() {
p.mu.Lock()
defer p.mu.Unlock()
for _, d := range decoded {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
FileNum: d.meta.DiskFileNum,
FileType: d.meta.FileType,
CreatorID: d.meta.Shared.CreatorID,
CreatorFileNum: d.meta.Shared.CreatorFileNum,
})
}
}()
if err := p.sharedSync(); err != nil {
return nil, err
}
metas := make([]objstorage.ObjectMetadata, len(decoded))
for i, d := range decoded {
metas[i] = d.meta
}
p.mu.Lock()
defer p.mu.Unlock()
for _, meta := range metas {
p.mu.knownObjects[meta.DiskFileNum] = meta
}
return metas, nil
}

80
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_obj_name.go generated vendored Normal file
View File

@@ -0,0 +1,80 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
)
// sharedObjectName returns the name of an object on shared storage.
//
// For sstables, the format is: <hash>-<creator-id>-<file-num>.sst
// For example: 1a3f-2-000001.sst
func sharedObjectName(meta objstorage.ObjectMetadata) string {
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst",
objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
)
}
panic("unknown FileType")
}
// sharedObjectRefName returns the name of the object's ref marker associated
// with a given referencing provider. This name is the object's name concatenated with
// ".ref.<ref-creator-id>.<local-file-num>".
//
// For example: 1a3f-2-000001.sst.ref.5.000008
func sharedObjectRefName(
meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum,
) string {
if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
panic("ref object used when ref tracking disabled")
}
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst.ref.%d.%06d",
objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
)
}
panic("unknown FileType")
}
func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst.ref.",
objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
)
}
panic("unknown FileType")
}
// sharedObjectRefName returns the name of the object's ref marker associated
// with this provider. This name is the object's name concatenated with
// ".ref.<creator-id>.<local-file-num>".
//
// For example: 1a3f-2-000001.sst.ref.5.000008
func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
panic("ref object used when ref tracking disabled")
}
return sharedObjectRefName(meta, p.shared.creatorID, meta.DiskFileNum)
}
// objHash returns a 16-bit hash value derived from the creator ID and creator
// file num. We prepend this value to object names to ensure balanced
// partitioning with AWS (and likely other blob storage providers).
func objHash(meta objstorage.ObjectMetadata) uint16 {
const prime1 = 7459
const prime2 = 17539
return uint16(uint64(meta.Shared.CreatorID)*prime1 + uint64(meta.Shared.CreatorFileNum.FileNum())*prime2)
}

65
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go generated vendored Normal file
View File

@@ -0,0 +1,65 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/shared"
)
// sharedReadable is a very simple implementation of Readable on top of the
// ReadCloser returned by shared.Storage.CreateObject.
type sharedReadable struct {
objReader shared.ObjectReader
size int64
}
var _ objstorage.Readable = (*sharedReadable)(nil)
func newSharedReadable(objReader shared.ObjectReader, size int64) *sharedReadable {
return &sharedReadable{
objReader: objReader,
size: size,
}
}
func (r *sharedReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
return r.objReader.ReadAt(ctx, p, offset)
}
func (r *sharedReadable) Close() error {
defer func() { r.objReader = nil }()
return r.objReader.Close()
}
func (r *sharedReadable) Size() int64 {
return r.size
}
func (r *sharedReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
// TODO(radu): use a pool.
return &sharedReadHandle{readable: r}
}
type sharedReadHandle struct {
readable *sharedReadable
}
var _ objstorage.ReadHandle = (*sharedReadHandle)(nil)
func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
return r.readable.ReadAt(ctx, p, offset)
}
func (r *sharedReadHandle) Close() error {
r.readable = nil
return nil
}
func (r *sharedReadHandle) MaxReadahead() {}
func (r *sharedReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {}

54
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go generated vendored Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"io"
"github.com/cockroachdb/pebble/objstorage"
)
// sharedWritable is a very simple implementation of Writable on top of the
// WriteCloser returned by shared.Storage.CreateObject.
type sharedWritable struct {
p *provider
meta objstorage.ObjectMetadata
storageWriter io.WriteCloser
}
var _ objstorage.Writable = (*sharedWritable)(nil)
// Write is part of the Writable interface.
func (w *sharedWritable) Write(p []byte) error {
_, err := w.storageWriter.Write(p)
return err
}
// Finish is part of the Writable interface.
func (w *sharedWritable) Finish() error {
err := w.storageWriter.Close()
w.storageWriter = nil
if err != nil {
w.Abort()
return err
}
// Create the marker object.
if err := w.p.sharedCreateRef(w.meta); err != nil {
w.Abort()
return err
}
return nil
}
// Abort is part of the Writable interface.
func (w *sharedWritable) Abort() {
if w.storageWriter != nil {
_ = w.storageWriter.Close()
w.storageWriter = nil
}
w.p.removeMetadata(w.meta.DiskFileNum)
// TODO(radu): delete the object if it was created.
}

376
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/catalog.go generated vendored Normal file
View File

@@ -0,0 +1,376 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sharedobjcat
import (
"fmt"
"io"
"sort"
"sync"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
// Catalog is used to manage the on-disk shared object catalog.
//
// The catalog file is a log of records, where each record is an encoded
// versionEdit.
type Catalog struct {
fs vfs.FS
dirname string
mu struct {
sync.Mutex
creatorID objstorage.CreatorID
objects map[base.DiskFileNum]SharedObjectMetadata
marker *atomicfs.Marker
catalogFile vfs.File
catalogRecWriter *record.Writer
rotationHelper record.RotationHelper
// catalogFilename is the filename of catalogFile when catalogFile != nil, otherwise
// it is the filename of the last catalog file.
catalogFilename string
}
}
// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
type SharedObjectMetadata struct {
// FileNum is the identifier for the object within the context of a single DB
// instance.
FileNum base.DiskFileNum
// FileType is the type of the object. Only certain FileTypes are possible.
FileType base.FileType
// CreatorID identifies the DB instance that originally created the object.
CreatorID objstorage.CreatorID
// CreatorFileNum is the identifier for the object within the context of the
// DB instance that originally created the object.
CreatorFileNum base.DiskFileNum
CleanupMethod objstorage.SharedCleanupMethod
}
const (
catalogFilenameBase = "SHARED-CATALOG"
catalogMarkerName = "shared-catalog"
// We create a new file when the size exceeds 1MB (and some other conditions
// hold; see record.RotationHelper).
rotateFileSize = 1024 * 1024 // 1MB
)
// CatalogContents contains the shared objects in the catalog.
type CatalogContents struct {
// CreatorID, if it is set.
CreatorID objstorage.CreatorID
Objects []SharedObjectMetadata
}
// Open creates a Catalog and loads any existing catalog file, returning the
// creator ID (if it is set) and the contents.
func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
c := &Catalog{
fs: fs,
dirname: dirname,
}
c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata)
var err error
c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
if err != nil {
return nil, CatalogContents{}, err
}
// If the filename is empty, there is no existing catalog.
if c.mu.catalogFilename != "" {
if err := c.loadFromCatalogFile(c.mu.catalogFilename); err != nil {
return nil, CatalogContents{}, err
}
if err := c.mu.marker.RemoveObsolete(); err != nil {
return nil, CatalogContents{}, err
}
// TODO(radu): remove obsolete catalog files.
}
res := CatalogContents{
CreatorID: c.mu.creatorID,
Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)),
}
for _, meta := range c.mu.objects {
res.Objects = append(res.Objects, meta)
}
// Sort the objects so the function is deterministic.
sort.Slice(res.Objects, func(i, j int) bool {
return res.Objects[i].FileNum.FileNum() < res.Objects[j].FileNum.FileNum()
})
return c, res, nil
}
// SetCreatorID sets the creator ID. If it is already set, it must match.
func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {
if !id.IsSet() {
return errors.AssertionFailedf("attempt to unset CreatorID")
}
c.mu.Lock()
defer c.mu.Unlock()
if c.mu.creatorID.IsSet() {
if c.mu.creatorID != id {
return errors.AssertionFailedf("attempt to change CreatorID from %s to %s", c.mu.creatorID, id)
}
return nil
}
ve := versionEdit{CreatorID: id}
if err := c.writeToCatalogFileLocked(&ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
}
c.mu.creatorID = id
return nil
}
// Close any open files.
func (c *Catalog) Close() error {
return c.closeCatalogFile()
}
func (c *Catalog) closeCatalogFile() error {
if c.mu.catalogFile == nil {
return nil
}
err1 := c.mu.catalogRecWriter.Close()
err2 := c.mu.catalogFile.Close()
c.mu.catalogRecWriter = nil
c.mu.catalogFile = nil
if err1 != nil {
return err1
}
return err2
}
// Batch is used to perform multiple object additions/deletions at once.
type Batch struct {
ve versionEdit
}
// AddObject adds a new object to the batch.
//
// The given FileNum must be new - it must not match that of any object that was
// ever in the catalog.
func (b *Batch) AddObject(meta SharedObjectMetadata) {
b.ve.NewObjects = append(b.ve.NewObjects, meta)
}
// DeleteObject adds an object removal to the batch.
func (b *Batch) DeleteObject(fileNum base.DiskFileNum) {
b.ve.DeletedObjects = append(b.ve.DeletedObjects, fileNum)
}
// Reset clears the batch.
func (b *Batch) Reset() {
b.ve.NewObjects = b.ve.NewObjects[:0]
b.ve.DeletedObjects = b.ve.DeletedObjects[:0]
}
// IsEmpty returns true if the batch is empty.
func (b *Batch) IsEmpty() bool {
return len(b.ve.NewObjects) == 0 && len(b.ve.DeletedObjects) == 0
}
// Copy returns a copy of the Batch.
func (b *Batch) Copy() Batch {
var res Batch
if len(b.ve.NewObjects) > 0 {
res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
copy(res.ve.NewObjects, b.ve.NewObjects)
}
if len(b.ve.DeletedObjects) > 0 {
res.ve.DeletedObjects = make([]base.DiskFileNum, len(b.ve.DeletedObjects))
copy(res.ve.DeletedObjects, b.ve.DeletedObjects)
}
return res
}
// Append merges two batches.
func (b *Batch) Append(other Batch) {
b.ve.NewObjects = append(b.ve.NewObjects, other.ve.NewObjects...)
b.ve.DeletedObjects = append(b.ve.DeletedObjects, other.ve.DeletedObjects...)
}
// ApplyBatch applies a batch of updates; returns after the change is stably
// recorded on storage.
func (c *Catalog) ApplyBatch(b Batch) error {
c.mu.Lock()
defer c.mu.Unlock()
for _, n := range b.ve.DeletedObjects {
if _, exists := c.mu.objects[n]; !exists {
return errors.AssertionFailedf("deleting non-existent object %s", n)
}
}
for _, meta := range b.ve.NewObjects {
if _, exists := c.mu.objects[meta.FileNum]; exists {
return errors.AssertionFailedf("adding existing object %s", meta.FileNum)
}
}
if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
}
// Apply the batch to our current state.
for _, n := range b.ve.DeletedObjects {
delete(c.mu.objects, n)
}
for _, meta := range b.ve.NewObjects {
c.mu.objects[meta.FileNum] = meta
}
b.Reset()
return nil
}
func (c *Catalog) loadFromCatalogFile(filename string) error {
catalogPath := c.fs.PathJoin(c.dirname, filename)
f, err := c.fs.Open(catalogPath)
if err != nil {
return errors.Wrapf(
err, "pebble: could not open shared object catalog file %q for DB %q",
errors.Safe(filename), c.dirname,
)
}
defer f.Close()
rr := record.NewReader(f, 0 /* logNum */)
for {
r, err := rr.Next()
if err == io.EOF || record.IsInvalidRecord(err) {
break
}
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
errors.Safe(filename))
}
var ve versionEdit
err = ve.Decode(r)
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
errors.Safe(filename))
}
// Apply the version edit to the current state.
if ve.CreatorID.IsSet() {
c.mu.creatorID = ve.CreatorID
}
for _, fileNum := range ve.DeletedObjects {
delete(c.mu.objects, fileNum)
}
for _, meta := range ve.NewObjects {
c.mu.objects[meta.FileNum] = meta
}
}
return nil
}
// writeToCatalogFileLocked writes a versionEdit to the catalog file.
// Creates a new file if this is the first write.
func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error {
c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects)))
snapshotSize := int64(len(c.mu.objects))
var shouldRotate bool
if c.mu.catalogFile == nil {
shouldRotate = true
} else if c.mu.catalogRecWriter.Size() >= rotateFileSize {
shouldRotate = c.mu.rotationHelper.ShouldRotate(snapshotSize)
}
if shouldRotate {
if c.mu.catalogFile != nil {
if err := c.closeCatalogFile(); err != nil {
return err
}
}
if err := c.createNewCatalogFileLocked(); err != nil {
return err
}
c.mu.rotationHelper.Rotate(snapshotSize)
}
return writeRecord(ve, c.mu.catalogFile, c.mu.catalogRecWriter)
}
func makeCatalogFilename(iter uint64) string {
return fmt.Sprintf("%s-%06d", catalogFilenameBase, iter)
}
// createNewCatalogFileLocked creates a new catalog file, populates it with the
// current catalog and sets c.mu.catalogFile and c.mu.catalogRecWriter.
func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
if c.mu.catalogFile != nil {
return errors.AssertionFailedf("catalogFile already open")
}
filename := makeCatalogFilename(c.mu.marker.NextIter())
filepath := c.fs.PathJoin(c.dirname, filename)
file, err := c.fs.Create(filepath)
if err != nil {
return err
}
recWriter := record.NewWriter(file)
err = func() error {
// Create a versionEdit that gets us from an empty catalog to the current state.
var ve versionEdit
ve.CreatorID = c.mu.creatorID
ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
for _, meta := range c.mu.objects {
ve.NewObjects = append(ve.NewObjects, meta)
}
if err := writeRecord(&ve, file, recWriter); err != nil {
return err
}
// Move the marker to the new filename. Move handles syncing the data
// directory as well.
if err := c.mu.marker.Move(filename); err != nil {
return errors.Wrap(err, "moving marker")
}
return nil
}()
if err != nil {
_ = recWriter.Close()
_ = file.Close()
_ = c.fs.Remove(filepath)
return err
}
// Remove any previous file (ignoring any error).
if c.mu.catalogFilename != "" {
_ = c.fs.Remove(c.fs.PathJoin(c.dirname, c.mu.catalogFilename))
}
c.mu.catalogFile = file
c.mu.catalogRecWriter = recWriter
c.mu.catalogFilename = filename
return nil
}
func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error {
w, err := recWriter.Next()
if err != nil {
return err
}
if err := ve.Encode(w); err != nil {
return err
}
if err := recWriter.Flush(); err != nil {
return err
}
return file.Sync()
}

165
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/version_edit.go generated vendored Normal file
View File

@@ -0,0 +1,165 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sharedobjcat
import (
"bufio"
"encoding/binary"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
)
// versionEdit is a modification to the shared object state which can be encoded
// into a record.
//
// TODO(radu): consider adding creation and deletion time for debugging purposes.
type versionEdit struct {
NewObjects []SharedObjectMetadata
DeletedObjects []base.DiskFileNum
CreatorID objstorage.CreatorID
}
const (
// tagNewObject is followed by the FileNum, creator ID, creator FileNum, and
// cleanup method.
tagNewObject = 1
// tagDeletedObject is followed by the FileNum.
tagDeletedObject = 2
// tagCreatorID is followed by the Creator ID for this store. This ID can
// never change.
tagCreatorID = 3
)
// Object type values. We don't want to encode FileType directly because it is
// more general (and we want freedom to change it in the future).
const (
objTypeTable = 1
)
func objTypeToFileType(objType uint64) (base.FileType, error) {
switch objType {
case objTypeTable:
return base.FileTypeTable, nil
default:
return 0, errors.Newf("unknown object type %d", objType)
}
}
func fileTypeToObjType(fileType base.FileType) (uint64, error) {
switch fileType {
case base.FileTypeTable:
return objTypeTable, nil
default:
return 0, errors.Newf("unknown object type for file type %d", fileType)
}
}
// Encode encodes an edit to the specified writer.
func (v *versionEdit) Encode(w io.Writer) error {
buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2))
for _, meta := range v.NewObjects {
objType, err := fileTypeToObjType(meta.FileType)
if err != nil {
return err
}
buf = binary.AppendUvarint(buf, uint64(tagNewObject))
buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum()))
buf = binary.AppendUvarint(buf, objType)
buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
}
for _, dfn := range v.DeletedObjects {
buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
buf = binary.AppendUvarint(buf, uint64(dfn.FileNum()))
}
if v.CreatorID.IsSet() {
buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
}
_, err := w.Write(buf)
return err
}
// Decode decodes an edit from the specified reader.
func (v *versionEdit) Decode(r io.Reader) error {
br, ok := r.(io.ByteReader)
if !ok {
br = bufio.NewReader(r)
}
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return err
}
err = nil
switch tag {
case tagNewObject:
var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
var fileType base.FileType
fileNum, err = binary.ReadUvarint(br)
if err == nil {
var objType uint64
objType, err = binary.ReadUvarint(br)
if err == nil {
fileType, err = objTypeToFileType(objType)
}
}
if err == nil {
creatorID, err = binary.ReadUvarint(br)
}
if err == nil {
creatorFileNum, err = binary.ReadUvarint(br)
}
if err == nil {
cleanupMethod, err = binary.ReadUvarint(br)
}
if err == nil {
v.NewObjects = append(v.NewObjects, SharedObjectMetadata{
FileNum: base.FileNum(fileNum).DiskFileNum(),
FileType: fileType,
CreatorID: objstorage.CreatorID(creatorID),
CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(),
CleanupMethod: objstorage.SharedCleanupMethod(cleanupMethod),
})
}
case tagDeletedObject:
var fileNum uint64
fileNum, err = binary.ReadUvarint(br)
if err == nil {
v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum())
}
case tagCreatorID:
var id uint64
id, err = binary.ReadUvarint(br)
if err == nil {
v.CreatorID = objstorage.CreatorID(id)
}
default:
// Ignore unknown tags.
}
if err != nil {
if err == io.EOF {
return errCorruptCatalog
}
return err
}
}
return nil
}
var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog")

109
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go generated vendored Normal file
View File

@@ -0,0 +1,109 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
func (p *provider) vfsPath(fileType base.FileType, fileNum base.DiskFileNum) string {
return base.MakeFilepath(p.st.FS, p.st.FSDirName, fileType, fileNum)
}
func (p *provider) vfsOpenForReading(
ctx context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
filename := p.vfsPath(fileType, fileNum)
file, err := p.st.FS.Open(filename, vfs.RandomReadsOption)
if err != nil {
if opts.MustExist {
base.MustExist(p.st.FS, filename, p.st.Logger, err)
}
return nil, err
}
return newFileReadable(file, p.st.FS, filename)
}
func (p *provider) vfsCreate(
_ context.Context, fileType base.FileType, fileNum base.DiskFileNum,
) (objstorage.Writable, objstorage.ObjectMetadata, error) {
filename := p.vfsPath(fileType, fileNum)
file, err := p.st.FS.Create(filename)
if err != nil {
return nil, objstorage.ObjectMetadata{}, err
}
file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
NoSyncOnClose: p.st.NoSyncOnClose,
BytesPerSync: p.st.BytesPerSync,
})
meta := objstorage.ObjectMetadata{
DiskFileNum: fileNum,
FileType: fileType,
}
return newFileBufferedWritable(file), meta, nil
}
func (p *provider) vfsRemove(fileType base.FileType, fileNum base.DiskFileNum) error {
return p.st.FSCleaner.Clean(p.st.FS, fileType, p.vfsPath(fileType, fileNum))
}
// vfsInit finds any local FS objects.
func (p *provider) vfsInit() error {
listing := p.st.FSDirInitialListing
if listing == nil {
var err error
listing, err = p.st.FS.List(p.st.FSDirName)
if err != nil {
return errors.Wrapf(err, "pebble: could not list store directory")
}
}
for _, filename := range listing {
fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename)
if ok && fileType == base.FileTypeTable {
o := objstorage.ObjectMetadata{
FileType: fileType,
DiskFileNum: fileNum,
}
p.mu.knownObjects[o.DiskFileNum] = o
}
}
return nil
}
func (p *provider) vfsSync() error {
p.mu.Lock()
shouldSync := p.mu.localObjectsChanged
p.mu.localObjectsChanged = false
p.mu.Unlock()
if !shouldSync {
return nil
}
if err := p.fsDir.Sync(); err != nil {
p.mu.Lock()
defer p.mu.Unlock()
p.mu.localObjectsChanged = true
return err
}
return nil
}
func (p *provider) vfsSize(fileType base.FileType, fileNum base.DiskFileNum) (int64, error) {
filename := p.vfsPath(fileType, fileNum)
stat, err := p.st.FS.Stat(filename)
if err != nil {
return 0, err
}
return stat.Size(), nil
}

210
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go generated vendored Normal file
View File

@@ -0,0 +1,210 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"fmt"
"os"
"sync"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// fileReadable implements objstorage.Readable on top of a vfs.File.
//
// The implementation might use Prealloc and might reopen the file with
// SequentialReadsOption.
type fileReadable struct {
file vfs.File
size int64
// The following fields are used to possibly open the file again using the
// sequential reads option (see vfsReadHandle).
filename string
fs vfs.FS
}
var _ objstorage.Readable = (*fileReadable)(nil)
func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, error) {
info, err := file.Stat()
if err != nil {
return nil, err
}
r := &fileReadable{
file: file,
size: info.Size(),
filename: filename,
fs: fs,
}
invariants.SetFinalizer(r, func(obj interface{}) {
if obj.(*fileReadable).file != nil {
fmt.Fprintf(os.Stderr, "Readable was not closed")
os.Exit(1)
}
})
return r, nil
}
// ReadAt is part of the objstorage.Readable interface.
func (r *fileReadable) ReadAt(_ context.Context, p []byte, off int64) error {
n, err := r.file.ReadAt(p, off)
if invariants.Enabled && err == nil && n != len(p) {
panic("short read")
}
return err
}
// Close is part of the objstorage.Readable interface.
func (r *fileReadable) Close() error {
defer func() { r.file = nil }()
return r.file.Close()
}
// Size is part of the objstorage.Readable interface.
func (r *fileReadable) Size() int64 {
return r.size
}
// NewReadHandle is part of the objstorage.Readable interface.
func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
rh := readHandlePool.Get().(*vfsReadHandle)
rh.r = r
rh.rs = makeReadaheadState()
return rh
}
type vfsReadHandle struct {
r *fileReadable
rs readaheadState
// sequentialFile holds a file descriptor to the same underlying File,
// except with fadvise(FADV_SEQUENTIAL) called on it to take advantage of
// OS-level readahead. Once this is non-nil, the other variables in
// readaheadState don't matter much as we defer to OS-level readahead.
sequentialFile vfs.File
}
var _ objstorage.ReadHandle = (*vfsReadHandle)(nil)
var readHandlePool = sync.Pool{
New: func() interface{} {
i := &vfsReadHandle{}
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(i, func(obj interface{}) {
if obj.(*vfsReadHandle).r != nil {
fmt.Fprintf(os.Stderr, "ReadHandle was not closed")
os.Exit(1)
}
})
return i
},
}
// Close is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) Close() error {
var err error
if rh.sequentialFile != nil {
err = rh.sequentialFile.Close()
}
*rh = vfsReadHandle{}
readHandlePool.Put(rh)
return err
}
// ReadAt is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error {
var n int
var err error
if rh.sequentialFile != nil {
// Use OS-level read-ahead.
n, err = rh.sequentialFile.ReadAt(p, offset)
} else {
if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
if readaheadSize >= maxReadaheadSize {
// We've reached the maximum readahead size. Beyond this point, rely on
// OS-level readahead.
rh.MaxReadahead()
} else {
_ = rh.r.file.Prefetch(offset, readaheadSize)
}
}
n, err = rh.r.file.ReadAt(p, offset)
}
if invariants.Enabled && err == nil && n != len(p) {
panic("short read")
}
return err
}
// MaxReadahead is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) MaxReadahead() {
if rh.sequentialFile != nil {
return
}
// TODO(radu): we could share the reopened file descriptor across multiple
// handles.
f, err := rh.r.fs.Open(rh.r.filename, vfs.SequentialReadsOption)
if err == nil {
rh.sequentialFile = f
}
}
// RecordCacheHit is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
if rh.sequentialFile != nil {
// Using OS-level readahead, so do nothing.
return
}
rh.rs.recordCacheHit(offset, size)
}
// TestingCheckMaxReadahead returns true if the ReadHandle has switched to
// OS-level read-ahead.
func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool {
switch rh := rh.(type) {
case *vfsReadHandle:
return rh.sequentialFile != nil
case *PreallocatedReadHandle:
return rh.sequentialFile != nil
default:
panic("unknown ReadHandle type")
}
}
// PreallocatedReadHandle is used to avoid an allocation in NewReadHandle; see
// UsePreallocatedReadHandle.
type PreallocatedReadHandle struct {
vfsReadHandle
}
// Close is part of the objstorage.ReadHandle interface.
func (rh *PreallocatedReadHandle) Close() error {
var err error
if rh.sequentialFile != nil {
err = rh.sequentialFile.Close()
}
rh.vfsReadHandle = vfsReadHandle{}
return err
}
// UsePreallocatedReadHandle is equivalent to calling readable.NewReadHandle()
// but uses the existing storage of a PreallocatedReadHandle when possible
// (currently this happens if we are reading from a local file).
// The returned handle still needs to be closed.
func UsePreallocatedReadHandle(
ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
) objstorage.ReadHandle {
if r, ok := readable.(*fileReadable); ok {
// See fileReadable.NewReadHandle.
rh.vfsReadHandle = vfsReadHandle{r: r}
return rh
}
return readable.NewReadHandle(ctx)
}

65
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go generated vendored Normal file
View File

@@ -0,0 +1,65 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"bufio"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// NewFileWritable returns a Writable that uses a file as underlying storage.
func NewFileWritable(file vfs.File) objstorage.Writable {
return newFileBufferedWritable(file)
}
type fileBufferedWritable struct {
file vfs.File
bw *bufio.Writer
}
var _ objstorage.Writable = (*fileBufferedWritable)(nil)
func newFileBufferedWritable(file vfs.File) *fileBufferedWritable {
return &fileBufferedWritable{
file: file,
bw: bufio.NewWriter(file),
}
}
// Write is part of the objstorage.Writable interface.
func (w *fileBufferedWritable) Write(p []byte) error {
// Ignoring the length written since bufio.Writer.Write is guaranteed to
// return an error if the length written is < len(p).
_, err := w.bw.Write(p)
return err
}
// Finish is part of the objstorage.Writable interface.
func (w *fileBufferedWritable) Finish() error {
err := w.bw.Flush()
if err == nil {
err = w.file.Sync()
}
err = firstError(err, w.file.Close())
w.bw = nil
w.file = nil
return err
}
// Abort is part of the objstorage.Writable interface.
func (w *fileBufferedWritable) Abort() {
_ = w.file.Close()
w.bw = nil
w.file = nil
}
func firstError(err0, err1 error) error {
if err0 != nil {
return err0
}
return err1
}

139
vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go generated vendored Normal file
View File

@@ -0,0 +1,139 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
import (
"context"
"fmt"
"io"
"sort"
)
// WithLogging wraps the given Storage implementation and emits logs for various
// operations.
func WithLogging(wrapped Storage, logf func(fmt string, args ...interface{})) Storage {
return &loggingStore{
logf: logf,
wrapped: wrapped,
}
}
// loggingStore wraps a shared.Storage implementation and emits logs of the
// operations.
type loggingStore struct {
logf func(fmt string, args ...interface{})
wrapped Storage
}
var _ Storage = (*loggingStore)(nil)
func (l *loggingStore) Close() error {
l.logf("close")
return l.wrapped.Close()
}
func (l *loggingStore) ReadObject(
ctx context.Context, objName string,
) (_ ObjectReader, objSize int64, _ error) {
r, size, err := l.wrapped.ReadObject(ctx, objName)
l.logf("create reader for object %q: %s", objName, errOrPrintf(err, "%d bytes", size))
if err != nil {
return nil, 0, err
}
return &loggingReader{
l: l,
name: objName,
wrapped: r,
}, size, nil
}
type loggingReader struct {
l *loggingStore
name string
wrapped ObjectReader
}
var _ ObjectReader = (*loggingReader)(nil)
func (l *loggingReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
if err := l.wrapped.ReadAt(ctx, p, offset); err != nil {
l.l.logf("read object %q at %d (length %d): error %v", l.name, offset, len(p), err)
return err
}
l.l.logf("read object %q at %d (length %d)", l.name, offset, len(p))
return nil
}
func (l *loggingReader) Close() error {
l.l.logf("close reader for %q", l.name)
return l.wrapped.Close()
}
func (l *loggingStore) CreateObject(objName string) (io.WriteCloser, error) {
l.logf("create object %q", objName)
writer, err := l.wrapped.CreateObject(objName)
if err != nil {
return nil, err
}
return &loggingWriter{
l: l,
name: objName,
WriteCloser: writer,
}, nil
}
type loggingWriter struct {
l *loggingStore
name string
bytesWritten int64
io.WriteCloser
}
func (l *loggingWriter) Write(p []byte) (n int, err error) {
n, err = l.WriteCloser.Write(p)
l.bytesWritten += int64(n)
return n, err
}
func (l *loggingWriter) Close() error {
l.l.logf("close writer for %q after %d bytes", l.name, l.bytesWritten)
return l.WriteCloser.Close()
}
func (l *loggingStore) List(prefix, delimiter string) ([]string, error) {
l.logf("list (prefix=%q, delimiter=%q)", prefix, delimiter)
res, err := l.wrapped.List(prefix, delimiter)
if err != nil {
return nil, err
}
sorted := append([]string(nil), res...)
sort.Strings(sorted)
for _, s := range sorted {
l.logf(" - %s", s)
}
return res, nil
}
func (l *loggingStore) Delete(objName string) error {
l.logf("delete object %q", objName)
return l.wrapped.Delete(objName)
}
func (l *loggingStore) Size(objName string) (int64, error) {
size, err := l.wrapped.Size(objName)
l.logf("size of object %q: %s", objName, errOrPrintf(err, "%d", size))
return size, err
}
func errOrPrintf(err error, format string, args ...interface{}) string {
if err != nil {
return fmt.Sprintf("error: %v", err)
}
return fmt.Sprintf(format, args...)
}
func (l *loggingStore) IsNotExistError(err error) bool {
return l.wrapped.IsNotExistError(err)
}

161
vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go generated vendored Normal file
View File

@@ -0,0 +1,161 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
import (
"bytes"
"context"
"io"
"os"
"strings"
"sync"
)
// NewInMem returns an in-memory implementation of the shared.Storage
// interface (for testing).
func NewInMem() Storage {
store := &inMemStore{}
store.mu.objects = make(map[string]*inMemObj)
return store
}
// inMemStore is an in-memory implementation of the shared.Storage interface
// (for testing).
type inMemStore struct {
mu struct {
sync.Mutex
objects map[string]*inMemObj
}
}
var _ Storage = (*inMemStore)(nil)
type inMemObj struct {
name string
data []byte
}
func (s *inMemStore) Close() error {
*s = inMemStore{}
return nil
}
func (s *inMemStore) ReadObject(
ctx context.Context, objName string,
) (_ ObjectReader, objSize int64, _ error) {
obj, err := s.getObj(objName)
if err != nil {
return nil, 0, err
}
return &inMemReader{data: obj.data}, int64(len(obj.data)), nil
}
type inMemReader struct {
data []byte
}
var _ ObjectReader = (*inMemReader)(nil)
func (r *inMemReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
if offset+int64(len(p)) > int64(len(r.data)) {
return io.EOF
}
copy(p, r.data[offset:])
return nil
}
func (r *inMemReader) Close() error {
r.data = nil
return nil
}
func (s *inMemStore) CreateObject(objName string) (io.WriteCloser, error) {
return &inMemWriter{
store: s,
name: objName,
}, nil
}
type inMemWriter struct {
store *inMemStore
name string
buf bytes.Buffer
}
var _ io.WriteCloser = (*inMemWriter)(nil)
func (o *inMemWriter) Write(p []byte) (n int, err error) {
if o.store == nil {
panic("Write after Close")
}
return o.buf.Write(p)
}
func (o *inMemWriter) Close() error {
if o.store != nil {
o.store.addObj(&inMemObj{
name: o.name,
data: o.buf.Bytes(),
})
o.store = nil
}
return nil
}
func (s *inMemStore) List(prefix, delimiter string) ([]string, error) {
if delimiter != "" {
panic("delimiter unimplemented")
}
s.mu.Lock()
defer s.mu.Unlock()
res := make([]string, 0, len(s.mu.objects))
for name := range s.mu.objects {
if strings.HasPrefix(name, prefix) {
res = append(res, name)
}
}
return res, nil
}
func (s *inMemStore) Delete(objName string) error {
s.rmObj(objName)
return nil
}
// Size returns the length of the named object in bytesWritten.
func (s *inMemStore) Size(objName string) (int64, error) {
obj, err := s.getObj(objName)
if err != nil {
return 0, err
}
return int64(len(obj.data)), nil
}
func (s *inMemStore) IsNotExistError(err error) bool {
return err == os.ErrNotExist
}
func (s *inMemStore) getObj(name string) (*inMemObj, error) {
s.mu.Lock()
defer s.mu.Unlock()
obj, ok := s.mu.objects[name]
if !ok {
return nil, os.ErrNotExist
}
return obj, nil
}
func (s *inMemStore) addObj(o *inMemObj) {
s.mu.Lock()
defer s.mu.Unlock()
s.mu.objects[o.name] = o
}
func (s *inMemStore) rmObj(name string) {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.mu.objects, name)
}

74
vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go generated vendored Normal file
View File

@@ -0,0 +1,74 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
import (
"context"
"io"
)
// Storage is an interface for a blob storage driver. This is lower-level
// than an FS-like interface, however FS/File-like abstractions can be built on
// top of these methods.
//
// TODO(bilal): Consider pushing shared file obsoletion as well as path
// generation behind this interface.
type Storage interface {
io.Closer
// ReadObject returns an ObjectReader that can be used to perform reads on an
// object, along with the total size of the object.
ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error)
// CreateObject returns a writer for the object at the request name. A new
// empty object is created if CreateObject is called on an existing object.
//
// A Writer *must* be closed via either Close, and if closing returns a
// non-nil error, that error should be handled or reported to the user -- an
// implementation may buffer written data until Close and only then return
// an error, or Write may return an opaque io.EOF with the underlying cause
// returned by the subsequent Close().
//
// TODO(radu): if we encounter some unrelated error while writing to the
// WriteCloser, we'd want to abort the whole thing rather than letting Close
// finalize the upload.
CreateObject(objName string) (io.WriteCloser, error)
// List enumerates files within the supplied prefix, returning a list of
// objects within that prefix. If delimiter is non-empty, names which have the
// same prefix, prior to the delimiter but after the prefix, are grouped into a
// single result which is that prefix. The order that results are returned is
// undefined. If a prefix is specified, the prefix is trimmed from the result
// list.
//
// An example would be, if the storage contains objects a, b/4, b/5 and b/6,
// these would be the return values:
// List("", "") -> ["a", "b/4", "b/5", "b/6"]
// List("", "/") -> ["a", "b"]
// List("b", "/") -> ["4", "5", "6"]
// List("b", "") -> ["/4", "/5", "/6"]
List(prefix, delimiter string) ([]string, error)
// Delete removes the named object from the store.
Delete(objName string) error
// Size returns the length of the named object in bytesWritten.
Size(objName string) (int64, error)
// IsNotExistError returns true if the given error (returned by a method in
// this interface) indicates that the object does not exist.
IsNotExistError(err error) bool
}
// ObjectReader is used to perform reads on an object.
type ObjectReader interface {
// ReadAt reads len(p) bytes into p starting at offset off.
//
// Clients of ReadAt can execute parallel ReadAt calls on the
// same ObjectReader.
ReadAt(ctx context.Context, p []byte, offset int64) error
Close() error
}

660
vendor/github.com/cockroachdb/pebble/open.go generated vendored
View File

@@ -6,6 +6,7 @@ package pebble
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"math"
@@ -15,12 +16,16 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/arenaskl"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/internal/manual"
"github.com/cockroachdb/pebble/internal/rate"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/vfs"
"github.com/prometheus/client_golang/prometheus"
@@ -58,6 +63,85 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
if err := opts.Validate(); err != nil {
return nil, err
}
if opts.LoggerAndTracer == nil {
opts.LoggerAndTracer = &base.LoggerWithNoopTracer{Logger: opts.Logger}
} else {
opts.Logger = opts.LoggerAndTracer
}
// In all error cases, we return db = nil; this is used by various
// deferred cleanups.
// Open the database and WAL directories first.
walDirname, dataDir, walDir, err := prepareAndOpenDirs(dirname, opts)
if err != nil {
return nil, err
}
defer func() {
if db == nil {
if walDir != dataDir {
walDir.Close()
}
dataDir.Close()
}
}()
// Lock the database directory.
var fileLock *Lock
if opts.Lock != nil {
// The caller already acquired the database lock. Ensure that the
// directory matches.
if dirname != opts.Lock.dirname {
return nil, errors.Newf("pebble: opts.Lock acquired in %q not %q", opts.Lock.dirname, dirname)
}
if err := opts.Lock.refForOpen(); err != nil {
return nil, err
}
fileLock = opts.Lock
} else {
fileLock, err = LockDirectory(dirname, opts.FS)
if err != nil {
return nil, err
}
}
defer func() {
if db == nil {
fileLock.Close()
}
}()
// Establish the format major version.
formatVersion, formatVersionMarker, err := lookupFormatMajorVersion(opts.FS, dirname)
if err != nil {
return nil, err
}
defer func() {
if db == nil {
formatVersionMarker.Close()
}
}()
// Find the currently active manifest, if there is one.
manifestMarker, manifestFileNum, manifestExists, err := findCurrentManifest(formatVersion, opts.FS, dirname)
if err != nil {
return nil, errors.Wrapf(err, "pebble: database %q", dirname)
}
defer func() {
if db == nil {
manifestMarker.Close()
}
}()
// Atomic markers may leave behind obsolete files if there's a crash
// mid-update. Clean these up if we're not in read-only mode.
if !opts.ReadOnly {
if err := formatVersionMarker.RemoveObsolete(); err != nil {
return nil, err
}
if err := manifestMarker.RemoveObsolete(); err != nil {
return nil, err
}
}
if opts.Cache == nil {
opts.Cache = cache.New(cacheDefaultSize)
@@ -68,7 +152,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
d := &DB{
cacheID: opts.Cache.NewID(),
dirname: dirname,
walDirname: opts.WALDir,
walDirname: walDirname,
opts: opts,
cmp: opts.Comparer.Compare,
equal: opts.equal(),
@@ -76,12 +160,15 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
split: opts.Comparer.Split,
abbreviatedKey: opts.Comparer.AbbreviatedKey,
largeBatchThreshold: (opts.MemTableSize - int(memTableEmptySize)) / 2,
fileLock: fileLock,
dataDir: dataDir,
walDir: walDir,
logRecycler: logRecycler{limit: opts.MemTableStopWritesThreshold + 1},
closed: new(atomic.Value),
closedCh: make(chan struct{}),
}
d.mu.versions = &versionSet{}
d.atomic.diskAvailBytes = math.MaxUint64
d.diskAvailBytes.Store(math.MaxUint64)
d.mu.versions.diskAvailBytes = d.getDiskAvailableBytesCached
defer func() {
@@ -107,20 +194,18 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
t.arenaBuf = nil
}
}
if d.objProvider != nil {
d.objProvider.Close()
}
if r != nil {
panic(r)
}
}
}()
tableCacheSize := TableCacheSize(opts.MaxOpenFiles)
d.tableCache = newTableCacheContainer(opts.TableCache, d.cacheID, dirname, opts.FS, d.opts, tableCacheSize)
d.newIters = d.tableCache.newIters
d.tableNewRangeKeyIter = d.tableCache.newRangeKeyIter
d.commit = newCommitPipeline(commitEnv{
logSeqNum: &d.mu.versions.atomic.logSeqNum,
visibleSeqNum: &d.mu.versions.atomic.visibleSeqNum,
logSeqNum: &d.mu.versions.logSeqNum,
visibleSeqNum: &d.mu.versions.visibleSeqNum,
apply: d.commitApply,
write: d.commitWrite,
})
@@ -132,150 +217,53 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
if d.mu.mem.nextSize > initialMemTableSize {
d.mu.mem.nextSize = initialMemTableSize
}
d.mu.mem.cond.L = &d.mu.Mutex
d.mu.cleaner.cond.L = &d.mu.Mutex
d.mu.compact.cond.L = &d.mu.Mutex
d.mu.compact.inProgress = make(map[*compaction]struct{})
d.mu.compact.noOngoingFlushStartTime = time.Now()
d.mu.snapshots.init()
// logSeqNum is the next sequence number that will be assigned. Start
// assigning sequence numbers from 1 to match rocksdb.
d.mu.versions.atomic.logSeqNum = 1
// assigning sequence numbers from base.SeqNumStart to leave room for reserved
// sequence numbers (see comments around SeqNumStart).
d.mu.versions.logSeqNum.Store(base.SeqNumStart)
d.mu.formatVers.vers = formatVersion
d.mu.formatVers.marker = formatVersionMarker
d.timeNow = time.Now
d.openedAt = d.timeNow()
d.mu.Lock()
defer d.mu.Unlock()
if !d.opts.ReadOnly {
err := opts.FS.MkdirAll(dirname, 0755)
if err != nil {
return nil, err
}
}
// Ensure we close resources if we error out early. If the database is
// successfully opened, the named return value `db` will be set to `d`.
defer func() {
if db != nil {
// The database was successfully opened.
return
}
if d.dataDir != nil {
d.dataDir.Close()
}
if d.walDirname != d.dirname && d.walDir != nil {
d.walDir.Close()
}
if d.mu.formatVers.marker != nil {
d.mu.formatVers.marker.Close()
}
}()
// Open the database and WAL directories first in order to check for their
// existence.
var err error
d.dataDir, err = opts.FS.OpenDir(dirname)
if err != nil {
return nil, err
}
if d.walDirname == "" {
d.walDirname = d.dirname
}
if d.walDirname == d.dirname {
d.walDir = d.dataDir
} else {
if !d.opts.ReadOnly {
err := opts.FS.MkdirAll(d.walDirname, 0755)
if err != nil {
return nil, err
}
}
d.walDir, err = opts.FS.OpenDir(d.walDirname)
if err != nil {
return nil, err
}
}
// Lock the database directory.
fileLock, err := opts.FS.Lock(base.MakeFilepath(opts.FS, dirname, fileTypeLock, 0))
if err != nil {
d.dataDir.Close()
if d.dataDir != d.walDir {
d.walDir.Close()
}
return nil, err
}
defer func() {
if fileLock != nil {
fileLock.Close()
}
}()
// Establish the format major version.
{
d.mu.formatVers.vers, d.mu.formatVers.marker, err = lookupFormatMajorVersion(opts.FS, dirname)
if err != nil {
return nil, err
}
if !d.opts.ReadOnly {
if err := d.mu.formatVers.marker.RemoveObsolete(); err != nil {
return nil, err
}
}
}
jobID := d.mu.nextJobID
d.mu.nextJobID++
// Find the currently active manifest, if there is one.
manifestMarker, manifestFileNum, exists, err := findCurrentManifest(d.mu.formatVers.vers, opts.FS, dirname)
setCurrent := setCurrentFunc(d.mu.formatVers.vers, manifestMarker, opts.FS, dirname, d.dataDir)
defer func() {
// Ensure we close the manifest marker if we error out for any reason.
// If the database is successfully opened, the *versionSet will take
// ownership over the manifest marker, ensuring it's closed when the DB
// is closed.
if db == nil {
manifestMarker.Close()
}
}()
if err != nil {
return nil, errors.Wrapf(err, "pebble: database %q", dirname)
} else if !exists && !d.opts.ReadOnly && !d.opts.ErrorIfNotExists {
// Create the DB if it did not already exist.
if !manifestExists {
// DB does not exist.
if d.opts.ErrorIfNotExists || d.opts.ReadOnly {
return nil, errors.Wrapf(ErrDBDoesNotExist, "dirname=%q", dirname)
}
// Create the DB.
if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
return nil, err
}
} else if opts.ErrorIfExists {
return nil, errors.Errorf("pebble: database %q already exists", dirname)
} else {
if opts.ErrorIfExists {
return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname)
}
// Load the version set.
if err := d.mu.versions.load(dirname, opts, manifestFileNum, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
return nil, err
}
if err := d.mu.versions.currentVersion().CheckConsistency(dirname, opts.FS); err != nil {
return nil, err
}
}
// If the Options specify a format major version higher than the
// loaded database's, upgrade it. If this is a new database, this
// code path also performs an initial upgrade from the starting
// implicit MostCompatible version.
if !d.opts.ReadOnly && opts.FormatMajorVersion > d.mu.formatVers.vers {
if err := d.ratchetFormatMajorVersionLocked(opts.FormatMajorVersion); err != nil {
return nil, err
}
}
// Atomic markers like the one used for the MANIFEST may leave
// behind obsolete files if there's a crash mid-update. Clean these
// up if we're not in read-only mode.
if !d.opts.ReadOnly {
if err := manifestMarker.RemoveObsolete(); err != nil {
return nil, err
if opts.ErrorIfNotPristine {
liveFileNums := make(map[base.DiskFileNum]struct{})
d.mu.versions.addLiveFileNums(liveFileNums)
if len(liveFileNums) != 0 {
return nil, errors.Wrapf(ErrDBNotPristine, "dirname=%q", dirname)
}
}
}
@@ -284,10 +272,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
// sequence number of the first batch that will be inserted.
if !d.opts.ReadOnly {
var entry *flushableEntry
d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.atomic.logSeqNum)
d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.logSeqNum.Load())
d.mu.mem.queue = append(d.mu.mem.queue, entry)
}
// List the objects
ls, err := opts.FS.List(d.walDirname)
if err != nil {
return nil, err
@@ -299,6 +288,33 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
ls = append(ls, ls2...)
}
providerSettings := objstorageprovider.Settings{
Logger: opts.Logger,
FS: opts.FS,
FSDirName: dirname,
FSDirInitialListing: ls,
FSCleaner: opts.Cleaner,
NoSyncOnClose: opts.NoSyncOnClose,
BytesPerSync: opts.BytesPerSync,
}
providerSettings.Shared.Storage = opts.Experimental.SharedStorage
d.objProvider, err = objstorageprovider.Open(providerSettings)
if err != nil {
return nil, err
}
if manifestExists {
curVersion := d.mu.versions.currentVersion()
if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil {
return nil, err
}
}
tableCacheSize := TableCacheSize(opts.MaxOpenFiles)
d.tableCache = newTableCacheContainer(opts.TableCache, d.cacheID, d.objProvider, d.opts, tableCacheSize)
d.newIters = d.tableCache.newIters
d.tableNewRangeKeyIter = d.tableCache.newRangeKeyIter
// Replay any newer log files than the ones named in the manifest.
type fileNumAndName struct {
@@ -316,21 +332,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
// Don't reuse any obsolete file numbers to avoid modifying an
// ingested sstable's original external file.
if d.mu.versions.nextFileNum <= fn {
d.mu.versions.nextFileNum = fn + 1
if d.mu.versions.nextFileNum <= fn.FileNum() {
d.mu.versions.nextFileNum = fn.FileNum() + 1
}
switch ft {
case fileTypeLog:
if fn >= d.mu.versions.minUnflushedLogNum {
logFiles = append(logFiles, fileNumAndName{fn, filename})
if fn.FileNum() >= d.mu.versions.minUnflushedLogNum {
logFiles = append(logFiles, fileNumAndName{fn.FileNum(), filename})
}
if d.logRecycler.minRecycleLogNum <= fn {
d.logRecycler.minRecycleLogNum = fn + 1
if d.logRecycler.minRecycleLogNum <= fn.FileNum() {
d.logRecycler.minRecycleLogNum = fn.FileNum() + 1
}
case fileTypeOptions:
if previousOptionsFileNum < fn {
previousOptionsFileNum = fn
if previousOptionsFileNum < fn.FileNum() {
previousOptionsFileNum = fn.FileNum()
previousOptionsFilename = filename
}
case fileTypeTemp, fileTypeOldTemp:
@@ -362,19 +378,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
})
var ve versionEdit
var toFlush flushableList
for i, lf := range logFiles {
lastWAL := i == len(logFiles)-1
maxSeqNum, err := d.replayWAL(jobID, &ve, opts.FS,
flush, maxSeqNum, err := d.replayWAL(jobID, &ve, opts.FS,
opts.FS.PathJoin(d.walDirname, lf.name), lf.num, strictWALTail && !lastWAL)
if err != nil {
return nil, err
}
toFlush = append(toFlush, flush...)
d.mu.versions.markFileNumUsed(lf.num)
if d.mu.versions.atomic.logSeqNum < maxSeqNum {
d.mu.versions.atomic.logSeqNum = maxSeqNum
if d.mu.versions.logSeqNum.Load() < maxSeqNum {
d.mu.versions.logSeqNum.Store(maxSeqNum)
}
}
d.mu.versions.atomic.visibleSeqNum = d.mu.versions.atomic.logSeqNum
d.mu.versions.visibleSeqNum.Store(d.mu.versions.logSeqNum.Load())
if !d.opts.ReadOnly {
// Create an empty .log file.
@@ -396,8 +414,12 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
return nil, err
}
newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum)
d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: 0})
for _, entry := range toFlush {
entry.readerUnrefLocked(true)
}
newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum.DiskFileNum())
d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum.DiskFileNum(), fileSize: 0})
logFile, err := opts.FS.Create(newLogName)
if err != nil {
return nil, err
@@ -426,15 +448,29 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
logWriterConfig := record.LogWriterConfig{
WALMinSyncInterval: d.opts.WALMinSyncInterval,
WALFsyncLatency: d.mu.log.metrics.fsyncLatency,
QueueSemChan: d.commit.logSyncQSem,
}
d.mu.log.LogWriter = record.NewLogWriter(logFile, newLogNum, logWriterConfig)
d.mu.versions.metrics.WAL.Files++
}
d.updateReadStateLocked(d.opts.DebugCheck)
// If the Options specify a format major version higher than the
// loaded database's, upgrade it. If this is a new database, this
// code path also performs an initial upgrade from the starting
// implicit MostCompatible version.
//
// We ratchet the version this far into Open so that migrations have a read
// state available.
if !d.opts.ReadOnly && opts.FormatMajorVersion > d.mu.formatVers.vers {
if err := d.ratchetFormatMajorVersionLocked(opts.FormatMajorVersion); err != nil {
return nil, err
}
}
if !d.opts.ReadOnly {
// Write the current options to disk.
d.optionsFileNum = d.mu.versions.getNextFileNum()
d.optionsFileNum = d.mu.versions.getNextFileNum().DiskFileNum()
tmpPath := base.MakeFilepath(opts.FS, dirname, fileTypeTemp, d.optionsFileNum)
optionsPath := base.MakeFilepath(opts.FS, dirname, fileTypeOptions, d.optionsFileNum)
@@ -509,10 +545,53 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
})
d.fileLock, fileLock = fileLock, nil
return d, nil
}
// prepareAndOpenDirs opens the directories for the store (and creates them if
// necessary).
//
// Returns an error if ReadOnly is set and the directories don't exist.
func prepareAndOpenDirs(
dirname string, opts *Options,
) (walDirname string, dataDir vfs.File, walDir vfs.File, err error) {
walDirname = opts.WALDir
if opts.WALDir == "" {
walDirname = dirname
}
// Create directories if needed.
if !opts.ReadOnly {
if err := opts.FS.MkdirAll(dirname, 0755); err != nil {
return "", nil, nil, err
}
if walDirname != dirname {
if err := opts.FS.MkdirAll(walDirname, 0755); err != nil {
return "", nil, nil, err
}
}
}
dataDir, err = opts.FS.OpenDir(dirname)
if err != nil {
if opts.ReadOnly && oserror.IsNotExist(err) {
return "", nil, nil, errors.Errorf("pebble: database %q does not exist", dirname)
}
return "", nil, nil, err
}
if walDirname == dirname {
walDir = dataDir
} else {
walDir, err = opts.FS.OpenDir(walDirname)
if err != nil {
dataDir.Close()
return "", nil, nil, err
}
}
return walDirname, dataDir, walDir, nil
}
// GetVersion returns the engine version string from the latest options
// file present in dir. Used to check what Pebble or RocksDB version was last
// used to write to the database stored in this directory. An empty string is
@@ -535,9 +614,9 @@ func GetVersion(dir string, fs vfs.FS) (string, error) {
// processed, reset version. This is because rocksdb often
// writes multiple options files without deleting previous ones.
// Otherwise, skip parsing this options file.
if fn > lastOptionsSeen {
if fn.FileNum() > lastOptionsSeen {
version = ""
lastOptionsSeen = fn
lastOptionsSeen = fn.FileNum()
} else {
continue
}
@@ -571,25 +650,31 @@ func GetVersion(dir string, fs vfs.FS) (string, error) {
return version, nil
}
// replayWAL replays the edits in the specified log file.
// replayWAL replays the edits in the specified log file. If the DB is in
// read only mode, then the WALs are replayed into memtables and not flushed. If
// the DB is not in read only mode, then the contents of the WAL are guaranteed
// to be flushed.
//
// The toFlush return value is a list of flushables associated with the WAL
// being replayed which will be flushed. Once the version edit has been applied
// to the manifest, it is up to the caller of replayWAL to unreference the
// toFlush flushables returned by replayWAL.
//
// d.mu must be held when calling this, but the mutex may be dropped and
// re-acquired during the course of this method.
func (d *DB) replayWAL(
jobID int, ve *versionEdit, fs vfs.FS, filename string, logNum FileNum, strictWALTail bool,
) (maxSeqNum uint64, err error) {
) (toFlush flushableList, maxSeqNum uint64, err error) {
file, err := fs.Open(filename)
if err != nil {
return 0, err
return nil, 0, err
}
defer file.Close()
var (
b Batch
buf bytes.Buffer
mem *memTable
entry *flushableEntry
toFlush flushableList
rr = record.NewReader(file, logNum)
offset int64 // byte offset in rr
lastFlushOffset int64
@@ -633,6 +718,23 @@ func (d *DB) replayWAL(
d.mu.mem.queue = append(d.mu.mem.queue, entry)
}
}
// updateVE is used to update ve with information about new files created
// during the flush of any flushable not of type ingestedFlushable. For the
// flushable of type ingestedFlushable we use custom handling below.
updateVE := func() error {
// TODO(bananabrick): See if we can use the actual base level here,
// instead of using 1.
c := newFlush(d.opts, d.mu.versions.currentVersion(),
1 /* base level */, toFlush, d.timeNow())
newVE, _, _, err := d.runCompaction(jobID, c)
if err != nil {
return errors.Wrapf(err, "running compaction during WAL replay")
}
ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...)
return nil
}
for {
offset = rr.Offset()
r, err := rr.Next()
@@ -650,14 +752,18 @@ func (d *DB) replayWAL(
} else if record.IsInvalidRecord(err) && !strictWALTail {
break
}
return 0, errors.Wrap(err, "pebble: error when replaying WAL")
return nil, 0, errors.Wrap(err, "pebble: error when replaying WAL")
}
if buf.Len() < batchHeaderLen {
return 0, base.CorruptionErrorf("pebble: corrupt log file %q (num %s)",
return nil, 0, base.CorruptionErrorf("pebble: corrupt log file %q (num %s)",
filename, errors.Safe(logNum))
}
if d.opts.ErrorIfNotPristine {
return nil, 0, errors.WithDetailf(ErrDBNotPristine, "location: %q", d.dirname)
}
// Specify Batch.db so that Batch.SetRepr will compute Batch.memTableSize
// which is used below.
b = Batch{db: d}
@@ -665,6 +771,100 @@ func (d *DB) replayWAL(
seqNum := b.SeqNum()
maxSeqNum = seqNum + uint64(b.Count())
{
br := b.Reader()
if kind, encodedFileNum, _, _ := br.Next(); kind == InternalKeyKindIngestSST {
fileNums := make([]base.DiskFileNum, 0, b.Count())
addFileNum := func(encodedFileNum []byte) {
fileNum, n := binary.Uvarint(encodedFileNum)
if n <= 0 {
panic("pebble: ingest sstable file num is invalid.")
}
fileNums = append(fileNums, base.FileNum(fileNum).DiskFileNum())
}
addFileNum(encodedFileNum)
for i := 1; i < int(b.Count()); i++ {
kind, encodedFileNum, _, ok := br.Next()
if kind != InternalKeyKindIngestSST {
panic("pebble: invalid batch key kind.")
}
if !ok {
panic("pebble: invalid batch count.")
}
addFileNum(encodedFileNum)
}
if _, _, _, ok := br.Next(); ok {
panic("pebble: invalid number of entries in batch.")
}
paths := make([]string, len(fileNums))
for i, n := range fileNums {
paths[i] = base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
}
var meta []*manifest.FileMetadata
meta, _, err = ingestLoad(
d.opts, d.mu.formatVers.vers, paths, d.cacheID, fileNums,
)
if err != nil {
return nil, 0, err
}
if uint32(len(meta)) != b.Count() {
panic("pebble: couldn't load all files in WAL entry.")
}
entry, err = d.newIngestedFlushableEntry(
meta, seqNum, logNum,
)
if err != nil {
return nil, 0, err
}
if d.opts.ReadOnly {
d.mu.mem.queue = append(d.mu.mem.queue, entry)
// We added the IngestSST flushable to the queue. But there
// must be at least one WAL entry waiting to be replayed. We
// have to ensure this newer WAL entry isn't replayed into
// the current value of d.mu.mem.mutable because the current
// mutable memtable exists before this flushable entry in
// the memtable queue. To ensure this, we just need to unset
// d.mu.mem.mutable. When a newer WAL is replayed, we will
// set d.mu.mem.mutable to a newer value.
d.mu.mem.mutable = nil
} else {
toFlush = append(toFlush, entry)
// During WAL replay, the lsm only has L0, hence, the
// baseLevel is 1. For the sake of simplicity, we place the
// ingested files in L0 here, instead of finding their
// target levels. This is a simplification for the sake of
// simpler code. It is expected that WAL replay should be
// rare, and that flushables of type ingestedFlushable
// should also be rare. So, placing the ingested files in L0
// is alright.
//
// TODO(bananabrick): Maybe refactor this function to allow
// us to easily place ingested files in levels as low as
// possible during WAL replay. It would require breaking up
// the application of ve to the manifest into chunks and is
// not pretty w/o a refactor to this function and how it's
// used.
c := newFlush(
d.opts, d.mu.versions.currentVersion(),
1, /* base level */
[]*flushableEntry{entry},
d.timeNow(),
)
for _, file := range c.flushing[0].flushable.(*ingestedFlushable).files {
ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file.FileMetadata})
}
}
return toFlush, maxSeqNum, nil
}
}
if b.memTableSize >= uint64(d.largeBatchThreshold) {
flushMem()
// Make a copy of the data slice since it is currently owned by buf and will
@@ -677,13 +877,22 @@ func (d *DB) replayWAL(
entry.readerRefs++
if d.opts.ReadOnly {
d.mu.mem.queue = append(d.mu.mem.queue, entry)
// We added the flushable batch to the flushable to the queue.
// But there must be at least one WAL entry waiting to be
// replayed. We have to ensure this newer WAL entry isn't
// replayed into the current value of d.mu.mem.mutable because
// the current mutable memtable exists before this flushable
// entry in the memtable queue. To ensure this, we just need to
// unset d.mu.mem.mutable. When a newer WAL is replayed, we will
// set d.mu.mem.mutable to a newer value.
d.mu.mem.mutable = nil
} else {
toFlush = append(toFlush, entry)
}
} else {
ensureMem(seqNum)
if err = mem.prepare(&b); err != nil && err != arenaskl.ErrArenaFull {
return 0, err
return nil, 0, err
}
// We loop since DB.newMemTable() slowly grows the size of allocated memtables, so the
// batch may not initially fit, but will eventually fit (since it is smaller than
@@ -693,11 +902,11 @@ func (d *DB) replayWAL(
ensureMem(seqNum)
err = mem.prepare(&b)
if err != nil && err != arenaskl.ErrArenaFull {
return 0, err
return nil, 0, err
}
}
if err = mem.apply(&b, seqNum); err != nil {
return 0, err
return nil, 0, err
}
mem.writerUnref()
}
@@ -706,18 +915,12 @@ func (d *DB) replayWAL(
flushMem()
// mem is nil here.
if !d.opts.ReadOnly {
c := newFlush(d.opts, d.mu.versions.currentVersion(),
1 /* base level */, toFlush)
newVE, _, err := d.runCompaction(jobID, c)
err = updateVE()
if err != nil {
return 0, err
}
ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...)
for i := range toFlush {
toFlush[i].readerUnref()
return nil, 0, err
}
}
return maxSeqNum, err
return toFlush, maxSeqNum, err
}
func checkOptions(opts *Options, path string) (strictWALTail bool, err error) {
@@ -748,7 +951,7 @@ type DBDesc struct {
// Peek looks for an existing database in dirname on the provided FS. It
// returns a brief description of the database. Peek is read-only and
// does not open the database.
// does not open the database
func Peek(dirname string, fs vfs.FS) (*DBDesc, error) {
vers, versMarker, err := lookupFormatMajorVersion(fs, dirname)
if err != nil {
@@ -780,3 +983,124 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) {
}
return desc, nil
}
// LockDirectory acquires the database directory lock in the named directory,
// preventing another process from opening the database. LockDirectory returns a
// handle to the held lock that may be passed to Open through Options.Lock to
// subsequently open the database, skipping lock acquistion during Open.
//
// LockDirectory may be used to expand the critical section protected by the
// database lock to include setup before the call to Open.
func LockDirectory(dirname string, fs vfs.FS) (*Lock, error) {
fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, base.FileNum(0).DiskFileNum()))
if err != nil {
return nil, err
}
l := &Lock{dirname: dirname, fileLock: fileLock}
l.refs.Store(1)
invariants.SetFinalizer(l, func(obj interface{}) {
if refs := obj.(*Lock).refs.Load(); refs > 0 {
panic(errors.AssertionFailedf("lock for %q finalized with %d refs", dirname, refs))
}
})
return l, nil
}
// Lock represents a file lock on a directory. It may be passed to Open through
// Options.Lock to elide lock aquisition during Open.
type Lock struct {
dirname string
fileLock io.Closer
// refs is a count of the number of handles on the lock. refs must be 0, 1
// or 2.
//
// When acquired by the client and passed to Open, refs = 1 and the Open
// call increments it to 2. When the database is closed, it's decremented to
// 1. Finally when the original caller, calls Close on the Lock, it's
// drecemented to zero and the underlying file lock is released.
//
// When Open acquires the file lock, refs remains at 1 until the database is
// closed.
refs atomic.Int32
}
func (l *Lock) refForOpen() error {
// During Open, when a user passed in a lock, the reference count must be
// exactly 1. If it's zero, the lock is no longer held and is invalid. If
// it's 2, the lock is already in use by another database within the
// process.
if !l.refs.CompareAndSwap(1, 2) {
return errors.Errorf("pebble: unexpected Lock reference count; is the lock already in use?")
}
return nil
}
// Close releases the lock, permitting another process to lock and open the
// database. Close must not be called until after a database using the Lock has
// been closed.
func (l *Lock) Close() error {
if l.refs.Add(-1) > 0 {
return nil
}
defer func() { l.fileLock = nil }()
return l.fileLock.Close()
}
// ErrDBDoesNotExist is generated when ErrorIfNotExists is set and the database
// does not exist.
//
// Note that errors can be wrapped with more details; use errors.Is().
var ErrDBDoesNotExist = errors.New("pebble: database does not exist")
// ErrDBAlreadyExists is generated when ErrorIfExists is set and the database
// already exists.
//
// Note that errors can be wrapped with more details; use errors.Is().
var ErrDBAlreadyExists = errors.New("pebble: database already exists")
// ErrDBNotPristine is generated when ErrorIfNotPristine is set and the database
// already exists and is not pristine.
//
// Note that errors can be wrapped with more details; use errors.Is().
var ErrDBNotPristine = errors.New("pebble: database already exists and is not pristine")
func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error {
var buf bytes.Buffer
var args []interface{}
dedup := make(map[base.DiskFileNum]struct{})
for level, files := range v.Levels {
iter := files.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
backingState := f.FileBacking
if _, ok := dedup[backingState.DiskFileNum]; ok {
continue
}
dedup[backingState.DiskFileNum] = struct{}{}
fileNum := backingState.DiskFileNum
fileSize := backingState.Size
meta, err := objProvider.Lookup(base.FileTypeTable, fileNum)
var size int64
if err == nil {
size, err = objProvider.Size(meta)
}
if err != nil {
buf.WriteString("L%d: %s: %v\n")
args = append(args, errors.Safe(level), errors.Safe(fileNum), err)
continue
}
if size != int64(fileSize) {
buf.WriteString("L%d: %s: object size mismatch (%s): %d (disk) != %d (MANIFEST)\n")
args = append(args, errors.Safe(level), errors.Safe(fileNum), objProvider.Path(meta),
errors.Safe(size), errors.Safe(fileSize))
continue
}
}
}
if buf.Len() == 0 {
return nil
}
return errors.Errorf(buf.String(), args...)
}

210
vendor/github.com/cockroachdb/pebble/options.go generated vendored
View File

@@ -18,6 +18,7 @@ import (
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
)
@@ -61,6 +62,12 @@ type BlockPropertyCollector = sstable.BlockPropertyCollector
// BlockPropertyFilter exports the sstable.BlockPropertyFilter type.
type BlockPropertyFilter = base.BlockPropertyFilter
// ShortAttributeExtractor exports the base.ShortAttributeExtractor type.
type ShortAttributeExtractor = base.ShortAttributeExtractor
// UserKeyPrefixBound exports the sstable.UserKeyPrefixBound type.
type UserKeyPrefixBound = sstable.UserKeyPrefixBound
// IterKeyType configures which types of keys an iterator should surface.
type IterKeyType int8
@@ -215,6 +222,16 @@ func (o *IterOptions) getLogger() Logger {
return o.logger
}
// scanInternalOptions is similar to IterOptions, meant for use with
// scanInternalIterator.
type scanInternalOptions struct {
IterOptions
// skipSharedLevels skips levels that are shareable (level >=
// sharedLevelStart).
skipSharedLevels bool
}
// RangeKeyMasking configures automatic hiding of point keys by range keys. A
// non-nil Suffix enables range-key masking. When enabled, range keys with
// suffixes ≥ Suffix behave as masks. All point keys that are contained within a
@@ -224,7 +241,7 @@ func (o *IterOptions) getLogger() Logger {
// Specifically, when configured with a RangeKeyMasking.Suffix _s_, and there
// exists a range key with suffix _r_ covering a point key with suffix _p_, and
//
// _s_ ≤ _r_ < _p_
// _s_ ≤ _r_ < _p_
//
// then the point key is elided.
//
@@ -387,6 +404,8 @@ func (o *LevelOptions) EnsureDefaults() *LevelOptions {
}
if o.BlockSize <= 0 {
o.BlockSize = base.DefaultBlockSize
} else if o.BlockSize > sstable.MaximumBlockSize {
panic(errors.Errorf("BlockSize %d exceeds MaximumBlockSize", o.BlockSize))
}
if o.BlockSizeThreshold <= 0 {
o.BlockSizeThreshold = base.DefaultBlockSizeThreshold
@@ -445,21 +464,31 @@ type Options struct {
// TODO(peter): untested
DisableWAL bool
// ErrorIfExists is whether it is an error if the database already exists.
// ErrorIfExists causes an error on Open if the database already exists.
// The error can be checked with errors.Is(err, ErrDBAlreadyExists).
//
// The default value is false.
ErrorIfExists bool
// ErrorIfNotExists is whether it is an error if the database does not
// already exist.
// ErrorIfNotExists causes an error on Open if the database does not already
// exist. The error can be checked with errors.Is(err, ErrDBDoesNotExist).
//
// The default value is false which will cause a database to be created if it
// does not already exist.
ErrorIfNotExists bool
// ErrorIfNotPristine causes an error on Open if the database already exists
// and any operations have been performed on the database. The error can be
// checked with errors.Is(err, ErrDBNotPristine).
//
// Note that a database that contained keys that were all subsequently deleted
// may or may not trigger the error. Currently, we check if there are any live
// SSTs or log records to replay.
ErrorIfNotPristine bool
// EventListener provides hooks to listening to significant DB events such as
// flushes, compactions, and table deletion.
EventListener EventListener
EventListener *EventListener
// Experimental contains experimental options which are off by default.
// These options are temporary and will eventually either be deleted, moved
@@ -554,10 +583,10 @@ type Options struct {
// desired size of each level of the LSM. Defaults to 10.
LevelMultiplier int
// MultiLevelCompaction allows the compaction of SSTs from more than two
// levels iff a conventional two level compaction will quickly trigger a
// compaction in the output level.
MultiLevelCompaction bool
// MultiLevelCompactionHueristic determines whether to add an additional
// level to a conventional two level compaction. If nil, a multilevel
// compaction will never get triggered.
MultiLevelCompactionHueristic MultiLevelHeuristic
// MaxWriterConcurrency is used to indicate the maximum number of
// compression workers the compression queue is allowed to use. If
@@ -582,6 +611,49 @@ type Options struct {
//
// The default value is 1, which results in no scaling of point tombstones.
PointTombstoneWeight float64
// EnableValueBlocks is used to decide whether to enable writing
// TableFormatPebblev3 sstables. This setting is only respected by a
// specific subset of format major versions: FormatSSTableValueBlocks,
// FormatFlushableIngest and FormatPrePebblev1MarkedCompacted. In lower
// format major versions, value blocks are never enabled. In higher
// format major versions, value blocks are always enabled.
EnableValueBlocks func() bool
// ShortAttributeExtractor is used iff EnableValueBlocks() returns true
// (else ignored). If non-nil, a ShortAttribute can be extracted from the
// value and stored with the key, when the value is stored elsewhere.
ShortAttributeExtractor ShortAttributeExtractor
// RequiredInPlaceValueBound specifies an optional span of user key
// prefixes for which the values must be stored with the key. This is
// useful for statically known exclusions to value separation. In
// CockroachDB, this will be used for the lock table key space that has
// non-empty suffixes, but those locks don't represent actual MVCC
// versions (the suffix ordering is arbitrary). We will also need to add
// support for dynamically configured exclusions (we want the default to
// be to allow Pebble to decide whether to separate the value or not,
// hence this is structured as exclusions), for example, for users of
// CockroachDB to dynamically exclude certain tables.
//
// Any change in exclusion behavior takes effect only on future written
// sstables, and does not start rewriting existing sstables.
RequiredInPlaceValueBound UserKeyPrefixBound
// DisableIngestAsFlushable disables lazy ingestion of sstables through
// a WAL write and memtable rotation. Only effectual if the the format
// major version is at least `FormatFlushableIngest`.
DisableIngestAsFlushable func() bool
// SharedStorage is a second FS-like storage medium that can be shared
// between multiple Pebble instances. It is used to store sstables only, and
// is managed by objstorage.Provider. Each sstable might only be written to
// by one Pebble instance, but other Pebble instances can possibly read the
// same files if they have the path to get to them. The pebble instance that
// wrote a file should not delete it if other Pebble instances are known to
// be reading this file. This FS is expected to have slower read/write
// performance than the default FS above.
SharedStorage shared.Storage
}
// Filters is a map from filter policy name to filter policy. It is used for
@@ -635,6 +707,17 @@ type Options struct {
// The default value uses the underlying operating system's file system.
FS vfs.FS
// Lock, if set, must be a database lock acquired through LockDirectory for
// the same directory passed to Open. If provided, Open will skip locking
// the directory. Closing the database will not release the lock, and it's
// the responsibility of the caller to release the lock after closing the
// database.
//
// Open will enforce that the Lock passed locks the same directory passed to
// Open. Concurrent calls to Open using the same Lock are detected and
// prohibited.
Lock *Lock
// The count of L0 files necessary to trigger an L0 compaction.
L0CompactionFileThreshold int
@@ -656,10 +739,15 @@ type Options struct {
// options for the last level are used for all subsequent levels.
Levels []LevelOptions
// LoggerAndTracer will be used, if non-nil, else Logger will be used and
// tracing will be a noop.
// Logger used to write log messages.
//
// The default logger uses the Go standard library log package.
Logger Logger
// LoggerAndTracer is used for writing log messages and traces.
LoggerAndTracer LoggerAndTracer
// MaxManifestFileSize is the maximum size the MANIFEST file is allowed to
// become. When the MANIFEST exceeds this size it is rolled over and a new
@@ -783,6 +871,23 @@ type Options struct {
// RocksDB 6.2.1.
strictWALTail bool
// disableDeleteOnlyCompactions prevents the scheduling of delete-only
// compactions that drop sstables wholy covered by range tombstones or
// range key tombstones.
disableDeleteOnlyCompactions bool
// disableElisionOnlyCompactions prevents the scheduling of elision-only
// compactions that rewrite sstables in place in order to elide obsolete
// keys.
disableElisionOnlyCompactions bool
// disableLazyCombinedIteration is a private option used by the
// metamorphic tests to test equivalence between lazy-combined iteration
// and constructing the range-key iterator upfront. It's a private
// option to avoid littering the public interface with options that we
// do not want to allow users to actually configure.
disableLazyCombinedIteration bool
// A private option to disable stats collection.
disableTableStats bool
@@ -795,13 +900,6 @@ type Options struct {
// against the FS are made after the DB is closed, the FS may leak a
// goroutine indefinitely.
fsCloser io.Closer
// disableLazyCombinedIteration is a private option used by the
// metamorphic tests to test equivalence between lazy-combined iteration
// and constructing the range-key iterator upfront. It's a private
// option to avoid littering the public interface with options that we
// do not want to allow users to actually configure.
disableLazyCombinedIteration bool
}
}
@@ -827,6 +925,9 @@ func (o *Options) EnsureDefaults() *Options {
if o.Comparer == nil {
o.Comparer = DefaultComparer
}
if o.Experimental.DisableIngestAsFlushable == nil {
o.Experimental.DisableIngestAsFlushable = func() bool { return false }
}
if o.Experimental.L0CompactionConcurrency <= 0 {
o.Experimental.L0CompactionConcurrency = 10
}
@@ -890,6 +991,9 @@ func (o *Options) EnsureDefaults() *Options {
if o.Logger == nil {
o.Logger = DefaultLogger
}
if o.EventListener == nil {
o.EventListener = &EventListener{}
}
o.EventListener.EnsureDefaults(o.Logger)
if o.MaxManifestFileSize == 0 {
o.MaxManifestFileSize = 128 << 20 // 128 MB
@@ -919,13 +1023,7 @@ func (o *Options) EnsureDefaults() *Options {
}
if o.FS == nil {
o.FS, o.private.fsCloser = vfs.WithDiskHealthChecks(vfs.Default, 5*time.Second,
func(name string, duration time.Duration) {
o.EventListener.DiskSlow(DiskSlowInfo{
Path: name,
Duration: duration,
})
})
o.WithFSDefaults()
}
if o.FlushSplitBytes <= 0 {
o.FlushSplitBytes = 2 * o.Levels[0].TargetFileSize
@@ -949,10 +1047,36 @@ func (o *Options) EnsureDefaults() *Options {
o.Experimental.PointTombstoneWeight = 1
}
if o.Experimental.MultiLevelCompactionHueristic == nil {
o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{}
}
o.initMaps()
return o
}
// WithFSDefaults configures the Options to wrap the configured filesystem with
// the default virtual file system middleware, like disk-health checking.
func (o *Options) WithFSDefaults() *Options {
if o.FS == nil {
o.FS = vfs.Default
}
o.FS, o.private.fsCloser = vfs.WithDiskHealthChecks(o.FS, 5*time.Second,
func(info vfs.DiskSlowInfo) {
o.EventListener.DiskSlow(info)
})
return o
}
// AddEventListener adds the provided event listener to the Options, in addition
// to any existing event listener.
func (o *Options) AddEventListener(l EventListener) {
if o.EventListener != nil {
l = TeeEventListener(l, *o.EventListener)
}
o.EventListener = &l
}
func (o *Options) equal() Equal {
if o.Comparer.Equal == nil {
return bytes.Equal
@@ -1023,6 +1147,9 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, " compaction_debt_concurrency=%d\n", o.Experimental.CompactionDebtConcurrency)
fmt.Fprintf(&buf, " comparer=%s\n", o.Comparer.Name)
fmt.Fprintf(&buf, " disable_wal=%t\n", o.DisableWAL)
if o.Experimental.DisableIngestAsFlushable != nil && o.Experimental.DisableIngestAsFlushable() {
fmt.Fprintf(&buf, " disable_ingest_as_flushable=%t\n", true)
}
fmt.Fprintf(&buf, " flush_delay_delete_range=%s\n", o.FlushDelayDeleteRange)
fmt.Fprintf(&buf, " flush_delay_range_key=%s\n", o.FlushDelayRangeKey)
fmt.Fprintf(&buf, " flush_split_bytes=%d\n", o.FlushSplitBytes)
@@ -1064,11 +1191,18 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, " force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism)
// Private options.
//
// These options are only encoded if true, because we do not want them to
// appear in production serialized Options files, since they're testing-only
// options. They're only serialized when true, which still ensures that the
// metamorphic tests may propagate them to subprocesses.
if o.private.disableDeleteOnlyCompactions {
fmt.Fprintln(&buf, " disable_delete_only_compactions=true")
}
if o.private.disableElisionOnlyCompactions {
fmt.Fprintln(&buf, " disable_elision_only_compactions=true")
}
if o.private.disableLazyCombinedIteration {
// This option is only encoded if true, because we do not want it to
// appear in production serialized Options files, since it's a
// testing-only option. It's only serialized when true, which still
// ensures that the metamorphic tests may propagate it to subprocesses.
fmt.Fprintln(&buf, " disable_lazy_combined_iteration=true")
}
@@ -1078,6 +1212,7 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, "[Level \"%d\"]\n", i)
fmt.Fprintf(&buf, " block_restart_interval=%d\n", l.BlockRestartInterval)
fmt.Fprintf(&buf, " block_size=%d\n", l.BlockSize)
fmt.Fprintf(&buf, " block_size_threshold=%d\n", l.BlockSizeThreshold)
fmt.Fprintf(&buf, " compression=%s\n", l.Compression)
fmt.Fprintf(&buf, " filter_policy=%s\n", filterPolicyName(l.FilterPolicy))
fmt.Fprintf(&buf, " filter_type=%s\n", l.FilterType)
@@ -1212,10 +1347,20 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
// NB: This is a deprecated serialization of the
// `flush_delay_delete_range`.
o.FlushDelayDeleteRange, err = time.ParseDuration(value)
case "disable_wal":
o.DisableWAL, err = strconv.ParseBool(value)
case "disable_delete_only_compactions":
o.private.disableDeleteOnlyCompactions, err = strconv.ParseBool(value)
case "disable_elision_only_compactions":
o.private.disableElisionOnlyCompactions, err = strconv.ParseBool(value)
case "disable_ingest_as_flushable":
var v bool
v, err = strconv.ParseBool(value)
if err == nil {
o.Experimental.DisableIngestAsFlushable = func() bool { return v }
}
case "disable_lazy_combined_iteration":
o.private.disableLazyCombinedIteration, err = strconv.ParseBool(value)
case "disable_wal":
o.DisableWAL, err = strconv.ParseBool(value)
case "flush_delay_delete_range":
o.FlushDelayDeleteRange, err = time.ParseDuration(value)
case "flush_delay_range_key":
@@ -1346,6 +1491,8 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
l.BlockRestartInterval, err = strconv.Atoi(value)
case "block_size":
l.BlockSize, err = strconv.Atoi(value)
case "block_size_threshold":
l.BlockSizeThreshold, err = strconv.Atoi(value)
case "compression":
switch value {
case "Default":
@@ -1471,6 +1618,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions {
if o.Merger != nil {
readerOpts.MergerName = o.Merger.Name
}
readerOpts.LoggerAndTracer = o.LoggerAndTracer
}
return readerOpts
}
@@ -1489,6 +1637,10 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab
writerOpts.TablePropertyCollectors = o.TablePropertyCollectors
writerOpts.BlockPropertyCollectors = o.BlockPropertyCollectors
}
if format >= sstable.TableFormatPebblev3 {
writerOpts.ShortAttributeExtractor = o.Experimental.ShortAttributeExtractor
writerOpts.RequiredInPlaceValueBound = o.Experimental.RequiredInPlaceValueBound
}
levelOpts := o.Level(level)
writerOpts.BlockRestartInterval = levelOpts.BlockRestartInterval
writerOpts.BlockSize = levelOpts.BlockSize

72
vendor/github.com/cockroachdb/pebble/range_keys.go generated vendored
View File

@@ -17,7 +17,7 @@ import (
func (i *Iterator) constructRangeKeyIter() {
i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
&i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
&i.hasPrefix, &i.prefixOrFullSeekKey)
&i.hasPrefix, &i.prefixOrFullSeekKey, true /* onlySets */, &i.rangeKey.rangeKeyBuffers.internal)
// If there's an indexed batch with range keys, include it.
if i.batch != nil {
@@ -80,7 +80,7 @@ func (i *Iterator) constructRangeKeyIter() {
li := i.rangeKey.iterConfig.NewLevelIter()
spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
li.Init(spanIterOpts, i.cmp, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
manifest.Level(level), i.opts.logger, manifest.KeyTypeRange)
manifest.Level(level), manifest.KeyTypeRange)
i.rangeKey.iterConfig.AddLevel(li)
}
}
@@ -266,24 +266,24 @@ func (m *rangeKeyMasking) SpanChanged(s *keyspan.Span) {
// _t_, and there exists a span with suffix _r_ covering a point key with suffix
// _p_, and
//
// _t_ ≤ _r_ < _p_
// _t_ ≤ _r_ < _p_
//
// then the point key is elided. Consider the following rendering, where using
// integer suffixes with higher integers sort before suffixes with lower
// integers, (for example @7 ≤ @6 < @5):
//
// ^
// @9 | •―――――――――――――――○ [e,m)@9
// s 8 | • l@8
// u 7 |------------------------------------ @7 RangeKeyMasking.Suffix
// f 6 | [h,q)@6 •―――――――――――――――――○ (threshold)
// f 5 | • h@5
// f 4 | • n@4
// i 3 | •―――――――――――○ [f,l)@3
// x 2 | • b@2
// 1 |
// 0 |___________________________________
// a b c d e f g h i j k l m n o p q
// ^
// @9 | •―――――――――――――――○ [e,m)@9
// s 8 | • l@8
// u 7 |------------------------------------ @7 RangeKeyMasking.Suffix
// f 6 | [h,q)@6 •―――――――――――――――――○ (threshold)
// f 5 | • h@5
// f 4 | • n@4
// i 3 | •―――――――――――○ [f,l)@3
// x 2 | • b@2
// 1 |
// 0 |___________________________________
// a b c d e f g h i j k l m n o p q
//
// An iterator scanning the entire keyspace with the masking threshold set to @7
// will observe point keys b@2 and l@8. The span keys [h,q)@6 and [f,l)@3 serve
@@ -297,6 +297,7 @@ func (m *rangeKeyMasking) SpanChanged(s *keyspan.Span) {
// Invariant: The userKey is within the user key bounds of the span most
// recently provided to `SpanChanged`.
func (m *rangeKeyMasking) SkipPoint(userKey []byte) bool {
m.parent.stats.RangeKeyStats.ContainedPoints++
if m.maskSpan == nil {
// No range key is currently acting as a mask, so don't skip.
return false
@@ -309,7 +310,11 @@ func (m *rangeKeyMasking) SkipPoint(userKey []byte) bool {
// the InterleavingIter). Skip the point key if the range key's suffix is
// greater than the point key's suffix.
pointSuffix := userKey[m.split(userKey):]
return len(pointSuffix) > 0 && m.cmp(m.maskActiveSuffix, pointSuffix) < 0
if len(pointSuffix) > 0 && m.cmp(m.maskActiveSuffix, pointSuffix) < 0 {
m.parent.stats.RangeKeyStats.SkippedPoints++
return true
}
return false
}
// The iteratorRangeKeyState type implements the sstable package's
@@ -322,9 +327,9 @@ func (m *rangeKeyMasking) SkipPoint(userKey []byte) bool {
// Consider the range key [a,m)@10, and an iterator positioned just before the
// below block, bounded by index separators `c` and `z`:
//
// c z
// x | c@9 c@5 c@1 d@7 e@4 y@4 | ...
// iter pos
// c z
// x | c@9 c@5 c@1 d@7 e@4 y@4 | ...
// iter pos
//
// The next block cannot be skipped, despite the range key suffix @10 is greater
// than all the block's keys' suffixes, because it contains a key (y@4) outside
@@ -496,24 +501,6 @@ func (i *lazyCombinedIter) initCombinedIteration(
}
}
if i.parent.hasPrefix {
si := i.parent.comparer.Split(seekKey)
if i.parent.cmp(seekKey[:si], i.parent.prefixOrFullSeekKey) > 0 {
// The earliest possible range key has a start key with a prefix
// greater than the current iteration prefix. There's no need to
// switch to combined iteration, because there are not any range
// keys within the bounds of the prefix. Additionally, using a seek
// key that is outside the scope of the prefix can violate
// invariants within the range key iterator stack. Optimizations
// that exit early due to exhausting the prefix may result in
// `seekKey` being larger than the next range key's start key.
//
// See the testdata/rangekeys test case associated with #1893.
i.combinedIterState = combinedIterState{initialized: false}
return pointKey, pointValue
}
}
// An operation on the point iterator observed a file containing range keys,
// so we must switch to combined interleaving iteration. First, construct
// the range key iterator stack. It must not exist, otherwise we'd already
@@ -628,6 +615,17 @@ func (i *lazyCombinedIter) Next() (*InternalKey, base.LazyValue) {
return k, v
}
func (i *lazyCombinedIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
if i.combinedIterState.initialized {
return i.parent.rangeKey.iiter.NextPrefix(succKey)
}
k, v := i.pointIter.NextPrefix(succKey)
if i.combinedIterState.triggered {
return i.initCombinedIteration(+1, k, v, nil)
}
return k, v
}
func (i *lazyCombinedIter) Prev() (*InternalKey, base.LazyValue) {
if i.combinedIterState.initialized {
return i.parent.rangeKey.iiter.Prev()

7
vendor/github.com/cockroachdb/pebble/read_state.go generated vendored
View File

@@ -40,7 +40,7 @@ func (s *readState) unref() {
}
s.current.Unref()
for _, mem := range s.memtables {
mem.readerUnref()
mem.readerUnref(true)
}
// The last reference to the readState was released. Check to see if there
@@ -58,7 +58,7 @@ func (s *readState) unrefLocked() {
}
s.current.UnrefLocked()
for _, mem := range s.memtables {
mem.readerUnref()
mem.readerUnrefLocked(true)
}
// NB: Unlike readState.unref(), we don't attempt to cleanup newly obsolete
@@ -78,8 +78,7 @@ func (d *DB) loadReadState() *readState {
// updateReadStateLocked creates a new readState from the current version and
// list of memtables. Requires DB.mu is held. If checker is not nil, it is
// called after installing the new readState. If atomicFunc is not nil, it is
// executed atomically with the transition to the new read state.
// called after installing the new readState.
func (d *DB) updateReadStateLocked(checker func(*DB) error) {
s := &readState{
db: d,

67
vendor/github.com/cockroachdb/pebble/record/log_writer.go generated vendored
View File

@@ -133,7 +133,8 @@ func (q *syncQueue) load() (head, tail, realLength uint32) {
return head, tail, realLength
}
func (q *syncQueue) pop(head, tail uint32, err error) error {
// REQUIRES: queueSemChan is non-nil.
func (q *syncQueue) pop(head, tail uint32, err error, queueSemChan chan struct{}) error {
if tail == head {
// Queue is empty.
return nil
@@ -153,6 +154,10 @@ func (q *syncQueue) pop(head, tail uint32, err error) error {
// will try to enqueue before we've "freed" space in the queue.
atomic.AddUint64(&q.headTail, 1)
wg.Done()
// Is always non-nil in production.
if queueSemChan != nil {
<-queueSemChan
}
}
return nil
@@ -291,12 +296,20 @@ type LogWriter struct {
// used for min-sync-interval. In normal operation this points to
// time.AfterFunc.
afterFunc func(d time.Duration, f func()) syncTimer
// See the comment for LogWriterConfig.QueueSemChan.
queueSemChan chan struct{}
}
// LogWriterConfig is a struct used for configuring new LogWriters
type LogWriterConfig struct {
WALMinSyncInterval durationFunc
WALFsyncLatency prometheus.Histogram
// QueueSemChan is an optional channel to pop from when popping from
// LogWriter.flusher.syncQueue. It functions as a semaphore that prevents
// the syncQueue from overflowing (which will cause a panic). All production
// code ensures this is non-nil.
QueueSemChan chan struct{}
}
// CapAllocatedBlocks is the maximum number of blocks allocated by the
@@ -319,6 +332,7 @@ func NewLogWriter(w io.Writer, logNum base.FileNum, logWriterConfig LogWriterCon
afterFunc: func(d time.Duration, f func()) syncTimer {
return time.AfterFunc(d, f)
},
queueSemChan: logWriterConfig.QueueSemChan,
}
r.free.cond.L = &r.free.Mutex
r.free.blocks = make([]*block, 0, CapAllocatedBlocks)
@@ -441,7 +455,7 @@ func (w *LogWriter) flushLoop(context.Context) {
// If flusher has an error, we propagate it to waiters. Note in spite of
// error we consume the pending list above to free blocks for writers.
if f.err != nil {
f.syncQ.pop(head, tail, f.err)
f.syncQ.pop(head, tail, f.err, w.queueSemChan)
// Update the idleStartTime if work could not be done, so that we don't
// include the duration we tried to do work as idle. We don't bother
// with the rest of the accounting, which means we will undercount.
@@ -518,7 +532,7 @@ func (w *LogWriter) flushPending(
syncLatency, err = w.syncWithLatency()
}
f := &w.flusher
if popErr := f.syncQ.pop(head, tail, err); popErr != nil {
if popErr := f.syncQ.pop(head, tail, err, w.queueSemChan); popErr != nil {
return synced, syncLatency, bytesWritten, popErr
}
}
@@ -548,7 +562,7 @@ func (w *LogWriter) flushBlock(b *block) error {
// queueBlock queues the current block for writing to the underlying writer,
// allocates a new block and reserves space for the next header.
func (w *LogWriter) queueBlock() {
func (w *LogWriter) queueBlock() (waitDuration time.Duration) {
// Allocate a new block, blocking until one is available. We do this first
// because w.block is protected by w.flusher.Mutex.
w.free.Lock()
@@ -557,9 +571,11 @@ func (w *LogWriter) queueBlock() {
w.free.allocated++
w.free.blocks = append(w.free.blocks, &block{})
} else {
now := time.Now()
for len(w.free.blocks) == 0 {
w.free.cond.Wait()
}
waitDuration = time.Since(now)
}
}
nextBlock := w.free.blocks[len(w.free.blocks)-1]
@@ -575,6 +591,28 @@ func (w *LogWriter) queueBlock() {
f.Unlock()
w.blockNum++
return waitDuration
}
// ReserveAllFreeBlocksForTesting is used to only for testing.
func (w *LogWriter) ReserveAllFreeBlocksForTesting() (releaseFunc func()) {
w.free.Lock()
defer w.free.Unlock()
free := w.free.blocks
w.free.blocks = nil
return func() {
w.free.Lock()
defer w.free.Unlock()
// It is possible that someone has pushed a free block and w.free.blocks
// is no longer nil. That is harmless. Also, the waiter loops on the
// condition len(w.free.blocks) == 0, so to actually unblock it we need to
// give it a free block.
if len(free) == 0 {
free = append(free, &block{})
}
w.free.blocks = free
w.free.cond.Broadcast()
}
}
// Close flushes and syncs any unwritten data and closes the writer.
@@ -626,7 +664,8 @@ func (w *LogWriter) Close() error {
// of the record.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) WriteRecord(p []byte) (int64, error) {
return w.SyncRecord(p, nil, nil)
logSize, _, err := w.SyncRecord(p, nil, nil)
return logSize, err
}
// SyncRecord writes a complete record. If wg!= nil the record will be
@@ -634,9 +673,11 @@ func (w *LogWriter) WriteRecord(p []byte) (int64, error) {
// the wait group upon completion. Returns the offset just past the end of the
// record.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64, error) {
func (w *LogWriter) SyncRecord(
p []byte, wg *sync.WaitGroup, err *error,
) (logSize int64, waitDuration time.Duration, err2 error) {
if w.err != nil {
return -1, w.err
return -1, 0, w.err
}
// The `i == 0` condition ensures we handle empty records. Such records can
@@ -644,7 +685,9 @@ func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64,
// MANIFEST is currently written using Writer, it is good to support the same
// semantics with LogWriter.
for i := 0; i == 0 || len(p) > 0; i++ {
p = w.emitFragment(i, p)
var wd time.Duration
p, wd = w.emitFragment(i, p)
waitDuration += wd
}
if wg != nil {
@@ -663,7 +706,7 @@ func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64,
// race with our read. That's ok because the only error we could be seeing is
// one to syncing for which the caller can receive notification of by passing
// in a non-nil err argument.
return offset, nil
return offset, waitDuration, nil
}
// Size returns the current size of the file.
@@ -684,7 +727,7 @@ func (w *LogWriter) emitEOFTrailer() {
atomic.StoreInt32(&b.written, i+int32(recyclableHeaderSize))
}
func (w *LogWriter) emitFragment(n int, p []byte) []byte {
func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDuration time.Duration) {
b := w.block
i := b.written
first := n == 0
@@ -718,9 +761,9 @@ func (w *LogWriter) emitFragment(n int, p []byte) []byte {
for i := b.written; i < blockSize; i++ {
b.buf[i] = 0
}
w.queueBlock()
waitDuration = w.queueBlock()
}
return p[r:]
return p[r:], waitDuration
}
// Metrics must be called after Close. The callee will no longer modify the

14
vendor/github.com/cockroachdb/pebble/record/record.go generated vendored
View File

@@ -19,6 +19,7 @@
// Neither Readers or Writers are safe to use concurrently.
//
// Example code:
//
// func read(r io.Reader) ([]string, error) {
// var ss []string
// records := record.NewReader(r)
@@ -65,9 +66,9 @@
// A record maps to one or more chunks. There are two chunk formats: legacy and
// recyclable. The legacy chunk format:
//
// +----------+-----------+-----------+--- ... ---+
// | CRC (4B) | Size (2B) | Type (1B) | Payload |
// +----------+-----------+-----------+--- ... ---+
// +----------+-----------+-----------+--- ... ---+
// | CRC (4B) | Size (2B) | Type (1B) | Payload |
// +----------+-----------+-----------+--- ... ---+
//
// CRC is computed over the type and payload
// Size is the length of the payload in bytes
@@ -84,9 +85,9 @@
// metadata. Additionally, recycling log files is a prequisite for using direct
// IO with log writing. The recyclyable format is:
//
// +----------+-----------+-----------+----------------+--- ... ---+
// | CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload |
// +----------+-----------+-----------+----------------+--- ... ---+
// +----------+-----------+-----------+----------------+--- ... ---+
// | CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload |
// +----------+-----------+-----------+----------------+--- ... ---+
//
// Recyclable chunks are distinguished from legacy chunks by the addition of 4
// extra "recyclable" chunk types that map directly to the legacy chunk types
@@ -247,6 +248,7 @@ func (r *Reader) nextChunk(wantFirst bool) error {
r.begin = r.end + headerSize
r.end = r.begin + int(length)
if r.end > r.n {
// The chunk straddles a 32KB boundary (or the end of file).
if r.recovering {
r.recover()
continue

82
vendor/github.com/cockroachdb/pebble/record/rotation.go generated vendored Normal file
View File

@@ -0,0 +1,82 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package record
// RotationHelper is a type used to inform the decision of rotating a record log
// file.
//
// The assumption is that multiple records can be coalesced into a single record
// (called a snapshot). Starting a new file, where the first record is a
// snapshot of the current state is referred to as "rotating" the log.
//
// Normally we rotate files when a certain file size is reached. But in certain
// cases (e.g. contents become very large), this can result in too frequent
// rotation. This helper contains logic to impose extra conditions on the
// rotation.
//
// The rotation helper uses "size" as a unit-less estimation that is correlated
// with the on-disk size of a record or snapshot.
type RotationHelper struct {
// lastSnapshotSize is the size of the last snapshot.
lastSnapshotSize int64
// sizeSinceLastSnapshot is the sum of sizes of records applied since the last
// snapshot.
sizeSinceLastSnapshot int64
lastRecordSize int64
}
// AddRecord makes the rotation helper aware of a new record.
func (rh *RotationHelper) AddRecord(recordSize int64) {
rh.sizeSinceLastSnapshot += recordSize
rh.lastRecordSize = recordSize
}
// ShouldRotate returns whether we should start a new log file (with a snapshot).
// Does not need to be called if other rotation factors (log file size) are not
// satisfied.
func (rh *RotationHelper) ShouldRotate(nextSnapshotSize int64) bool {
// The primary goal is to ensure that when reopening a log file, the number of
// edits that need to be replayed on top of the snapshot is "sane" while
// keeping the rotation frequency as low as possible.
//
// For the purposes of this description, we assume that the log is mainly
// storing a collection of "entries", with edits adding or removing entries.
// Consider the following cases:
//
// - The number of live entries is roughly stable: after writing the snapshot
// (with S entries), we require that there be enough edits such that the
// cumulative number of entries in those edits, E, be greater than S. This
// will ensure that at most 50% of data written out is due to rotation.
//
// - The number of live entries K in the DB is shrinking drastically, say from
// S to S/10: After this shrinking, E = 0.9S, and so if we used the previous
// snapshot entry count, S, as the threshold that needs to be exceeded, we
// will further delay the snapshot writing. Which means on reopen we will
// need to replay 0.9S edits to get to a version with 0.1S entries. It would
// be better to create a new snapshot when E exceeds the number of entries in
// the current version.
//
// - The number of live entries L in the DB is growing; say the last snapshot
// had S entries, and now we have 10S entries, so E = 9S. If we required
// that E is at least the current number of entries, we would further delay
// writing a new snapshot (which is not desirable).
//
// The logic below uses the min of the last snapshot size count and the size
// count in the current version.
return rh.sizeSinceLastSnapshot > rh.lastSnapshotSize || rh.sizeSinceLastSnapshot > nextSnapshotSize
}
// Rotate makes the rotation helper aware that we are rotating to a new snapshot
// (to which we will apply the latest edit).
func (rh *RotationHelper) Rotate(snapshotSize int64) {
rh.lastSnapshotSize = snapshotSize
rh.sizeSinceLastSnapshot = rh.lastRecordSize
}
// DebugInfo returns the last snapshot size and size of the edits since the last
// snapshot; used for testing and debugging.
func (rh *RotationHelper) DebugInfo() (lastSnapshotSize int64, sizeSinceLastSnapshot int64) {
return rh.lastSnapshotSize, rh.sizeSinceLastSnapshot
}

925
vendor/github.com/cockroachdb/pebble/scan_internal.go generated vendored Normal file
View File

@@ -0,0 +1,925 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"context"
"fmt"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/objstorage"
)
const (
// In skip-shared iteration mode, keys in levels sharedLevelsStart and greater
// (i.e. lower in the LSM) are skipped.
sharedLevelsStart = 5
)
// ErrInvalidSkipSharedIteration is returned by ScanInternal if it was called
// with a shared file visitor function, and a file in a shareable level (i.e.
// level >= sharedLevelsStart) was found to not be in shared storage according
// to objstorage.Provider.
var ErrInvalidSkipSharedIteration = errors.New("pebble: cannot use skip-shared iteration due to non-shared files in lower levels")
// SharedSSTMeta represents an sstable on shared storage that can be ingested
// by another pebble instance. This struct must contain all fields that are
// required for a Pebble instance to ingest a foreign sstable on shared storage,
// including constructing any relevant objstorage.Provider / sharedobjcat.Catalog
// data structures, as well as creating virtual FileMetadatas.
//
// Note that the Pebble instance creating and returning a SharedSSTMeta might
// not be the one that created the underlying sstable on shared storage to begin
// with; it's possible for a Pebble instance to reshare an sstable that was
// shared to it.
type SharedSSTMeta struct {
// Backing is the shared object underlying this SST. Can be attached to an
// objstorage.Provider.
Backing objstorage.SharedObjectBackingHandle
// Smallest and Largest internal keys for the overall bounds. The kind and
// SeqNum of these will reflect what is physically present on the source Pebble
// instance's view of the sstable; it's up to the ingesting instance to set the
// sequence number in the trailer to match the read-time sequence numbers
// reserved for the level this SST is being ingested into. The Kind is expected
// to remain unchanged by the ingesting instance.
//
// Note that these bounds could be narrower than the bounds of the underlying
// sstable; ScanInternal is expected to truncate sstable bounds to the user key
// bounds passed into that method.
Smallest, Largest InternalKey
// SmallestRangeKey and LargestRangeKey are internal keys that denote the
// range key bounds of this sstable. Must lie within [Smallest, Largest].
SmallestRangeKey, LargestRangeKey InternalKey
// SmallestPointKey and LargestPointKey are internal keys that denote the
// point key bounds of this sstable. Must lie within [Smallest, Largest].
SmallestPointKey, LargestPointKey InternalKey
// Level denotes the level at which this file was present at read time.
// For files visited by ScanInternal, this value will only be 5 or 6.
Level uint8
// Size contains an estimate of the size of this sstable.
Size uint64
// fileNum at time of creation in the creator instance. Only used for
// debugging/tests.
fileNum base.FileNum
}
func (s *SharedSSTMeta) cloneFromFileMeta(f *fileMetadata) {
*s = SharedSSTMeta{
Smallest: f.Smallest.Clone(),
Largest: f.Largest.Clone(),
SmallestRangeKey: f.SmallestRangeKey.Clone(),
LargestRangeKey: f.LargestRangeKey.Clone(),
SmallestPointKey: f.SmallestPointKey.Clone(),
LargestPointKey: f.LargestPointKey.Clone(),
Size: f.Size,
fileNum: f.FileNum,
}
}
type pcIterPos int
const (
pcIterPosCur pcIterPos = iota
pcIterPosNext
)
// pointCollapsingIterator is an internalIterator that collapses point keys and
// returns at most one point internal key for each user key. Merges are merged,
// sets are emitted as-is, and SingleDeletes are collapsed with the next point
// key, however we don't guarantee that we generate and return a SetWithDel if
// a set shadows a del. Point keys deleted by rangedels are also considered shadowed and not
// exposed.
//
// TODO(bilal): Implement the unimplemented internalIterator methods below.
// Currently this iterator only supports the forward iteration case necessary
// for scanInternal, however foreign sstable iterators will also need to use this
// or a simplified version of this.
type pointCollapsingIterator struct {
iter keyspan.InterleavingIter
pos pcIterPos
comparer *base.Comparer
merge base.Merge
err error
seqNum uint64
// The current position of `iter`. Could be backed by savedKey (i.e. iterKey ==
// &savedKey) or could be owned by `iter`. findNextEntry and similar methods
// are expected to save the current value of this to savedKey if they're
// iterating away from the current key but still need to retain it. See
// comments in findNextEntry on how this field is used.
//
// At the end of a positioning call:
// - if pos == pcIterPosNext, iterKey is pointing to the next user key owned
// by `iter` while savedKey is holding a copy to our current key.
// - If pos == pcIterPosCur, iterKey is pointing to either a savedKey-backed
// copy of the current key, or an `iter`-owned current key in which case
// the value of savedKey is undefined.
iterKey *InternalKey
savedKey InternalKey
// Value at the current iterator position, at iterKey.
value base.LazyValue
// Used for Merge keys only.
valueMerger ValueMerger
valueBuf []byte
}
// SeekPrefixGE implements the InternalIterator interface.
func (p *pointCollapsingIterator) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// SeekGE implements the InternalIterator interface.
func (p *pointCollapsingIterator) SeekGE(
key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
p.resetKey()
p.iterKey, p.value = p.iter.SeekGE(key, flags)
p.pos = pcIterPosCur
if p.iterKey == nil {
return nil, base.LazyValue{}
}
return p.findNextEntry()
}
// SeekLT implements the InternalIterator interface.
func (p *pointCollapsingIterator) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
func (p *pointCollapsingIterator) resetKey() {
p.savedKey.UserKey = p.savedKey.UserKey[:0]
p.savedKey.Trailer = 0
p.valueMerger = nil
p.valueBuf = p.valueBuf[:0]
p.iterKey = nil
p.pos = pcIterPosCur
}
// findNextEntry is called to return the next key. p.iter must be positioned at
// the start of the first user key we are interested in.
func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyValue) {
finishAndReturnMerge := func() (*base.InternalKey, base.LazyValue) {
value, closer, err := p.valueMerger.Finish(true /* includesBase */)
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
p.valueBuf = append(p.valueBuf[:0], value...)
if closer != nil {
_ = closer.Close()
}
p.valueMerger = nil
newValue := base.MakeInPlaceValue(value)
return &p.savedKey, newValue
}
// saveKey sets p.iterKey (if not-nil) to &p.savedKey. We can use this equality
// as a proxy to determine if we're at the first internal key for a user key.
p.saveKey()
for p.iterKey != nil {
// NB: p.savedKey is either the current key (iff p.iterKey == &p.savedKey),
// or the previous key.
if p.iterKey != &p.savedKey && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
if p.valueMerger != nil {
if p.savedKey.Kind() != InternalKeyKindMerge {
panic(fmt.Sprintf("expected key %s to have MERGE kind", p.iterKey))
}
p.pos = pcIterPosNext
return finishAndReturnMerge()
}
p.saveKey()
continue
}
if s := p.iter.Span(); s != nil && s.CoversAt(p.seqNum, p.iterKey.SeqNum()) {
// All future keys for this user key must be deleted.
if p.valueMerger != nil {
return finishAndReturnMerge()
} else if p.savedKey.Kind() == InternalKeyKindSingleDelete {
panic("cannot process singledel key in point collapsing iterator")
}
// Fast forward to the next user key.
p.saveKey()
p.iterKey, p.value = p.iter.Next()
for p.iterKey != nil && p.savedKey.SeqNum() >= p.iterKey.SeqNum() && p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
p.iterKey, p.value = p.iter.Next()
}
continue
}
switch p.savedKey.Kind() {
case InternalKeyKindSet, InternalKeyKindDelete, InternalKeyKindSetWithDelete:
p.saveKey()
// Note that we return SETs directly, even if they would otherwise get
// compacted into a Del to turn into a SetWithDelete. This is a fast
// path optimization that can break SINGLEDEL determinism. To lead to
// consistent SINGLEDEL behaviour, this iterator should *not* be used for
// a keyspace where SINGLEDELs could be in use. If this iterator observes
// a SINGLEDEL as the first internal key for a user key, it will panic.
//
// As p.value is a lazy value owned by the child iterator, we can thread
// it through without loading it into p.valueBuf.
//
// TODO(bilal): We can even avoid saving the key in this fast path if
// we are in a block where setHasSamePrefix = false in a v3 sstable,
// guaranteeing that there's only one internal key for each user key.
// Thread this logic through the sstable iterators and/or consider
// collapsing (ha) this logic into the sstable iterators that are aware
// of blocks and can determine user key changes without doing key saves
// or comparisons.
p.pos = pcIterPosCur
return p.iterKey, p.value
case InternalKeyKindSingleDelete:
// Panic, as this iterator is not expected to observe single deletes.
panic("cannot process singledel key in point collapsing iterator")
case InternalKeyKindMerge:
if p.valueMerger == nil {
// Set up merger. This is the first Merge key encountered.
value, callerOwned, err := p.value.Value(p.valueBuf[:0])
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
if !callerOwned {
p.valueBuf = append(p.valueBuf[:0], value...)
} else {
p.valueBuf = value
}
p.valueMerger, err = p.merge(p.iterKey.UserKey, p.valueBuf)
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
p.saveKey()
p.iterKey, p.value = p.iter.Next()
continue
}
switch p.iterKey.Kind() {
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// Merge into key.
value, callerOwned, err := p.value.Value(p.valueBuf[:0])
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
if !callerOwned {
p.valueBuf = append(p.valueBuf[:0], value...)
} else {
p.valueBuf = value
}
if err := p.valueMerger.MergeOlder(value); err != nil {
p.err = err
return nil, base.LazyValue{}
}
}
if p.iterKey.Kind() != InternalKeyKindMerge {
p.savedKey.SetKind(p.iterKey.Kind())
p.pos = pcIterPosCur
return finishAndReturnMerge()
}
p.iterKey, p.value = p.iter.Next()
case InternalKeyKindRangeDelete:
// These are interleaved by the interleaving iterator ahead of all points.
// We should pass them as-is, but also account for any points ahead of
// them.
p.pos = pcIterPosCur
return p.iterKey, p.value
default:
panic(fmt.Sprintf("unexpected kind: %d", p.iterKey.Kind()))
}
}
if p.valueMerger != nil {
p.pos = pcIterPosNext
return finishAndReturnMerge()
}
p.resetKey()
return nil, base.LazyValue{}
}
// First implements the InternalIterator interface.
func (p *pointCollapsingIterator) First() (*base.InternalKey, base.LazyValue) {
p.resetKey()
p.iterKey, p.value = p.iter.First()
p.pos = pcIterPosCur
if p.iterKey == nil {
return nil, base.LazyValue{}
}
return p.findNextEntry()
}
// Last implements the InternalIterator interface.
func (p *pointCollapsingIterator) Last() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
func (p *pointCollapsingIterator) saveKey() {
if p.iterKey == nil {
p.savedKey = InternalKey{UserKey: p.savedKey.UserKey[:0]}
return
} else if p.iterKey == &p.savedKey {
// Do nothing.
return
}
p.savedKey.CopyFrom(*p.iterKey)
p.iterKey = &p.savedKey
}
// Next implements the InternalIterator interface.
func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) {
switch p.pos {
case pcIterPosCur:
p.saveKey()
if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete {
// Step over the interleaved range delete and process the very next
// internal key, even if it's at the same user key. This is because a
// point for that user key has not been returned yet.
p.iterKey, p.value = p.iter.Next()
break
}
// Fast forward to the next user key. p.iterKey stayed at p.savedKey for
// this loop.
key, val := p.iter.Next()
// p.iterKey.SeqNum() >= key.SeqNum() is an optimization that allows us to
// use p.iterKey.SeqNum() < key.SeqNum() as a sign that the user key has
// changed, without needing to do the full key comparison.
for p.iterKey != nil && key != nil && p.iterKey.SeqNum() >= key.SeqNum() &&
p.comparer.Equal(p.iterKey.UserKey, key.UserKey) {
key, val = p.iter.Next()
}
if key == nil {
// There are no keys to return.
p.resetKey()
return nil, base.LazyValue{}
}
p.iterKey, p.value = key, val
case pcIterPosNext:
p.pos = pcIterPosCur
}
if p.iterKey == nil {
p.resetKey()
return nil, base.LazyValue{}
}
return p.findNextEntry()
}
// NextPrefix implements the InternalIterator interface.
func (p *pointCollapsingIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Prev implements the InternalIterator interface.
func (p *pointCollapsingIterator) Prev() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Error implements the InternalIterator interface.
func (p *pointCollapsingIterator) Error() error {
if p.err != nil {
return p.err
}
return p.iter.Error()
}
// Close implements the InternalIterator interface.
func (p *pointCollapsingIterator) Close() error {
return p.iter.Close()
}
// SetBounds implements the InternalIterator interface.
func (p *pointCollapsingIterator) SetBounds(lower, upper []byte) {
p.resetKey()
p.iter.SetBounds(lower, upper)
}
// String implements the InternalIterator interface.
func (p *pointCollapsingIterator) String() string {
return p.iter.String()
}
var _ internalIterator = &pointCollapsingIterator{}
// scanInternalIterator is an iterator that returns all internal keys, including
// tombstones. For instance, an InternalKeyKindDelete would be returned as an
// InternalKeyKindDelete instead of the iterator skipping over to the next key.
// Internal keys within a user key are collapsed, eg. if there are two SETs, the
// one with the higher sequence is returned. Useful if an external user of Pebble
// needs to observe and rebuild Pebble's history of internal keys, such as in
// node-to-node replication. For use with {db,snapshot}.ScanInternal().
//
// scanInternalIterator is expected to ignore point keys deleted by range
// deletions, and range keys shadowed by a range key unset or delete, however it
// *must* return the range delete as well as the range key unset/delete that did
// the shadowing.
type scanInternalIterator struct {
opts scanInternalOptions
comparer *base.Comparer
merge Merge
iter internalIterator
readState *readState
rangeKey *iteratorRangeKeyState
pointKeyIter pointCollapsingIterator
iterKey *InternalKey
iterValue LazyValue
alloc *iterAlloc
newIters tableNewIters
newIterRangeKey keyspan.TableNewSpanIter
seqNum uint64
// boundsBuf holds two buffers used to store the lower and upper bounds.
// Whenever the InternalIterator's bounds change, the new bounds are copied
// into boundsBuf[boundsBufIdx]. The two bounds share a slice to reduce
// allocations. opts.LowerBound and opts.UpperBound point into this slice.
boundsBuf [2][]byte
boundsBufIdx int
}
// truncateSharedFile truncates a shared file's [Smallest, Largest] fields to
// [lower, upper), potentially opening iterators on the file to find keys within
// the requested bounds. A SharedSSTMeta is produced that is suitable for
// external consumption by other Pebble instances. If shouldSkip is true, this
// file does not contain any keys in [lower, upper) and can be skipped.
//
// TODO(bilal): If opening iterators and doing reads in this method is too
// inefficient, consider producing non-tight file bounds instead.
func (d *DB) truncateSharedFile(
ctx context.Context,
lower, upper []byte,
level int,
file *fileMetadata,
objMeta objstorage.ObjectMetadata,
) (sst *SharedSSTMeta, shouldSkip bool, err error) {
cmp := d.cmp
sst = &SharedSSTMeta{}
sst.cloneFromFileMeta(file)
sst.Level = uint8(level)
sst.Backing, err = d.objProvider.SharedObjectBacking(&objMeta)
if err != nil {
return nil, false, err
}
needsLowerTruncate := cmp(lower, file.Smallest.UserKey) > 0
needsUpperTruncate := cmp(upper, file.Largest.UserKey) < 0 || (cmp(upper, file.Largest.UserKey) == 0 && !file.Largest.IsExclusiveSentinel())
// Fast path: file is entirely within [lower, upper).
if !needsLowerTruncate && !needsUpperTruncate {
return sst, false, nil
}
// We will need to truncate file bounds in at least one direction. Open all
// relevant iterators.
//
// TODO(bilal): Once virtual sstables go in, verify that the constraining of
// bounds to virtual sstable bounds happens below this method, so we aren't
// unintentionally exposing keys we shouldn't be exposing.
iter, rangeDelIter, err := d.newIters(ctx, file, &IterOptions{
LowerBound: lower,
UpperBound: upper,
level: manifest.Level(level),
}, internalIterOpts{})
if err != nil {
return nil, false, err
}
defer iter.Close()
if rangeDelIter != nil {
rangeDelIter = keyspan.Truncate(
cmp, rangeDelIter, lower, upper, nil, nil,
false, /* panicOnPartialOverlap */
)
defer rangeDelIter.Close()
}
rangeKeyIter, err := d.tableNewRangeKeyIter(file, nil /* spanIterOptions */)
if err != nil {
return nil, false, err
}
if rangeKeyIter != nil {
rangeKeyIter = keyspan.Truncate(
cmp, rangeKeyIter, lower, upper, nil, nil,
false, /* panicOnPartialOverlap */
)
defer rangeKeyIter.Close()
}
// Check if we need to truncate on the left side. This means finding a new
// LargestPointKey and LargestRangeKey that is >= lower.
if needsLowerTruncate {
sst.SmallestPointKey.UserKey = sst.SmallestPointKey.UserKey[:0]
sst.SmallestPointKey.Trailer = 0
key, _ := iter.SeekGE(lower, base.SeekGEFlagsNone)
if key != nil {
sst.SmallestPointKey.CopyFrom(*key)
}
if rangeDelIter != nil {
span := rangeDelIter.SeekGE(lower)
if span != nil && (len(sst.SmallestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.SmallestKey(), sst.SmallestPointKey) < 0) {
sst.SmallestPointKey.CopyFrom(span.SmallestKey())
}
}
sst.SmallestRangeKey.UserKey = sst.SmallestRangeKey.UserKey[:0]
sst.SmallestRangeKey.Trailer = 0
if rangeKeyIter != nil {
span := rangeKeyIter.SeekGE(lower)
if span != nil {
sst.SmallestRangeKey.CopyFrom(span.SmallestKey())
}
}
}
// Check if we need to truncate on the right side. This means finding a new
// LargestPointKey and LargestRangeKey that is < upper.
if needsUpperTruncate {
sst.LargestPointKey.UserKey = sst.LargestPointKey.UserKey[:0]
sst.LargestPointKey.Trailer = 0
key, _ := iter.SeekLT(upper, base.SeekLTFlagsNone)
if key != nil {
sst.LargestPointKey.CopyFrom(*key)
}
if rangeDelIter != nil {
span := rangeDelIter.SeekLT(upper)
if span != nil && (len(sst.LargestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.LargestKey(), sst.LargestPointKey) > 0) {
sst.LargestPointKey.CopyFrom(span.LargestKey())
}
}
sst.LargestRangeKey.UserKey = sst.LargestRangeKey.UserKey[:0]
sst.LargestRangeKey.Trailer = 0
if rangeKeyIter != nil {
span := rangeKeyIter.SeekLT(upper)
if span != nil {
sst.LargestRangeKey.CopyFrom(span.LargestKey())
}
}
}
// Set overall bounds based on {Smallest,Largest}{Point,Range}Key.
switch {
case len(sst.SmallestRangeKey.UserKey) == 0:
sst.Smallest = sst.SmallestPointKey
case len(sst.SmallestPointKey.UserKey) == 0:
sst.Smallest = sst.SmallestRangeKey
default:
sst.Smallest = sst.SmallestPointKey
if base.InternalCompare(cmp, sst.SmallestRangeKey, sst.SmallestPointKey) < 0 {
sst.Smallest = sst.SmallestRangeKey
}
}
switch {
case len(sst.LargestRangeKey.UserKey) == 0:
sst.Largest = sst.LargestPointKey
case len(sst.LargestPointKey.UserKey) == 0:
sst.Largest = sst.LargestRangeKey
default:
sst.Largest = sst.LargestPointKey
if base.InternalCompare(cmp, sst.LargestRangeKey, sst.LargestPointKey) > 0 {
sst.Largest = sst.LargestRangeKey
}
}
// On rare occasion, a file might overlap with [lower, upper) but not actually
// have any keys within those bounds. Skip such files.
if len(sst.Smallest.UserKey) == 0 {
return nil, true, nil
}
return sst, false, nil
}
func scanInternalImpl(
ctx context.Context,
lower, upper []byte,
iter *scanInternalIterator,
visitPointKey func(key *InternalKey, value LazyValue) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []keyspan.Key) error,
visitSharedFile func(sst *SharedSSTMeta) error,
) error {
if visitSharedFile != nil && (lower == nil || upper == nil) {
panic("lower and upper bounds must be specified in skip-shared iteration mode")
}
// Before starting iteration, check if any files in levels sharedLevelsStart
// and below are *not* shared. Error out if that is the case, as skip-shared
// iteration will not produce a consistent point-in-time view of this range
// of keys. For files that are shared, call visitSharedFile with a truncated
// version of that file.
cmp := iter.comparer.Compare
db := iter.readState.db
provider := db.objProvider
if visitSharedFile != nil {
if provider == nil {
panic("expected non-nil Provider in skip-shared iteration mode")
}
for level := sharedLevelsStart; level < numLevels; level++ {
files := iter.readState.current.Levels[level].Iter()
for f := files.SeekGE(cmp, lower); f != nil && cmp(f.Smallest.UserKey, upper) < 0; f = files.Next() {
var objMeta objstorage.ObjectMetadata
var err error
objMeta, err = provider.Lookup(fileTypeTable, f.FileNum.DiskFileNum())
if err != nil {
return err
}
if !objMeta.IsShared() {
return errors.Wrapf(ErrInvalidSkipSharedIteration, "when processing file %s", objMeta.DiskFileNum)
}
var sst *SharedSSTMeta
var skip bool
sst, skip, err = iter.readState.db.truncateSharedFile(ctx, lower, upper, level, f, objMeta)
if err != nil {
return err
}
if skip {
continue
}
if err = visitSharedFile(sst); err != nil {
return err
}
}
}
}
for valid := iter.seekGE(lower); valid && iter.error() == nil; valid = iter.next() {
key := iter.unsafeKey()
switch key.Kind() {
case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
span := iter.unsafeSpan()
if err := visitRangeKey(span.Start, span.End, span.Keys); err != nil {
return err
}
case InternalKeyKindRangeDelete:
rangeDel := iter.unsafeRangeDel()
if err := visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil {
return err
}
default:
val := iter.lazyValue()
if err := visitPointKey(key, val); err != nil {
return err
}
}
}
return nil
}
// constructPointIter constructs a merging iterator and sets i.iter to it.
func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *iterAlloc) {
// Merging levels and levels from iterAlloc.
mlevels := buf.mlevels[:0]
levels := buf.levels[:0]
// We compute the number of levels needed ahead of time and reallocate a slice if
// the array from the iterAlloc isn't large enough. Doing this allocation once
// should improve the performance.
numMergingLevels := len(memtables)
numLevelIters := 0
current := i.readState.current
numMergingLevels += len(current.L0SublevelFiles)
numLevelIters += len(current.L0SublevelFiles)
for level := 1; level < len(current.Levels); level++ {
if current.Levels[level].Empty() {
continue
}
if i.opts.skipSharedLevels && level >= sharedLevelsStart {
continue
}
numMergingLevels++
numLevelIters++
}
if numMergingLevels > cap(mlevels) {
mlevels = make([]mergingIterLevel, 0, numMergingLevels)
}
if numLevelIters > cap(levels) {
levels = make([]levelIter, 0, numLevelIters)
}
// TODO(bilal): Push these into the iterAlloc buf.
var rangeDelMiter keyspan.MergingIter
rangeDelIters := make([]keyspan.FragmentIterator, 0, numMergingLevels)
rangeDelLevels := make([]keyspan.LevelIter, 0, numLevelIters)
// Next are the memtables.
for j := len(memtables) - 1; j >= 0; j-- {
mem := memtables[j]
mlevels = append(mlevels, mergingIterLevel{
iter: mem.newIter(&i.opts.IterOptions),
})
if rdi := mem.newRangeDelIter(&i.opts.IterOptions); rdi != nil {
rangeDelIters = append(rangeDelIters, rdi)
}
}
// Next are the file levels: L0 sub-levels followed by lower levels.
mlevelsIndex := len(mlevels)
levelsIndex := len(levels)
mlevels = mlevels[:numMergingLevels]
levels = levels[:numLevelIters]
rangeDelLevels = rangeDelLevels[:numLevelIters]
addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) {
li := &levels[levelsIndex]
rli := &rangeDelLevels[levelsIndex]
li.init(
context.Background(), i.opts.IterOptions, i.comparer.Compare, i.comparer.Split, i.newIters, files, level,
internalIterOpts{})
li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext)
mlevels[mlevelsIndex].iter = li
rli.Init(keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters},
i.comparer.Compare, tableNewRangeDelIter(context.Background(), i.newIters), files, level,
manifest.KeyTypePoint)
rangeDelIters = append(rangeDelIters, rli)
levelsIndex++
mlevelsIndex++
}
// Add level iterators for the L0 sublevels, iterating from newest to
// oldest.
for i := len(current.L0SublevelFiles) - 1; i >= 0; i-- {
addLevelIterForFiles(current.L0SublevelFiles[i].Iter(), manifest.L0Sublevel(i))
}
// Add level iterators for the non-empty non-L0 levels.
for level := 1; level < numLevels; level++ {
if current.Levels[level].Empty() {
continue
}
if i.opts.skipSharedLevels && level >= sharedLevelsStart {
continue
}
addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level))
}
buf.merging.init(&i.opts.IterOptions, &InternalIteratorStats{}, i.comparer.Compare, i.comparer.Split, mlevels...)
buf.merging.snapshot = i.seqNum
rangeDelMiter.Init(i.comparer.Compare, keyspan.VisibleTransform(i.seqNum), new(keyspan.MergingBuffers), rangeDelIters...)
i.pointKeyIter = pointCollapsingIterator{
comparer: i.comparer,
merge: i.merge,
seqNum: i.seqNum,
}
i.pointKeyIter.iter.Init(i.comparer, &buf.merging, &rangeDelMiter, nil /* mask */, i.opts.LowerBound, i.opts.UpperBound)
i.iter = &i.pointKeyIter
}
// constructRangeKeyIter constructs the range-key iterator stack, populating
// i.rangeKey.rangeKeyIter with the resulting iterator. This is similar to
// Iterator.constructRangeKeyIter, except it doesn't handle batches and ensures
// iterConfig does *not* elide unsets/deletes.
func (i *scanInternalIterator) constructRangeKeyIter() {
// We want the bounded iter from iterConfig, but not the collapsing of
// RangeKeyUnsets and RangeKeyDels.
i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
nil /* hasPrefix */, nil /* prefix */, false, /* onlySets */
&i.rangeKey.rangeKeyBuffers.internal)
// Next are the flushables: memtables and large batches.
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts.IterOptions); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
}
}
current := i.readState.current
// Next are the file levels: L0 sub-levels followed by lower levels.
//
// Add file-specific iterators for L0 files containing range keys. This is less
// efficient than using levelIters for sublevels of L0 files containing
// range keys, but range keys are expected to be sparse anyway, reducing the
// cost benefit of maintaining a separate L0Sublevels instance for range key
// files and then using it here.
//
// NB: We iterate L0's files in reverse order. They're sorted by
// LargestSeqNum ascending, and we need to add them to the merging iterator
// in LargestSeqNum descending to preserve the merging iterator's invariants
// around Key Trailer order.
iter := current.RangeKeyLevels[0].Iter()
for f := iter.Last(); f != nil; f = iter.Prev() {
spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
spanIter, err := i.newIterRangeKey(f, spanIterOpts)
if err != nil {
i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
continue
}
i.rangeKey.iterConfig.AddLevel(spanIter)
}
// Add level iterators for the non-empty non-L0 levels.
for level := 1; level < len(current.RangeKeyLevels); level++ {
if current.RangeKeyLevels[level].Empty() {
continue
}
if i.opts.skipSharedLevels && level >= sharedLevelsStart {
continue
}
li := i.rangeKey.iterConfig.NewLevelIter()
spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
manifest.Level(level), manifest.KeyTypeRange)
i.rangeKey.iterConfig.AddLevel(li)
}
}
// seekGE seeks this iterator to the first key that's greater than or equal
// to the specified user key.
func (i *scanInternalIterator) seekGE(key []byte) bool {
i.iterKey, i.iterValue = i.iter.SeekGE(key, base.SeekGEFlagsNone)
return i.iterKey != nil
}
// unsafeKey returns the unsafe InternalKey at the current position. The value
// is nil if the iterator is invalid or exhausted.
func (i *scanInternalIterator) unsafeKey() *InternalKey {
return i.iterKey
}
// lazyValue returns a value pointer to the value at the current iterator
// position. Behaviour undefined if unsafeKey() returns a Range key or Rangedel
// kind key.
func (i *scanInternalIterator) lazyValue() LazyValue {
return i.iterValue
}
// unsafeRangeDel returns a range key span. Behaviour undefined if UnsafeKey returns
// a non-rangedel kind.
func (i *scanInternalIterator) unsafeRangeDel() *keyspan.Span {
return i.pointKeyIter.iter.Span()
}
// unsafeSpan returns a range key span. Behaviour undefined if UnsafeKey returns
// a non-rangekey type.
func (i *scanInternalIterator) unsafeSpan() *keyspan.Span {
return i.rangeKey.iiter.Span()
}
// next advances the iterator in the forward direction, and returns the
// iterator's new validity state.
func (i *scanInternalIterator) next() bool {
i.iterKey, i.iterValue = i.iter.Next()
return i.iterKey != nil
}
// error returns an error from the internal iterator, if there's any.
func (i *scanInternalIterator) error() error {
return i.iter.Error()
}
// close closes this iterator, and releases any pooled objects.
func (i *scanInternalIterator) close() error {
if err := i.iter.Close(); err != nil {
return err
}
i.readState.unref()
if i.rangeKey != nil {
i.rangeKey.PrepareForReuse()
*i.rangeKey = iteratorRangeKeyState{
rangeKeyBuffers: i.rangeKey.rangeKeyBuffers,
}
iterRangeKeyStateAllocPool.Put(i.rangeKey)
i.rangeKey = nil
}
if alloc := i.alloc; alloc != nil {
for j := range i.boundsBuf {
if cap(i.boundsBuf[j]) >= maxKeyBufCacheSize {
alloc.boundsBuf[j] = nil
} else {
alloc.boundsBuf[j] = i.boundsBuf[j]
}
}
*alloc = iterAlloc{
keyBuf: alloc.keyBuf[:0],
boundsBuf: alloc.boundsBuf,
prefixOrFullSeekKey: alloc.prefixOrFullSeekKey[:0],
}
iterAllocPool.Put(alloc)
i.alloc = nil
}
return nil
}
func (i *scanInternalIterator) initializeBoundBufs(lower, upper []byte) {
buf := i.boundsBuf[i.boundsBufIdx][:0]
if lower != nil {
buf = append(buf, lower...)
i.opts.LowerBound = buf
} else {
i.opts.LowerBound = nil
}
if upper != nil {
buf = append(buf, upper...)
i.opts.UpperBound = buf[len(buf)-len(upper):]
} else {
i.opts.UpperBound = nil
}
i.boundsBuf[i.boundsBufIdx] = buf
i.boundsBufIdx = 1 - i.boundsBufIdx
}

41
vendor/github.com/cockroachdb/pebble/snapshot.go generated vendored
View File

@@ -5,8 +5,11 @@
package pebble
import (
"context"
"io"
"math"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// Snapshot provides a read-only point-in-time view of the DB state.
@@ -42,10 +45,46 @@ func (s *Snapshot) Get(key []byte) ([]byte, io.Closer, error) {
// return false). The iterator can be positioned via a call to SeekGE,
// SeekLT, First or Last.
func (s *Snapshot) NewIter(o *IterOptions) *Iterator {
return s.NewIterWithContext(context.Background(), o)
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
// tracing.
func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator {
if s.db == nil {
panic(ErrClosed)
}
return s.db.newIterInternal(nil /* batch */, s, o)
return s.db.newIter(ctx, nil /* batch */, s, o)
}
// ScanInternal scans all internal keys within the specified bounds, truncating
// any rangedels and rangekeys to those bounds. For use when an external user
// needs to be aware of all internal keys that make up a key range.
//
// See comment on db.ScanInternal for the behaviour that can be expected of
// point keys deleted by range dels and keys masked by range keys.
func (s *Snapshot) ScanInternal(
ctx context.Context,
lower, upper []byte,
visitPointKey func(key *InternalKey, value LazyValue) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []keyspan.Key) error,
visitSharedFile func(sst *SharedSSTMeta) error,
) error {
if s.db == nil {
panic(ErrClosed)
}
iter := s.db.newInternalIter(s, &scanInternalOptions{
IterOptions: IterOptions{
KeyTypes: IterKeyTypePointsAndRanges,
LowerBound: lower,
UpperBound: upper,
},
skipSharedLevels: visitSharedFile != nil,
})
defer iter.close()
return scanInternalImpl(ctx, lower, upper, iter, visitPointKey, visitRangeDel, visitRangeKey, visitSharedFile)
}
// Close closes the snapshot, releasing its resources. Close must be called.

Some files were not shown because too many files have changed in this diff Show More