aleesa-bot/aleesa-phrases-go
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update dependencies

Browse Source
This commit is contained in:
eleksir
2024-07-24 22:57:56 +03:00
parent ac243bf4fd
commit 00257b8e6c
729 changed files with 47714 additions and 29066 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
vendor/github.com/cockroachdb/pebble/.gitignore generated vendored
View File

@@ -1,6 +1,9 @@
# Github action artifacts.
artifacts
# Profiling artifacts.
cpu.*.prof
heap.prof
mutex.prof
coverprofile.out
# Testing artifacts
meta.*.test

15
vendor/github.com/cockroachdb/pebble/Makefile generated vendored
View File

@@ -4,7 +4,8 @@ GOFLAGS :=
STRESSFLAGS :=
TAGS := invariants
TESTS := .
LATEST_RELEASE := $(shell git fetch origin && git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$$' | sort | tail -1)
PREV_RELEASE := crl-release-23.1
COVER_PROFILE := coverprofile.out
.PHONY: all
all:
@@ -15,6 +16,7 @@ all:
@echo " make stressrace"
@echo " make stressmeta"
@echo " make crossversion-meta"
@echo " make testcoverage"
@echo " make mod-update"
@echo " make generate"
@echo " make generate-test-data"
@@ -25,6 +27,10 @@ override testflags :=
test:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG}
.PHONY: testcoverage
testcoverage:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} -coverprofile ${COVER_PROFILE}
.PHONY: testrace
testrace: testflags += -race -timeout 20m
testrace: test
@@ -57,11 +63,11 @@ stressmeta: stress
.PHONY: crossversion-meta
crossversion-meta:
git checkout ${LATEST_RELEASE}; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \
git checkout ${PREV_RELEASE}; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${PREV_RELEASE}.test'; \
git checkout -; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${PREV_RELEASE},${PREV_RELEASE},${PREV_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
.PHONY: stress-crossversion
stress-crossversion:
@@ -80,6 +86,7 @@ generate-test-data:
${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
${GO} run -tags make_test_find_db ./tool/make_test_find_db.go
${GO} run -tags make_test_sstables ./tool/make_test_sstables.go
${GO} run -tags make_test_remotecat ./tool/make_test_remotecat.go
mod-update:
${GO} get -u

84
vendor/github.com/cockroachdb/pebble/README.md generated vendored
View File

@@ -1,4 +1,4 @@
# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble)
# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) <sup><sub><sub>[Coverage](https://storage.googleapis.com/crl-codecover-public/pebble/index.html)</sub></sub></sup>
#### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
@@ -8,9 +8,8 @@ RocksDB file formats and a few extensions such as range deletion
tombstones, table-level bloom filters, and updates to the MANIFEST
format.
Pebble intentionally does not aspire to include every feature in
RocksDB and is specifically targetting the use case and feature set
needed by CockroachDB:
Pebble intentionally does not aspire to include every feature in RocksDB and
specifically targets the use case and feature set needed by CockroachDB:
* Block-based tables
* Checkpoints
@@ -68,6 +67,17 @@ Pebble offers several improvements over RocksDB:
* Faster commit pipeline that achieves better concurrency.
* Seamless merged iteration of indexed batches. The mutations in the
batch conceptually occupy another memtable level.
* L0 sublevels and flush splitting for concurrent compactions out of L0 and
reduced read-amplification during heavy write load.
* Faster LSM edits in LSMs with large numbers of sstables through use of a
copy-on-write B-tree to hold file metadata.
* Delete-only compactions that drop whole sstables that fall within the bounds
of a range deletion.
* Block-property collectors and filters that enable iterators to skip tables,
index blocks and data blocks that are irrelevant, according to user-defined
properties over key-value pairs.
* Range keys API, allowing KV pairs defined over a range of keyspace with
user-defined semantics and interleaved during iteration.
* Smaller, more approachable code base.
See the [Pebble vs RocksDB: Implementation
@@ -100,10 +110,68 @@ known incompatibilities.
format.
* Plain table format. Pebble does not support the plain table sstable
format.
* SSTable format version 3 and 4. Pebble does not currently support
version 3 and version 4 format sstables. The sstable format version
is controlled by the `BlockBasedTableOptions::format_version`
option. See [#97](https://github.com/cockroachdb/pebble/issues/97).
* SSTable format version 3 and 4. Pebble does not support version 3
and version 4 format sstables. The sstable format version is
controlled by the `BlockBasedTableOptions::format_version` option.
See [#97](https://github.com/cockroachdb/pebble/issues/97).
## Format major versions
Over time Pebble has introduced new physical file formats. Backwards
incompatible changes are made through the introduction of 'format major
versions'. By default, when Pebble opens a database, it defaults to
`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB
6.2.1 (with the caveats described above).
To opt into new formats, a user may set `FormatMajorVersion` on the
[`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options)
supplied to
[`Open`](https://pkg.go.dev/github.com/cockroachdb/pebble#Open), or
upgrade the format major version at runtime using
[`DB.RatchetFormatMajorVersion`](https://pkg.go.dev/github.com/cockroachdb/pebble#DB.RatchetFormatMajorVersion).
Format major version upgrades are permanent; There is no option to
return to an earlier format.
The table below outlines the history of format major versions:
| Name | Value | Migration |
|------------------------------------|-------|------------|
| FormatMostCompatible | 1 | No |
| FormatVersioned | 3 | No |
| FormatSetWithDelete | 4 | No |
| FormatBlockPropertyCollector | 5 | No |
| FormatSplitUserKeysMarked | 6 | Background |
| FormatSplitUserKeysMarkedCompacted | 7 | Blocking |
| FormatRangeKeys | 8 | No |
| FormatMinTableFormatPebblev1 | 9 | No |
| FormatPrePebblev1Marked | 10 | Background |
| FormatSSTableValueBlocks | 12 | No |
| FormatFlushableIngest | 13 | No |
| FormatPrePebblev1MarkedCompacted | 14 | Blocking |
| FormatDeleteSizedAndObsolete | 15 | No |
| FormatVirtualSSTables | 16 | No |
Upgrading to a format major version with 'Background' in the migration
column may trigger background activity to rewrite physical file
formats, typically through compactions. Upgrading to a format major
version with 'Blocking' in the migration column will block until a
migration is complete. The database may continue to serve reads and
writes if upgrading a live database through
`RatchetFormatMajorVersion`, but the method call will not return until
the migration is complete.
For reference, the table below lists the range of supported Pebble format major
versions for CockroachDB releases.
| CockroachDB release | Earliest supported | Latest supported |
|---------------------|------------------------------------|---------------------------|
| 20.1 through 21.1 | FormatMostCompatible | FormatMostCompatible |
| 21.2 | FormatMostCompatible | FormatSetWithDelete |
| 21.2 | FormatMostCompatible | FormatSetWithDelete |
| 22.1 | FormatMostCompatible | FormatSplitUserKeysMarked |
| 22.2 | FormatMostCompatible | FormatPrePebblev1Marked |
| 23.1 | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest |
| 23.2 | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables |
## Pedigree

194
vendor/github.com/cockroachdb/pebble/batch.go generated vendored
View File

@@ -41,10 +41,10 @@ const (
var ErrNotIndexed = errors.New("pebble: batch not indexed")
// ErrInvalidBatch indicates that a batch is invalid or otherwise corrupted.
var ErrInvalidBatch = errors.New("pebble: invalid batch")
var ErrInvalidBatch = base.MarkCorruptionError(errors.New("pebble: invalid batch"))
// ErrBatchTooLarge indicates that a batch is invalid or otherwise corrupted.
var ErrBatchTooLarge = errors.Newf("pebble: batch too large: >= %s", humanize.Uint64(maxBatchSize))
var ErrBatchTooLarge = base.MarkCorruptionError(errors.Newf("pebble: batch too large: >= %s", humanize.Bytes.Uint64(maxBatchSize)))
// DeferredBatchOp represents a batch operation (eg. set, merge, delete) that is
// being inserted into the batch. Indexing is not performed on the specified key
@@ -183,6 +183,13 @@ func (d DeferredBatchOp) Finish() error {
// WAL, and thus stable. New record kinds may be added, but the existing ones
// will not be modified.
type Batch struct {
batchInternal
applied atomic.Bool
}
// batchInternal contains the set of fields within Batch that are non-atomic and
// capable of being reset using a *b = batchInternal{} struct copy.
type batchInternal struct {
// Data is the wire format of a batch's log entry:
// - 8 bytes for a sequence number of the first batch element,
// or zeroes if the batch has not yet been applied,
@@ -260,11 +267,6 @@ type Batch struct {
// memtable.
flushable *flushableBatch
// ingestedSSTBatch indicates that the batch contains one or more key kinds
// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
// then it will only contain key kinds of IngestSST.
ingestedSSTBatch bool
// minimumFormatMajorVersion indicates the format major version required in
// order to commit this batch. If an operation requires a particular format
// major version, it ratchets the batch's minimumFormatMajorVersion. When
@@ -292,7 +294,22 @@ type Batch struct {
commitStats BatchCommitStats
commitErr error
applied atomic.Bool
// Position bools together to reduce the sizeof the struct.
// ingestedSSTBatch indicates that the batch contains one or more key kinds
// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
// then it will only contain key kinds of IngestSST.
ingestedSSTBatch bool
// committing is set to true when a batch begins to commit. It's used to
// ensure the batch is not mutated concurrently. It is not an atomic
// deliberately, so as to avoid the overhead on batch mutations. This is
// okay, because under correct usage this field will never be accessed
// concurrently. It's only under incorrect usage the memory accesses of this
// variable may violate memory safety. Since we don't use atomics here,
// false negatives are possible.
committing bool
}
// BatchCommitStats exposes stats related to committing a batch.
@@ -319,7 +336,11 @@ type BatchCommitStats struct {
// commitPipeline.Commit.
SemaphoreWaitDuration time.Duration
// WALQueueWaitDuration is the wait time for allocating memory blocks in the
// LogWriter (due to the LogWriter not writing fast enough).
// LogWriter (due to the LogWriter not writing fast enough). At the moment
// this is duration is always zero because a single WAL will allow
// allocating memory blocks up to the entire memtable size. In the future,
// we may pipeline WALs and bound the WAL queued blocks separately, so this
// field is preserved for that possibility.
WALQueueWaitDuration time.Duration
// MemTableWriteStallDuration is the wait caused by a write stall due to too
// many memtables (due to not flushing fast enough).
@@ -363,6 +384,14 @@ func newBatch(db *DB) *Batch {
return b
}
func newBatchWithSize(db *DB, size int) *Batch {
b := newBatch(db)
if cap(b.data) < size {
b.data = rawalloc.New(0, size)
}
return b
}
func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
i := indexedBatchPool.Get().(*indexedBatch)
i.batch.cmp = comparer.Compare
@@ -374,6 +403,14 @@ func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
return &i.batch
}
func newIndexedBatchWithSize(db *DB, comparer *Comparer, size int) *Batch {
b := newIndexedBatch(db, comparer)
if cap(b.data) < size {
b.data = rawalloc.New(0, size)
}
return b
}
// nextSeqNum returns the batch "sequence number" that will be given to the next
// key written to the batch. During iteration keys within an indexed batch are
// given a sequence number consisting of their offset within the batch combined
@@ -412,18 +449,21 @@ func (b *Batch) release() {
}
}
func (b *Batch) refreshMemTableSize() {
func (b *Batch) refreshMemTableSize() error {
b.memTableSize = 0
if len(b.data) < batchHeaderLen {
return
return nil
}
b.countRangeDels = 0
b.countRangeKeys = 0
b.minimumFormatMajorVersion = 0
for r := b.Reader(); ; {
kind, key, value, ok := r.Next()
kind, key, value, ok, err := r.Next()
if !ok {
if err != nil {
return err
}
break
}
switch kind {
@@ -432,8 +472,8 @@ func (b *Batch) refreshMemTableSize() {
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
b.countRangeKeys++
case InternalKeyKindDeleteSized:
if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
}
case InternalKeyKindIngestSST:
if b.minimumFormatMajorVersion < FormatFlushableIngest {
@@ -447,6 +487,7 @@ func (b *Batch) refreshMemTableSize() {
if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys {
b.minimumFormatMajorVersion = FormatRangeKeys
}
return nil
}
// Apply the operations contained in the batch to the receiver batch.
@@ -460,7 +501,7 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
return nil
}
if len(batch.data) < batchHeaderLen {
return base.CorruptionErrorf("pebble: invalid batch")
return ErrInvalidBatch
}
offset := len(b.data)
@@ -477,8 +518,11 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
// order to update the index.
for iter := BatchReader(b.data[offset:]); len(iter) > 0; {
offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
kind, key, value, ok := iter.Next()
kind, key, value, ok, err := iter.Next()
if !ok {
if err != nil {
return err
}
break
}
switch kind {
@@ -534,6 +578,9 @@ func (b *Batch) Get(key []byte) ([]byte, io.Closer, error) {
}
func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind InternalKeyKind) {
if b.committing {
panic("pebble: batch already committing")
}
if len(b.data) == 0 {
b.init(keyLen + valueLen + 2*binary.MaxVarintLen64 + batchHeaderLen)
}
@@ -583,6 +630,9 @@ func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind Interna
}
func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
if b.committing {
panic("pebble: batch already committing")
}
if len(b.data) == 0 {
b.init(keyLen + binary.MaxVarintLen64 + batchHeaderLen)
}
@@ -764,8 +814,8 @@ func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions
// complete key slice, letting the caller encode into the DeferredBatchOp.Key
// slice and then call Finish() on the returned object.
func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp {
if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
}
// Encode the sum of the key length and the value in the value.
@@ -1053,11 +1103,12 @@ func (b *Batch) SetRepr(data []byte) error {
}
b.data = data
b.count = uint64(binary.LittleEndian.Uint32(b.countData()))
var err error
if b.db != nil {
// Only track memTableSize for batches that will be committed to the DB.
b.refreshMemTableSize()
err = b.refreshMemTableSize()
}
return nil
return err
}
// NewIter returns an iterator that is unpositioned (Iterator.Valid() will
@@ -1067,8 +1118,8 @@ func (b *Batch) SetRepr(data []byte) error {
// The returned Iterator observes all of the Batch's existing mutations, but no
// later mutations. Its view can be refreshed via RefreshBatchSnapshot or
// SetOptions().
func (b *Batch) NewIter(o *IterOptions) *Iterator {
return b.NewIterWithContext(context.Background(), o)
func (b *Batch) NewIter(o *IterOptions) (*Iterator, error) {
return b.NewIterWithContext(context.Background(), o), nil
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
@@ -1077,7 +1128,7 @@ func (b *Batch) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterato
if b.index == nil {
return &Iterator{err: ErrNotIndexed}
}
return b.db.newIter(ctx, b, nil /* snapshot */, o)
return b.db.newIter(ctx, b, snapshotIterOpts{}, o)
}
// newInternalIter creates a new internalIterator that iterates over the
@@ -1290,39 +1341,40 @@ func (b *Batch) Indexed() bool {
return b.index != nil
}
func (b *Batch) init(cap int) {
// init ensures that the batch data slice is initialized to meet the
// minimum required size and allocates space for the batch header.
func (b *Batch) init(size int) {
n := batchInitialSize
for n < cap {
for n < size {
n *= 2
}
b.data = rawalloc.New(batchHeaderLen, n)
b.setCount(0)
b.setSeqNum(0)
if cap(b.data) < n {
b.data = rawalloc.New(batchHeaderLen, n)
}
b.data = b.data[:batchHeaderLen]
// Zero the sequence number in the header.
for i := 0; i < len(b.data); i++ {
b.data[i] = 0
}
}
// Reset resets the batch for reuse. The underlying byte slice (that is
// returned by Repr()) is not modified. It is only necessary to call this
// returned by Repr()) may not be modified. It is only necessary to call this
// method if a batch is explicitly being reused. Close automatically takes are
// of releasing resources when appropriate for batches that are internally
// being reused.
func (b *Batch) Reset() {
b.count = 0
b.countRangeDels = 0
b.countRangeKeys = 0
b.memTableSize = 0
b.deferredOp = DeferredBatchOp{}
b.tombstones = nil
b.tombstonesSeqNum = 0
b.rangeKeys = nil
b.rangeKeysSeqNum = 0
b.flushable = nil
b.commit = sync.WaitGroup{}
b.fsyncWait = sync.WaitGroup{}
b.commitStats = BatchCommitStats{}
b.commitErr = nil
// Zero out the struct, retaining only the fields necessary for manual
// reuse.
b.batchInternal = batchInternal{
data: b.data,
cmp: b.cmp,
formatKey: b.formatKey,
abbreviatedKey: b.abbreviatedKey,
index: b.index,
db: b.db,
}
b.applied.Store(false)
b.minimumFormatMajorVersion = 0
if b.data != nil {
if cap(b.data) > batchMaxRetainedSize {
// If the capacity of the buffer is larger than our maximum
@@ -1333,13 +1385,14 @@ func (b *Batch) Reset() {
} else {
// Otherwise, reset the buffer for re-use.
b.data = b.data[:batchHeaderLen]
b.setSeqNum(0)
// Zero the sequence number in the header.
for i := 0; i < len(b.data); i++ {
b.data[i] = 0
}
}
}
if b.index != nil {
b.index.Init(&b.data, b.cmp, b.abbreviatedKey)
b.rangeDelIndex = nil
b.rangeKeyIndex = nil
}
}
@@ -1411,6 +1464,11 @@ func (b *Batch) Reader() BatchReader {
}
func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) {
// TODO(jackson): This will index out of bounds if there's no varint or an
// invalid varint (eg, a single 0xff byte). Correcting will add a bit of
// overhead. We could avoid that overhead whenever len(data) >=
// binary.MaxVarint32?
var v uint32
var n int
ptr := unsafe.Pointer(&data[0])
@@ -1473,19 +1531,21 @@ func ReadBatch(repr []byte) (r BatchReader, count uint32) {
return repr[batchHeaderLen:], count
}
// Next returns the next entry in this batch. The final return value is false
// if the batch is corrupt. The end of batch is reached when len(r)==0.
func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool) {
// Next returns the next entry in this batch, if there is one. If the reader has
// reached the end of the batch, Next returns ok=false and a nil error. If the
// batch is corrupt and the next entry is illegible, Next returns ok=false and a
// non-nil error.
func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool, err error) {
if len(*r) == 0 {
return 0, nil, nil, false
return 0, nil, nil, false, nil
}
kind = InternalKeyKind((*r)[0])
if kind > InternalKeyKindMax {
return 0, nil, nil, false
return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "invalid key kind 0x%x", (*r)[0])
}
*r, ukey, ok = batchDecodeStr((*r)[1:])
if !ok {
return 0, nil, nil, false
return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding user key")
}
switch kind {
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
@@ -1493,10 +1553,10 @@ func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, o
InternalKeyKindDeleteSized:
*r, value, ok = batchDecodeStr(*r)
if !ok {
return 0, nil, nil, false
return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding %s value", kind)
}
}
return kind, ukey, value, true
return kind, ukey, value, true, nil
}
// Note: batchIter mirrors the implementation of flushableBatchIter. Keep the
@@ -1698,7 +1758,7 @@ var _ flushable = (*flushableBatch)(nil)
// interface. This allows the batch to act like a memtable and be placed in the
// queue of flushable memtables. Note that the flushable batch takes ownership
// of the batch data.
func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
func newFlushableBatch(batch *Batch, comparer *Comparer) (*flushableBatch, error) {
b := &flushableBatch{
data: batch.data,
cmp: comparer.Compare,
@@ -1719,8 +1779,11 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
var index uint32
for iter := BatchReader(b.data[batchHeaderLen:]); len(iter) > 0; index++ {
offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
kind, key, _, ok := iter.Next()
kind, key, _, ok, err := iter.Next()
if !ok {
if err != nil {
return nil, err
}
break
}
entry := flushableBatchEntry{
@@ -1792,7 +1855,7 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
}
fragmentRangeKeys(frag, it, len(rangeKeyOffsets))
}
return b
return b, nil
}
func (b *flushableBatch) setSeqNum(seqNum uint64) {
@@ -1841,6 +1904,7 @@ func (b *flushableBatch) Swap(i, j int) {
b.offsets[i], b.offsets[j] = b.offsets[j], b.offsets[i]
}
// newIter is part of the flushable interface.
func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
return &flushableBatchIter{
batch: b,
@@ -1853,6 +1917,7 @@ func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
}
}
// newFlushIter is part of the flushable interface.
func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
return &flushFlushableBatchIter{
flushableBatchIter: flushableBatchIter{
@@ -1866,6 +1931,7 @@ func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) inte
}
}
// newRangeDelIter is part of the flushable interface.
func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterator {
if len(b.tombstones) == 0 {
return nil
@@ -1873,6 +1939,7 @@ func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterato
return keyspan.NewIter(b.cmp, b.tombstones)
}
// newRangeKeyIter is part of the flushable interface.
func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
if len(b.rangeKeys) == 0 {
return nil
@@ -1880,17 +1947,23 @@ func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterato
return keyspan.NewIter(b.cmp, b.rangeKeys)
}
// containsRangeKeys is part of the flushable interface.
func (b *flushableBatch) containsRangeKeys() bool { return len(b.rangeKeys) > 0 }
// inuseBytes is part of the flushable interface.
func (b *flushableBatch) inuseBytes() uint64 {
return uint64(len(b.data) - batchHeaderLen)
}
// totalBytes is part of the flushable interface.
func (b *flushableBatch) totalBytes() uint64 {
return uint64(cap(b.data))
}
// readyForFlush is part of the flushable interface.
func (b *flushableBatch) readyForFlush() bool {
// A flushable batch is always ready for flush; it must be flushed together
// with the previous memtable.
return true
}
@@ -2208,7 +2281,10 @@ func batchSort(
rangeKeyIter := b.newRangeKeyIter(nil, math.MaxUint64)
return pointIter, rangeDelIter, rangeKeyIter
}
f := newFlushableBatch(b, b.db.opts.Comparer)
f, err := newFlushableBatch(b, b.db.opts.Comparer)
if err != nil {
panic(err)
}
return f.newIter(nil), f.newRangeDelIter(nil), f.newRangeKeyIter(nil)
}

13
vendor/github.com/cockroachdb/pebble/checkpoint.go generated vendored
View File

@@ -135,9 +135,6 @@ func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
// space overhead for a checkpoint if hard links are disabled. Also beware that
// even if hard links are used, the space overhead for the checkpoint will
// increase over time as the DB performs compactions.
//
// TODO(bananabrick): Test checkpointing of virtual sstables once virtual
// sstables is running e2e.
func (d *DB) Checkpoint(
destDir string, opts ...CheckpointOption,
) (
@@ -187,12 +184,12 @@ func (d *DB) Checkpoint(
// file number.
memQueue := d.mu.mem.queue
current := d.mu.versions.currentVersion()
formatVers := d.mu.formatVers.vers
formatVers := d.FormatMajorVersion()
manifestFileNum := d.mu.versions.manifestFileNum
manifestSize := d.mu.versions.manifest.Size()
optionsFileNum := d.optionsFileNum
virtualBackingFiles := make(map[base.DiskFileNum]struct{})
for diskFileNum := range d.mu.versions.fileBackingMap {
for diskFileNum := range d.mu.versions.backingState.fileBackingMap {
virtualBackingFiles[diskFileNum] = struct{}{}
}
// Release the manifest and DB.mu so we don't block other operations on
@@ -215,11 +212,7 @@ func (d *DB) Checkpoint(
}
if ckErr != nil {
// Attempt to cleanup on error.
paths, _ := fs.List(destDir)
for _, path := range paths {
_ = fs.Remove(path)
}
_ = fs.Remove(destDir)
_ = fs.RemoveAll(destDir)
}
}()
dir, ckErr = mkdirAllAndSyncParents(fs, destDir)

281
vendor/github.com/cockroachdb/pebble/cleaner.go generated vendored
View File

@@ -4,7 +4,18 @@
package pebble
import "github.com/cockroachdb/pebble/internal/base"
import (
"context"
"runtime/pprof"
"sync"
"time"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/tokenbucket"
)
// Cleaner exports the base.Cleaner type.
type Cleaner = base.Cleaner
@@ -14,3 +25,271 @@ type DeleteCleaner = base.DeleteCleaner
// ArchiveCleaner exports the base.ArchiveCleaner type.
type ArchiveCleaner = base.ArchiveCleaner
type cleanupManager struct {
opts *Options
objProvider objstorage.Provider
onTableDeleteFn func(fileSize uint64)
deletePacer *deletionPacer
// jobsCh is used as the cleanup job queue.
jobsCh chan *cleanupJob
// waitGroup is used to wait for the background goroutine to exit.
waitGroup sync.WaitGroup
mu struct {
sync.Mutex
// totalJobs is the total number of enqueued jobs (completed or in progress).
totalJobs int
completedJobs int
completedJobsCond sync.Cond
jobsQueueWarningIssued bool
}
}
// We can queue this many jobs before we have to block EnqueueJob.
const jobsQueueDepth = 1000
// obsoleteFile holds information about a file that needs to be deleted soon.
type obsoleteFile struct {
dir string
fileNum base.DiskFileNum
fileType fileType
fileSize uint64
}
type cleanupJob struct {
jobID int
obsoleteFiles []obsoleteFile
}
// openCleanupManager creates a cleanupManager and starts its background goroutine.
// The cleanupManager must be Close()d.
func openCleanupManager(
opts *Options,
objProvider objstorage.Provider,
onTableDeleteFn func(fileSize uint64),
getDeletePacerInfo func() deletionPacerInfo,
) *cleanupManager {
cm := &cleanupManager{
opts: opts,
objProvider: objProvider,
onTableDeleteFn: onTableDeleteFn,
deletePacer: newDeletionPacer(time.Now(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo),
jobsCh: make(chan *cleanupJob, jobsQueueDepth),
}
cm.mu.completedJobsCond.L = &cm.mu.Mutex
cm.waitGroup.Add(1)
go func() {
pprof.Do(context.Background(), gcLabels, func(context.Context) {
cm.mainLoop()
})
}()
return cm
}
// Close stops the background goroutine, waiting until all queued jobs are completed.
// Delete pacing is disabled for the remaining jobs.
func (cm *cleanupManager) Close() {
close(cm.jobsCh)
cm.waitGroup.Wait()
}
// EnqueueJob adds a cleanup job to the manager's queue.
func (cm *cleanupManager) EnqueueJob(jobID int, obsoleteFiles []obsoleteFile) {
job := &cleanupJob{
jobID: jobID,
obsoleteFiles: obsoleteFiles,
}
// Report deleted bytes to the pacer, which can use this data to potentially
// increase the deletion rate to keep up. We want to do this at enqueue time
// rather than when we get to the job, otherwise the reported bytes will be
// subject to the throttling rate which defeats the purpose.
var pacingBytes uint64
for _, of := range obsoleteFiles {
if cm.needsPacing(of.fileType, of.fileNum) {
pacingBytes += of.fileSize
}
}
if pacingBytes > 0 {
cm.deletePacer.ReportDeletion(time.Now(), pacingBytes)
}
cm.mu.Lock()
cm.mu.totalJobs++
cm.maybeLogLocked()
cm.mu.Unlock()
if invariants.Enabled && len(cm.jobsCh) >= cap(cm.jobsCh)-2 {
panic("cleanup jobs queue full")
}
cm.jobsCh <- job
}
// Wait until the completion of all jobs that were already queued.
//
// Does not wait for jobs that are enqueued during the call.
//
// Note that DB.mu should not be held while calling this method; the background
// goroutine needs to acquire DB.mu to update deleted table metrics.
func (cm *cleanupManager) Wait() {
cm.mu.Lock()
defer cm.mu.Unlock()
n := cm.mu.totalJobs
for cm.mu.completedJobs < n {
cm.mu.completedJobsCond.Wait()
}
}
// mainLoop runs the manager's background goroutine.
func (cm *cleanupManager) mainLoop() {
defer cm.waitGroup.Done()
var tb tokenbucket.TokenBucket
// Use a token bucket with 1 token / second refill rate and 1 token burst.
tb.Init(1.0, 1.0)
for job := range cm.jobsCh {
for _, of := range job.obsoleteFiles {
if of.fileType != fileTypeTable {
path := base.MakeFilepath(cm.opts.FS, of.dir, of.fileType, of.fileNum)
cm.deleteObsoleteFile(of.fileType, job.jobID, path, of.fileNum, of.fileSize)
} else {
cm.maybePace(&tb, of.fileType, of.fileNum, of.fileSize)
cm.onTableDeleteFn(of.fileSize)
cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.fileNum)
}
}
cm.mu.Lock()
cm.mu.completedJobs++
cm.mu.completedJobsCond.Broadcast()
cm.maybeLogLocked()
cm.mu.Unlock()
}
}
func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNum base.DiskFileNum) bool {
if fileType != fileTypeTable {
return false
}
meta, err := cm.objProvider.Lookup(fileType, fileNum)
if err != nil {
// The object was already removed from the provider; we won't actually
// delete anything, so we don't need to pace.
return false
}
// Don't throttle deletion of remote objects.
return !meta.IsRemote()
}
// maybePace sleeps before deleting an object if appropriate. It is always
// called from the background goroutine.
func (cm *cleanupManager) maybePace(
tb *tokenbucket.TokenBucket, fileType base.FileType, fileNum base.DiskFileNum, fileSize uint64,
) {
if !cm.needsPacing(fileType, fileNum) {
return
}
tokens := cm.deletePacer.PacingDelay(time.Now(), fileSize)
if tokens == 0.0 {
// The token bucket might be in debt; it could make us wait even for 0
// tokens. We don't want that if the pacer decided throttling should be
// disabled.
return
}
// Wait for tokens. We use a token bucket instead of sleeping outright because
// the token bucket accumulates up to one second of unused tokens.
for {
ok, d := tb.TryToFulfill(tokenbucket.Tokens(tokens))
if ok {
break
}
time.Sleep(d)
}
}
// deleteObsoleteFile deletes a (non-object) file that is no longer needed.
func (cm *cleanupManager) deleteObsoleteFile(
fileType fileType, jobID int, path string, fileNum base.DiskFileNum, fileSize uint64,
) {
// TODO(peter): need to handle this error, probably by re-adding the
// file that couldn't be deleted to one of the obsolete slices map.
err := cm.opts.Cleaner.Clean(cm.opts.FS, fileType, path)
if oserror.IsNotExist(err) {
return
}
switch fileType {
case fileTypeLog:
cm.opts.EventListener.WALDeleted(WALDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
case fileTypeManifest:
cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
case fileTypeTable:
panic("invalid deletion of object file")
}
}
func (cm *cleanupManager) deleteObsoleteObject(
fileType fileType, jobID int, fileNum base.DiskFileNum,
) {
if fileType != fileTypeTable {
panic("not an object")
}
var path string
meta, err := cm.objProvider.Lookup(fileType, fileNum)
if err != nil {
path = "<nil>"
} else {
path = cm.objProvider.Path(meta)
err = cm.objProvider.Remove(fileType, fileNum)
}
if cm.objProvider.IsNotExistError(err) {
return
}
switch fileType {
case fileTypeTable:
cm.opts.EventListener.TableDeleted(TableDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
}
}
// maybeLogLocked issues a log if the job queue gets 75% full and issues a log
// when the job queue gets back to less than 10% full.
//
// Must be called with cm.mu locked.
func (cm *cleanupManager) maybeLogLocked() {
const highThreshold = jobsQueueDepth * 3 / 4
const lowThreshold = jobsQueueDepth / 10
jobsInQueue := cm.mu.totalJobs - cm.mu.completedJobs
if !cm.mu.jobsQueueWarningIssued && jobsInQueue > highThreshold {
cm.mu.jobsQueueWarningIssued = true
cm.opts.Logger.Infof("cleanup falling behind; job queue has over %d jobs", highThreshold)
}
if cm.mu.jobsQueueWarningIssued && jobsInQueue < lowThreshold {
cm.mu.jobsQueueWarningIssued = false
cm.opts.Logger.Infof("cleanup back to normal; job queue has under %d jobs", lowThreshold)
}
}

29
vendor/github.com/cockroachdb/pebble/commit.go generated vendored
View File

@@ -9,7 +9,6 @@ import (
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/pebble/record"
)
@@ -39,7 +38,7 @@ type commitQueue struct {
// power of 2. A slot is in use until *both* the tail index has moved beyond
// it and the slot value has been set to nil. The slot value is set to nil
// atomically by the consumer and read atomically by the producer.
slots [record.SyncConcurrency]unsafe.Pointer
slots [record.SyncConcurrency]atomic.Pointer[Batch]
}
const dequeueBits = 32
@@ -67,8 +66,8 @@ func (q *commitQueue) enqueue(b *Batch) {
}
slot := &q.slots[head&uint32(len(q.slots)-1)]
// Check if the head slot has been released by dequeue.
for atomic.LoadPointer(slot) != nil {
// Check if the head slot has been released by dequeueApplied.
for slot.Load() != nil {
// Another goroutine is still cleaning up the tail, so the queue is
// actually still full. We spin because this should resolve itself
// momentarily.
@@ -76,14 +75,18 @@ func (q *commitQueue) enqueue(b *Batch) {
}
// The head slot is free, so we own it.
atomic.StorePointer(slot, unsafe.Pointer(b))
slot.Store(b)
// Increment head. This passes ownership of slot to dequeue and acts as a
// Increment head. This passes ownership of slot to dequeueApplied and acts as a
// store barrier for writing the slot.
q.headTail.Add(1 << dequeueBits)
}
func (q *commitQueue) dequeue() *Batch {
// dequeueApplied removes the earliest enqueued Batch, if it is applied.
//
// Returns nil if the commit queue is empty or the earliest Batch is not yet
// applied.
func (q *commitQueue) dequeueApplied() *Batch {
for {
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
@@ -93,7 +96,7 @@ func (q *commitQueue) dequeue() *Batch {
}
slot := &q.slots[tail&uint32(len(q.slots)-1)]
b := (*Batch)(atomic.LoadPointer(slot))
b := slot.Load()
if b == nil || !b.applied.Load() {
// The batch is not ready to be dequeued, or another goroutine has
// already dequeued it.
@@ -109,7 +112,7 @@ func (q *commitQueue) dequeue() *Batch {
// Tell enqueue that we're done with this slot. Zeroing the slot is also
// important so we don't leave behind references that could keep this object
// live longer than necessary.
atomic.StorePointer(slot, nil)
slot.Store(nil)
// At this point enqueue owns the slot.
return b
}
@@ -239,6 +242,7 @@ type commitPipeline struct {
// logSyncQSem are used for this reservation.
commitQueueSem chan struct{}
logSyncQSem chan struct{}
ingestSem chan struct{}
// The mutex to use for synchronizing access to logSeqNum and serializing
// calls to commitEnv.write().
mu sync.Mutex
@@ -263,6 +267,7 @@ func newCommitPipeline(env commitEnv) *commitPipeline {
// and sync the WAL.
commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
logSyncQSem: make(chan struct{}, record.SyncConcurrency-1),
ingestSem: make(chan struct{}, 1),
}
return p
}
@@ -471,13 +476,13 @@ func (p *commitPipeline) publish(b *Batch) {
// Loop dequeuing applied batches from the pending queue. If our batch was
// the head of the pending queue we are guaranteed that either we'll publish
// it or someone else will dequeue and publish it. If our batch is not the
// head of the queue then either we'll dequeue applied batches and reach our
// it or someone else will dequeueApplied and publish it. If our batch is not the
// head of the queue then either we'll dequeueApplied applied batches and reach our
// batch or there is an unapplied batch blocking us. When that unapplied
// batch applies it will go through the same process and publish our batch
// for us.
for {
t := p.pending.dequeue()
t := p.pending.dequeueApplied()
if t == nil {
// Wait for another goroutine to publish us. We might also be waiting for
// the WAL sync to finish.

1281
vendor/github.com/cockroachdb/pebble/compaction.go generated vendored
View File

File diff suppressed because it is too large Load Diff

686
vendor/github.com/cockroachdb/pebble/compaction_iter.go generated vendored
View File

@@ -15,9 +15,9 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/rangekey"
"github.com/cockroachdb/redact"
)
// compactionIter provides a forward-only iterator that encapsulates the logic
@@ -207,14 +207,6 @@ type compactionIter struct {
// its elision. This field only applies to point keys, and not to range
// deletions or range keys.
//
// snapshotPinned is also used to set the forceObsolete value in the call to
// Writer.AddWithForceObsolete. Note that in that call, it is sufficient to
// mark all keys obsoleted by RANGEDELs as forceObsolete=true and that the
// implementation of Writer.AddWithForceObsolete will itself discover other
// causes of obsolescence. We mention this since in the presence of MERGE,
// obsolescence due to multiple keys at the same user key is not fully
// represented by snapshotPinned=true:
//
// For MERGE, it is possible that doing the merge is interrupted even when
// the next point key is in the same stripe. This can happen if the loop in
// mergeNext gets interrupted by sameStripeNonSkippable.
@@ -222,6 +214,21 @@ type compactionIter struct {
// SET/MERGE/DEL with the same seqnum, so the RANGEDEL does not necessarily
// delete the subsequent SET/MERGE/DEL keys.
snapshotPinned bool
// forceObsoleteDueToRangeDel is set to true in a subset of the cases that
// snapshotPinned is true. This value is true when the point is obsolete due
// to a RANGEDEL but could not be deleted due to a snapshot.
//
// NB: it may seem that the additional cases that snapshotPinned captures
// are harmless in that they can also be used to mark a point as obsolete
// (it is merely a duplication of some logic that happens in
// Writer.AddWithForceObsolete), but that is not quite accurate as of this
// writing -- snapshotPinned originated in stats collection and for a
// sequence MERGE, SET, where the MERGE cannot merge with the (older) SET
// due to a snapshot, the snapshotPinned value for the SET is true.
//
// TODO(sumeer,jackson): improve the logic of snapshotPinned and reconsider
// whether we need forceObsoleteDueToRangeDel.
forceObsoleteDueToRangeDel bool
// The index of the snapshot for the current key within the snapshots slice.
curSnapshotIdx int
curSnapshotSeqNum uint64
@@ -249,10 +256,12 @@ type compactionIter struct {
// The fragmented range keys.
rangeKeys []keyspan.Span
// Byte allocator for the tombstone keys.
alloc bytealloc.A
allowZeroSeqNum bool
elideTombstone func(key []byte) bool
elideRangeTombstone func(start, end []byte) bool
alloc bytealloc.A
allowZeroSeqNum bool
elideTombstone func(key []byte) bool
elideRangeTombstone func(start, end []byte) bool
ineffectualSingleDeleteCallback func(userKey []byte)
singleDeleteInvariantViolationCallback func(userKey []byte)
// The on-disk format major version. This informs the types of keys that
// may be written to disk during a compaction.
formatVersion FormatMajorVersion
@@ -274,20 +283,24 @@ func newCompactionIter(
allowZeroSeqNum bool,
elideTombstone func(key []byte) bool,
elideRangeTombstone func(start, end []byte) bool,
ineffectualSingleDeleteCallback func(userKey []byte),
singleDeleteInvariantViolationCallback func(userKey []byte),
formatVersion FormatMajorVersion,
) *compactionIter {
i := &compactionIter{
equal: equal,
merge: merge,
iter: iter,
snapshots: snapshots,
frontiers: frontiers{cmp: cmp},
rangeDelFrag: rangeDelFrag,
rangeKeyFrag: rangeKeyFrag,
allowZeroSeqNum: allowZeroSeqNum,
elideTombstone: elideTombstone,
elideRangeTombstone: elideRangeTombstone,
formatVersion: formatVersion,
equal: equal,
merge: merge,
iter: iter,
snapshots: snapshots,
frontiers: frontiers{cmp: cmp},
rangeDelFrag: rangeDelFrag,
rangeKeyFrag: rangeKeyFrag,
allowZeroSeqNum: allowZeroSeqNum,
elideTombstone: elideTombstone,
elideRangeTombstone: elideRangeTombstone,
ineffectualSingleDeleteCallback: ineffectualSingleDeleteCallback,
singleDeleteInvariantViolationCallback: singleDeleteInvariantViolationCallback,
formatVersion: formatVersion,
}
i.rangeDelFrag.Cmp = cmp
i.rangeDelFrag.Format = formatKey
@@ -326,7 +339,7 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
return nil, nil
}
// Prior to this call to `Next()` we are in one of three situations with
// Prior to this call to `Next()` we are in one of four situations with
// respect to `iterKey` and related state:
//
// - `!skip && pos == iterPosNext`: `iterKey` is already at the next key.
@@ -335,12 +348,34 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
// snapshot stripe.
// - `skip && pos == iterPosCurForward`: We are at the key that has been returned.
// To move forward we skip skippable entries in the stripe.
// - `skip && pos == iterPosNext && i.iterStripeChange == sameStripeNonSkippable`:
// This case may occur when skipping within a snapshot stripe and we
// encounter either:
// a) an invalid key kind; The previous call will have returned
// whatever key it was processing and deferred handling of the
// invalid key to this invocation of Next(). We're responsible for
// ignoring skip=true and falling into the invalid key kind case
// down below.
// b) an interleaved range delete; This is a wart of the current code
// structure. While skipping within a snapshot stripe, a range
// delete interleaved at its start key and sequence number
// interrupts the sequence of point keys. After we return the range
// delete to the caller, we need to pick up skipping at where we
// left off, so we preserve skip=true.
// TODO(jackson): This last case is confusing and can be removed if we
// interleave range deletions at the maximal sequence number using the
// keyspan interleaving iterator. This is the treatment given to range
// keys today.
if i.pos == iterPosCurForward {
if i.skip {
i.skipInStripe()
} else {
i.nextInStripe()
}
} else if i.skip {
if i.iterStripeChange != sameStripeNonSkippable {
panic(errors.AssertionFailedf("compaction iterator has skip=true, but iterator is at iterPosNext"))
}
}
i.pos = iterPosCurForward
@@ -394,6 +429,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
return &i.key, i.value
}
// TODO(sumeer): we could avoid calling Covers if i.iterStripeChange ==
// sameStripeSameKey since that check has already been done in
// nextInStripeHelper. However, we also need to handle the case of
// CoversInvisibly below.
if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly {
// A pending range deletion deletes this key. Skip it.
i.saveKey()
@@ -410,6 +449,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
// key is in the same snapshot stripe. Hence, snapshotPinned is by
// definition false in those cases.
i.snapshotPinned = true
i.forceObsoleteDueToRangeDel = true
} else {
i.forceObsoleteDueToRangeDel = false
}
switch i.iterKey.Kind() {
@@ -419,7 +461,17 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
// If we're at the last snapshot stripe and the tombstone
// can be elided skip skippable keys in the same stripe.
i.saveKey()
i.skipInStripe()
if i.key.Kind() == InternalKeyKindSingleDelete {
i.skipDueToSingleDeleteElision()
} else {
i.skipInStripe()
if !i.skip && i.iterStripeChange != newStripeNewKey {
panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe disabled skip without advancing to new key"))
}
}
if i.iterStripeChange == newStripeSameKey {
panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe found a new stripe within the same key"))
}
continue
} else {
// We're not at the last snapshot stripe, so the tombstone
@@ -446,8 +498,14 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
case InternalKeyKindSingleDelete:
if i.singleDeleteNext() {
return &i.key, i.value
} else if i.err != nil {
return nil, nil
}
continue
default:
panic(errors.AssertionFailedf(
"unexpected kind %s", redact.SafeString(i.iterKey.Kind().String())))
}
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -457,6 +515,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
// preserving the original value, and potentially mutating the key
// kind.
i.setNext()
if i.err != nil {
return nil, nil
}
return &i.key, i.value
case InternalKeyKindMerge:
@@ -473,7 +534,15 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
if i.err == nil {
// includesBase is true whenever we've transformed the MERGE record
// into a SET.
includesBase := i.key.Kind() == InternalKeyKindSet
var includesBase bool
switch i.key.Kind() {
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
includesBase = true
case InternalKeyKindMerge:
default:
panic(errors.AssertionFailedf(
"unexpected kind %s", redact.SafeString(i.key.Kind().String())))
}
i.value, needDelete, i.valueCloser, i.err = finishValueMerger(valueMerger, includesBase)
}
if i.err == nil {
@@ -502,6 +571,8 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
}
if i.err != nil {
i.valid = false
// TODO(sumeer): why is MarkCorruptionError only being called for
// MERGE?
i.err = base.MarkCorruptionError(i.err)
}
return nil, nil
@@ -541,10 +612,16 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) {
return index, snapshots[index]
}
// skipInStripe skips over skippable keys in the same stripe and user key.
// skipInStripe skips over skippable keys in the same stripe and user key. It
// may set i.err, in which case i.iterKey will be nil.
func (i *compactionIter) skipInStripe() {
i.skip = true
// TODO(sumeer): we can avoid the overhead of calling i.rangeDelFrag.Covers,
// in this case of nextInStripe, since we are skipping all of them anyway.
for i.nextInStripe() == sameStripeSkippable {
if i.err != nil {
panic(i.err)
}
}
// Reset skip if we landed outside the original stripe. Otherwise, we landed
// in the same stripe on a non-skippable key. In that case we should preserve
@@ -582,11 +659,17 @@ const (
// nextInStripe advances the iterator and returns one of the above const ints
// indicating how its state changed.
//
// All sameStripeSkippable keys that are covered by a RANGEDEL will be skipped
// and not returned.
//
// Calls to nextInStripe must be preceded by a call to saveKey to retain a
// temporary reference to the original key, so that forward iteration can
// proceed with a reference to the original key. Care should be taken to avoid
// overwriting or mutating the saved key or value before they have been returned
// to the caller of the exported function (i.e. the caller of Next, First, etc.)
//
// nextInStripe may set i.err, in which case the return value will be
// newStripeNewKey, and i.iterKey will be nil.
func (i *compactionIter) nextInStripe() stripeChangeType {
i.iterStripeChange = i.nextInStripeHelper()
return i.iterStripeChange
@@ -595,58 +678,72 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
// nextInStripeHelper is an internal helper for nextInStripe; callers should use
// nextInStripe and not call nextInStripeHelper.
func (i *compactionIter) nextInStripeHelper() stripeChangeType {
if !i.iterNext() {
return newStripeNewKey
}
key := i.iterKey
origSnapshotIdx := i.curSnapshotIdx
for {
if !i.iterNext() {
return newStripeNewKey
}
key := i.iterKey
// NB: The below conditional is an optimization to avoid a user key
// comparison in many cases. Internal keys with the same user key are
// ordered in (strictly) descending order by trailer. If the new key has a
// greater or equal trailer, or the previous key had a zero sequence number,
// the new key must have a new user key.
//
// A couple things make these cases common:
// - Sequence-number zeroing ensures ~all of the keys in L6 have a zero
// sequence number.
// - Ingested sstables' keys all adopt the same sequence number.
if i.keyTrailer <= base.InternalKeyZeroSeqnumMaxTrailer || key.Trailer >= i.keyTrailer {
if invariants.Enabled && i.equal(i.key.UserKey, key.UserKey) {
if !i.equal(i.key.UserKey, key.UserKey) {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripeNewKey
}
// If i.key and key have the same user key, then
// 1. i.key must not have had a zero sequence number (or it would've be the last
// key with its user key).
// 2. i.key must have a strictly larger sequence number
// There's an exception in that either key may be a range delete. Range
// deletes may share a sequence number with a point key if the keys were
// ingested together. Range keys may also share the sequence number if they
// were ingested, but range keys are interleaved into the compaction
// iterator's input iterator at the maximal sequence number so their
// original sequence number will not be observed here.
if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) &&
i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete {
prevKey := i.key
prevKey.Trailer = i.keyTrailer
panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey))
panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key))
}
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripeNewKey
} else if !i.equal(i.key.UserKey, key.UserKey) {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripeNewKey
}
origSnapshotIdx := i.curSnapshotIdx
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
switch key.Kind() {
case InternalKeyKindRangeDelete:
// Range tombstones need to be exposed by the compactionIter to the upper level
// `compaction` object, so return them regardless of whether they are in the same
// snapshot stripe.
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
switch key.Kind() {
case InternalKeyKindRangeDelete:
// Range tombstones need to be exposed by the compactionIter to the upper level
// `compaction` object, so return them regardless of whether they are in the same
// snapshot stripe.
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripeSameKey
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
// Range keys are interleaved at the max sequence number for a given user
// key, so we should not see any more range keys in this stripe.
panic("unreachable")
case InternalKeyKindInvalid:
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripeSameKey
case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete,
InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
// Fall through
default:
i.iterKey = nil
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
return newStripeNewKey
}
return newStripeSameKey
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
// Range keys are interleaved at the max sequence number for a given user
// key, so we should not see any more range keys in this stripe.
panic("unreachable")
case InternalKeyKindInvalid:
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
// Same snapshot.
if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
continue
}
return sameStripeSkippable
}
return newStripeSameKey
}
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeSkippable
}
return newStripeSameKey
}
func (i *compactionIter) setNext() {
@@ -672,6 +769,10 @@ func (i *compactionIter) setNext() {
// Else, we continue to loop through entries in the stripe looking for a
// DEL. Note that we may stop *before* encountering a DEL, if one exists.
//
// NB: nextInStripe will skip sameStripeSkippable keys that are visibly
// covered by a RANGEDEL. This can include DELs -- this is fine since such
// DELs don't need to be combined with SET to make SETWITHDEL.
for {
switch i.nextInStripe() {
case newStripeNewKey, newStripeSameKey:
@@ -722,12 +823,16 @@ func (i *compactionIter) setNext() {
// We're still in the same stripe. If this is a
// DEL/SINGLEDEL/DELSIZED, we stop looking and emit a SETWITHDEL.
// Subsequent keys are eligible for skipping.
if i.iterKey.Kind() == InternalKeyKindDelete ||
i.iterKey.Kind() == InternalKeyKindSingleDelete ||
i.iterKey.Kind() == InternalKeyKindDeleteSized {
switch i.iterKey.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
i.key.SetKind(InternalKeyKindSetWithDelete)
i.skip = true
return
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// Do nothing
default:
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
}
default:
panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange)))
@@ -747,13 +852,24 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
i.pos = iterPosNext
return i.iterStripeChange
}
if i.err != nil {
panic(i.err)
}
// NB: MERGE#10+RANGEDEL#9 stays a MERGE, since nextInStripe skips
// sameStripeSkippable keys that are visibly covered by a RANGEDEL. There
// may be MERGE#7 that is invisibly covered and will be preserved, but
// there is no risk that MERGE#10 and MERGE#7 will get merged in the
// future as the RANGEDEL still exists and will be used in user-facing
// reads that see MERGE#10, and will also eventually cause MERGE#7 to be
// deleted in a compaction.
key := i.iterKey
switch key.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
// We've hit a deletion tombstone. Return everything up to this point and
// then skip entries until the next snapshot stripe. We change the kind
// of the result key to a Set so that it shadows keys in lower
// levels. That is, MERGE+DEL -> SET.
// levels. That is, MERGE+DEL -> SETWITHDEL.
//
// We do the same for SingleDelete since SingleDelete is only
// permitted (with deterministic behavior) for keys that have been
// set once since the last SingleDelete/Delete, so everything
@@ -766,21 +882,11 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
// single Set, and then merge in any following Sets, but that is
// complicated wrt code and unnecessary given the narrow permitted
// use of SingleDelete.
i.key.SetKind(InternalKeyKindSet)
i.key.SetKind(InternalKeyKindSetWithDelete)
i.skip = true
return sameStripeSkippable
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
// MERGE+DEL.
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
}
// We've hit a Set or SetWithDel value. Merge with the existing
// value and return. We change the kind of the resulting key to a
// Set so that it shadows keys in lower levels. That is:
@@ -795,16 +901,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
return sameStripeSkippable
case InternalKeyKindMerge:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
// MERGE+DEL.
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
}
// We've hit another Merge value. Merge with the existing value and
// continue looping.
i.err = valueMerger.MergeOlder(i.iterValue)
@@ -821,6 +917,16 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
}
}
// singleDeleteNext processes a SingleDelete point tombstone. A SingleDelete, or
// SINGLEDEL, is unique in that it deletes exactly 1 internal key. It's a
// performance optimization when the client knows a user key has not been
// overwritten, allowing the elision of the tombstone earlier, avoiding write
// amplification.
//
// singleDeleteNext returns a boolean indicating whether or not the caller
// should yield the SingleDelete key to the consumer of the compactionIter. If
// singleDeleteNext returns false, the caller may consume/elide the
// SingleDelete.
func (i *compactionIter) singleDeleteNext() bool {
// Save the current key.
i.saveKey()
@@ -829,26 +935,85 @@ func (i *compactionIter) singleDeleteNext() bool {
// Loop until finds a key to be passed to the next level.
for {
// If we find a key that can't be skipped, return true so that the
// caller yields the SingleDelete to the caller.
if i.nextInStripe() != sameStripeSkippable {
// This defers additional error checking regarding single delete
// invariants to the compaction where the keys with the same user key as
// the single delete are in the same stripe.
i.pos = iterPosNext
return true
return i.err == nil
}
if i.err != nil {
panic(i.err)
}
// INVARIANT: sameStripeSkippable.
key := i.iterKey
switch key.Kind() {
case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
// We've hit a Delete, DeleteSized, Merge, SetWithDelete, transform
kind := key.Kind()
switch kind {
case InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
if (kind == InternalKeyKindDelete || kind == InternalKeyKindDeleteSized) &&
i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
// We've hit a Delete, DeleteSized, SetWithDelete, transform
// the SingleDelete into a full Delete.
i.key.SetKind(InternalKeyKindDelete)
i.skip = true
return true
case InternalKeyKindSet:
i.nextInStripe()
case InternalKeyKindSet, InternalKeyKindMerge:
// This SingleDelete deletes the Set/Merge, and we can now elide the
// SingleDel as well. We advance past the Set and return false to
// indicate to the main compaction loop that we should NOT yield the
// current SingleDel key to the compaction loop.
//
// NB: singleDeleteNext was called with i.pos == iterPosCurForward, and
// after the call to nextInStripe, we are still at iterPosCurForward,
// since we are at the key after the Set/Merge that was single deleted.
change := i.nextInStripe()
switch change {
case sameStripeSkippable, newStripeSameKey:
// On the same user key.
nextKind := i.iterKey.Kind()
switch nextKind {
case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
if i.singleDeleteInvariantViolationCallback != nil {
// sameStripeSkippable keys returned by nextInStripe() are already
// known to not be covered by a RANGEDEL, so it is an invariant
// violation. The rare case is newStripeSameKey, where it is a
// violation if not covered by a RANGEDEL.
if change == sameStripeSkippable ||
i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.NoCover {
i.singleDeleteInvariantViolationCallback(i.key.UserKey)
}
}
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
InternalKeyKindRangeDelete:
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
case sameStripeNonSkippable:
// No ability to check whether there is another Set/Merge below with
// the same user key.
//
// TODO(sumeer): once range deletions are interleaved at the maximal
// sequence number, this case will go away.
case newStripeNewKey:
default:
panic("unreachable")
}
i.valid = false
return false
case InternalKeyKindSingleDelete:
// Two single deletes met in a compaction. The first single delete is
// ineffectual.
if i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
// Continue to apply the second single delete.
continue
default:
@@ -859,6 +1024,135 @@ func (i *compactionIter) singleDeleteNext() bool {
}
}
// skipDueToSingleDeleteElision is called when the SingleDelete is being
// elided because it is in the final snapshot stripe and there are no keys
// with the same user key in lower levels in the LSM (below the files in this
// compaction).
//
// TODO(sumeer): the only difference between singleDeleteNext and
// skipDueToSingleDeleteElision is the fact that the caller knows it will be
// eliding the single delete in the latter case. There are some similar things
// happening in both implementations. My first attempt at combining them into
// a single method was hard to comprehend. Try again.
func (i *compactionIter) skipDueToSingleDeleteElision() {
for {
stripeChange := i.nextInStripe()
if i.err != nil {
panic(i.err)
}
switch stripeChange {
case newStripeNewKey:
// The single delete is only now being elided, meaning it did not elide
// any keys earlier in its descent down the LSM. We stepped onto a new
// user key, meaning that even now at its moment of elision, it still
// hasn't elided any other keys. The single delete was ineffectual (a
// no-op).
if i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
i.skip = false
return
case newStripeSameKey:
// This should be impossible. If we're eliding a single delete, we
// determined that the tombstone is in the final snapshot stripe, but we
// stepped into a new stripe of the same key.
panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
case sameStripeNonSkippable:
// There's a key that we cannot skip. There are two possible cases:
// a. The key is invalid. This is an error.
// b. The key is a range deletion.
// The second case may also be an ineffectual single delete. However, it
// is possible that there is a SET that is at the same seqnum as the
// RANGEDEL, and so is not deleted by that RANGEDEL, and will be deleted
// by this single delete. So we cannot be certain that this is an
// ineffectual single delete.
//
// TODO(sumeer): the existing todo to interleave range deletions at the
// maximal sequence number will allow us to address this ambiguity.
//
// TODO(sumeer): by setting skip to true, the compactionIter is making a
// single delete stronger (like a del), which will hide bugs in the use of
// single delete.
i.skip = true
return
case sameStripeSkippable:
kind := i.iterKey.Kind()
switch kind {
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
if i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
switch kind {
case InternalKeyKindDelete, InternalKeyKindDeleteSized:
i.skipInStripe()
return
case InternalKeyKindSingleDelete:
// Repeat the same with this SingleDelete. We don't want to simply
// call skipInStripe(), since it increases the strength of the
// SingleDel, which hides bugs in the use of single delete.
continue
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
case InternalKeyKindSetWithDelete:
// The SingleDelete should behave like a Delete.
i.skipInStripe()
return
case InternalKeyKindSet, InternalKeyKindMerge:
// This SingleDelete deletes the Set/Merge, and we are eliding the
// SingleDel as well. Step to the next key (this is not deleted by the
// SingleDelete).
//
// NB: skipDueToSingleDeleteElision was called with i.pos ==
// iterPosCurForward, and after the call to nextInStripe, we are still
// at iterPosCurForward, since we are at the key after the Set/Merge
// that was single deleted.
change := i.nextInStripe()
if i.err != nil {
panic(i.err)
}
switch change {
case newStripeSameKey:
panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
case newStripeNewKey:
case sameStripeSkippable:
// On the same key.
nextKind := i.iterKey.Kind()
switch nextKind {
case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
if i.singleDeleteInvariantViolationCallback != nil {
i.singleDeleteInvariantViolationCallback(i.key.UserKey)
}
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
InternalKeyKindRangeDelete:
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
case sameStripeNonSkippable:
// No ability to check whether there is another Set/Merge below with
// the same user key.
//
// TODO(sumeer): once range deletions are interleaved at the maximal
// sequence number, this case will go away.
default:
panic("unreachable")
}
// Whether in same stripe or new stripe, this key is not consumed by
// the SingleDelete.
i.skip = false
return
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
default:
panic("unreachable")
}
}
}
// deleteSizedNext processes a DELSIZED point tombstone. Unlike ordinary DELs,
// these tombstones carry a value that's a varint indicating the size of the
// entry (len(key)+len(value)) that the tombstone is expected to delete.
@@ -881,28 +1175,137 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
i.value = i.valueBuf
}
// Loop through all the keys within this stripe that are skippable,
// totalling the size of their keys and values.
// Loop through all the keys within this stripe that are skippable.
i.pos = iterPosNext
var elidedSize uint64
for i.nextInStripe() == sameStripeSkippable {
elidedSize += uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue))
if i.iterKey.Kind() == InternalKeyKindDeleteSized {
// We encountered a DELSIZED that's deleted by the original
// DELSIZED. These tombstones could've been intended to delete two
// distinct values, eg:
if i.err != nil {
panic(i.err)
}
switch i.iterKey.Kind() {
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
// We encountered a tombstone (DEL, or DELSIZED) that's deleted by
// the original DELSIZED tombstone. This can happen in two cases:
//
// (1) These tombstones were intended to delete two distinct values,
// and this DELSIZED has already dropped the relevant key. For
// example:
//
// a.DELSIZED.9 a.SET.7 a.DELSIZED.5 a.SET.4
//
// If a.DELSIZED.9 has already deleted a.SET.7, its size has already
// been zeroed out. In this case, we want to adopt the value of the
// DELSIZED with the lower sequence number, in case the a.SET.4 key
// has not yet been elided.
// If a.DELSIZED.9 has already deleted a.SET.7, its size has
// already been zeroed out. In this case, we want to adopt the
// value of the DELSIZED with the lower sequence number, in
// case the a.SET.4 key has not yet been elided.
//
// (2) This DELSIZED was missized. The user thought they were
// deleting a key with this user key, but this user key had
// already been deleted.
//
// We can differentiate these two cases by examining the length of
// the DELSIZED's value. A DELSIZED's value holds the size of both
// the user key and value that it intends to delete. For any user
// key with a length > 0, a DELSIZED that has not deleted a key must
// have a value with a length > 0.
//
// We treat both cases the same functionally, adopting the identity
// of the lower-sequence numbered tombstone. However in the second
// case, we also increment the stat counting missized tombstones.
if len(i.value) > 0 {
// The original DELSIZED key was missized. The key that the user
// thought they were deleting does not exist.
i.stats.countMissizedDels++
}
i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
i.value = i.valueBuf
// Reset the elided total.
elidedSize = 0
if i.iterKey.Kind() != InternalKeyKindDeleteSized {
// Convert the DELSIZED to a DEL—The DEL/SINGLEDEL we're eliding
// may not have deleted the key(s) it was intended to yet. The
// ordinary DEL compaction heuristics are better suited at that,
// plus we don't want to count it as a missized DEL. We early
// exit in this case, after skipping the remainder of the
// snapshot stripe.
i.key.SetKind(InternalKeyKindDelete)
// NB: We skipInStripe now, rather than returning leaving
// i.skip=true and returning early, because Next() requires
// that i.skip=true only if i.iterPos = iterPosCurForward.
//
// Ignore any error caused by skipInStripe since it does not affect
// the key/value being returned here, and the next call to Next() will
// expose it.
i.skipInStripe()
return &i.key, i.value
}
// Continue, in case we uncover another DELSIZED or a key this
// DELSIZED deletes.
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// If the DELSIZED is value-less, it already deleted the key that it
// was intended to delete. This is possible with a sequence like:
//
// DELSIZED.8 SET.7 SET.3
//
// The DELSIZED only describes the size of the SET.7, which in this
// case has already been elided. We don't count it as a missizing,
// instead converting the DELSIZED to a DEL. Skip the remainder of
// the snapshot stripe and return.
if len(i.value) == 0 {
i.key.SetKind(InternalKeyKindDelete)
// NB: We skipInStripe now, rather than returning leaving
// i.skip=true and returning early, because Next() requires
// that i.skip=true only if i.iterPos = iterPosCurForward.
//
// Ignore any error caused by skipInStripe since it does not affect
// the key/value being returned here, and the next call to Next() will
// expose it.
i.skipInStripe()
return &i.key, i.value
}
// The deleted key is not a DEL, DELSIZED, and the DELSIZED in i.key
// has a positive size.
expectedSize, n := binary.Uvarint(i.value)
if n != len(i.value) {
i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value))
i.valid = false
return nil, nil
}
elidedSize := uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue))
if elidedSize != expectedSize {
// The original DELSIZED key was missized. It's unclear what to
// do. The user-provided size was wrong, so it's unlikely to be
// accurate or meaningful. We could:
//
// 1. return the DELSIZED with the original user-provided size unmodified
// 2. return the DELZIZED with a zeroed size to reflect that a key was
// elided, even if it wasn't the anticipated size.
// 3. subtract the elided size from the estimate and re-encode.
// 4. convert the DELSIZED into a value-less DEL, so that
// ordinary DEL heuristics apply.
//
// We opt for (4) under the rationale that we can't rely on the
// user-provided size for accuracy, so ordinary DEL heuristics
// are safer.
i.stats.countMissizedDels++
i.key.SetKind(InternalKeyKindDelete)
i.value = i.valueBuf[:0]
// NB: We skipInStripe now, rather than returning leaving
// i.skip=true and returning early, because Next() requires
// that i.skip=true only if i.iterPos = iterPosCurForward.
//
// Ignore any error caused by skipInStripe since it does not affect
// the key/value being returned here, and the next call to Next() will
// expose it.
i.skipInStripe()
return &i.key, i.value
}
// NB: We remove the value regardless of whether the key was sized
// appropriately. The size encoded is 'consumed' the first time it
// meets a key that it deletes.
i.value = i.valueBuf[:0]
default:
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
return nil, nil
}
}
// Reset skip if we landed outside the original stripe. Otherwise, we landed
@@ -912,53 +1315,8 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey {
i.skip = false
}
if elidedSize == 0 {
// If we didn't find any keys that we could elide due to this tombstone
// (or at least none since the last DELSIZED encountered), we can return
// the key and our existing saved value verbatim.
return &i.key, i.value
}
// Some key(s) were elided as a result of this tombstone. Decode the
// tombstone's value to see if it matches the amount of data actually
// elided.
var v uint64
if len(i.value) > 0 {
var n int
v, n = binary.Uvarint(i.value)
if n != len(i.value) {
i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value))
i.valid = false
return nil, nil
}
}
// If the user's prediction of the value size was correct, then from now on
// it's predicted to delete nothing. Return an empty value.
//
// NB: It's important that we still always return the tombstone, because
// DELSIZED's values are a best effort optimization, not a correctness
// mechanic. There is no guarantee that this key was only set once before
// the DELSIZED was written.
// If the tombstone carried a larger size than what was actually elided,
// it's unclear what to do. The user-provided size was wrong, so it's
// unlikely to be accurate or meaningful. We could:
//
// 1. return the DELSIZED with the original user-provided size unmodified
// 2. return the DELZIZED with a zeroed size to reflect that a key was
// elided, even if it wasn't the anticipated size.
// 3. subtract the elided size from the estimate and re-encode.
// 4. convert the DELSIZED into a value-less DEL, so that ordinary DEL
// heuristics apply.
//
// We opt for (4) under the rationale that we can't rely on the
// user-provided size for accuracy, so ordinary DEL heuristics are safer.
i.value = i.valueBuf[:0]
if elidedSize != v {
i.stats.countMissizedDels++
i.key.SetKind(InternalKeyKindDelete)
if i.err != nil {
return nil, nil
}
return &i.key, i.value
}

645
vendor/github.com/cockroachdb/pebble/compaction_picker.go generated vendored
View File

File diff suppressed because it is too large Load Diff

760
vendor/github.com/cockroachdb/pebble/db.go generated vendored
View File

File diff suppressed because it is too large Load Diff

80
vendor/github.com/cockroachdb/pebble/event.go generated vendored
View File

@@ -39,10 +39,11 @@ func formatFileNums(tables []TableInfo) string {
return buf.String()
}
// LevelInfo contains info pertaining to a partificular level.
// LevelInfo contains info pertaining to a particular level.
type LevelInfo struct {
Level int
Tables []TableInfo
Score float64
}
func (i LevelInfo) String() string {
@@ -51,8 +52,11 @@ func (i LevelInfo) String() string {
// SafeFormat implements redact.SafeFormatter.
func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables))))
w.Printf("L%d [%s] (%s) Score=%.2f",
redact.Safe(i.Level),
redact.Safe(formatFileNums(i.Tables)),
redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))),
redact.Safe(i.Score))
}
// CompactionInfo contains the info for a compaction event.
@@ -75,6 +79,27 @@ type CompactionInfo struct {
TotalDuration time.Duration
Done bool
Err error
SingleLevelOverlappingRatio float64
MultiLevelOverlappingRatio float64
// Annotations specifies additional info to appear in a compaction's event log line
Annotations compactionAnnotations
}
type compactionAnnotations []string
// SafeFormat implements redact.SafeFormatter.
func (ca compactionAnnotations) SafeFormat(w redact.SafePrinter, _ rune) {
if len(ca) == 0 {
return
}
for i := range ca {
if i != 0 {
w.Print(" ")
}
w.Printf("%s", redact.SafeString(ca[i]))
}
}
func (i CompactionInfo) String() string {
@@ -90,20 +115,25 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
}
if !i.Done {
w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Print(levelInfos(i.Input))
w.Printf("[JOB %d] compacting(%s) ",
redact.Safe(i.JobID),
redact.SafeString(i.Reason))
w.Printf("%s", i.Annotations)
w.Printf("%s; ", levelInfos(i.Input))
w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio)
return
}
outputSize := tablesTotalSize(i.Output.Tables)
w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Printf("%s", i.Annotations)
w.Print(levelInfos(i.Input))
w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Output.Level),
redact.Safe(formatFileNums(i.Output.Tables)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(humanize.Bytes.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
type levelInfos []LevelInfo
@@ -129,6 +159,9 @@ type FlushInfo struct {
Reason string
// Input contains the count of input memtables that were flushed.
Input int
// InputBytes contains the total in-memory size of the memtable(s) that were
// flushed. This size includes skiplist indexing data structures.
InputBytes uint64
// Output contains the ouptut table generated by the flush. The output info
// is empty for the flush begin event.
Output []TableInfo
@@ -168,7 +201,7 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if !i.Ingest {
w.Printf("flushing %d memtable", redact.Safe(i.Input))
w.SafeString(plural)
w.Printf(" to L0")
w.Printf(" (%s) to L0", redact.Safe(humanize.Bytes.Uint64(i.InputBytes)))
} else {
w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural)
}
@@ -180,13 +213,14 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if invariants.Enabled && len(i.IngestLevels) > 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0"))
}
w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
w.Printf("[JOB %d] flushed %d memtable%s (%s) to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.JobID), redact.Safe(i.Input), plural,
redact.Safe(humanize.Bytes.Uint64(i.InputBytes)),
redact.Safe(formatFileNums(i.Output)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(humanize.Bytes.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
} else {
if invariants.Enabled && len(i.IngestLevels) == 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0"))
@@ -198,12 +232,12 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if j > 0 {
w.Printf(" +")
}
w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.IEC.Uint64(file.Size))
w.Printf(" L%d:%s (%s)", level, file.FileNum, humanize.Bytes.Uint64(file.Size))
}
w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
}
@@ -227,7 +261,7 @@ func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), i.FileNum)
}
// ManifestDeleteInfo contains the info for a Manifest deletion event.
@@ -249,7 +283,7 @@ func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), i.FileNum)
}
// TableCreateInfo contains the info for a table creation event.
@@ -269,7 +303,7 @@ func (i TableCreateInfo) String() string {
// SafeFormat implements redact.SafeFormatter.
func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] %s: sstable created %s",
redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
redact.Safe(i.JobID), redact.Safe(i.Reason), i.FileNum)
}
// TableDeleteInfo contains the info for a table deletion event.
@@ -288,10 +322,10 @@ func (i TableDeleteInfo) String() string {
func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] sstable delete error %s: %s",
redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
redact.Safe(i.JobID), i.FileNum, i.Err)
return
}
w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), i.FileNum)
}
// TableIngestInfo contains the info for a table ingestion event.
@@ -337,8 +371,8 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if !i.flushable {
levelStr = fmt.Sprintf("L%d:", t.Level)
}
w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum),
redact.Safe(humanize.Uint64(t.Size)))
w.Printf(" %s%s (%s)", redact.Safe(levelStr), t.FileNum,
redact.Safe(humanize.Bytes.Uint64(t.Size)))
}
}
@@ -399,12 +433,12 @@ func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
}
if i.RecycledFileNum == 0 {
w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), i.FileNum)
return
}
w.Printf("[JOB %d] WAL created %s (recycled %s)",
redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
redact.Safe(i.JobID), i.FileNum, i.RecycledFileNum)
}
// WALDeleteInfo contains the info for a WAL deletion event.
@@ -426,7 +460,7 @@ func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), i.FileNum)
}
// WriteStallBeginInfo contains the info for a write stall begin event.

10
vendor/github.com/cockroachdb/pebble/external_iterator.go generated vendored
View File

@@ -311,7 +311,7 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
base.InternalKeySeqNumMax,
it.opts.LowerBound, it.opts.UpperBound,
&it.hasPrefix, &it.prefixOrFullSeekKey,
true /* onlySets */, &it.rangeKey.internal,
false /* internalKeys */, &it.rangeKey.internal,
)
for i := range rangeKeyIters {
it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])
@@ -319,8 +319,12 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
}
}
if it.rangeKey != nil {
it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter, &it.rangeKeyMasking,
it.opts.LowerBound, it.opts.UpperBound)
it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter,
keyspan.InterleavingIterOpts{
Mask: &it.rangeKeyMasking,
LowerBound: it.opts.LowerBound,
UpperBound: it.opts.UpperBound,
})
it.iter = &it.rangeKey.iiter
}
}

31
vendor/github.com/cockroachdb/pebble/flushable.go generated vendored
View File

@@ -60,7 +60,7 @@ type flushableEntry struct {
// flushable is a memTable, when the reader refs drops to zero, the writer
// refs will already be zero because the memtable will have been flushed and
// that only occurs once the writer refs drops to zero.
readerRefs int32
readerRefs atomic.Int32
// Closure to invoke to release memory accounting.
releaseMemAccounting func()
// unrefFiles, if not nil, should be invoked to decrease the ref count of
@@ -73,7 +73,7 @@ type flushableEntry struct {
}
func (e *flushableEntry) readerRef() {
switch v := atomic.AddInt32(&e.readerRefs, 1); {
switch v := e.readerRefs.Add(1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
@@ -92,7 +92,7 @@ func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) {
func (e *flushableEntry) readerUnrefHelper(
deleteFiles bool, deleteFn func(obsolete []*fileBacking),
) {
switch v := atomic.AddInt32(&e.readerRefs, -1); {
switch v := e.readerRefs.Add(-1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
@@ -117,8 +117,7 @@ type flushableList []*flushableEntry
// ingesting sstables which are added to the flushable list.
type ingestedFlushable struct {
files []physicalMeta
cmp Compare
split Split
comparer *Comparer
newIters tableNewIters
newRangeKeyIters keyspan.TableNewSpanIter
@@ -131,8 +130,7 @@ type ingestedFlushable struct {
func newIngestedFlushable(
files []*fileMetadata,
cmp Compare,
split Split,
comparer *Comparer,
newIters tableNewIters,
newRangeKeyIters keyspan.TableNewSpanIter,
) *ingestedFlushable {
@@ -147,12 +145,11 @@ func newIngestedFlushable(
ret := &ingestedFlushable{
files: physicalFiles,
cmp: cmp,
split: split,
comparer: comparer,
newIters: newIters,
newRangeKeyIters: newRangeKeyIters,
// slice is immutable and can be set once and used many times.
slice: manifest.NewLevelSliceKeySorted(cmp, files),
slice: manifest.NewLevelSliceKeySorted(comparer.Compare, files),
hasRangeKeys: hasRangeKeys,
}
@@ -162,6 +159,7 @@ func newIngestedFlushable(
// TODO(sumeer): ingestedFlushable iters also need to plumb context for
// tracing.
// newIter is part of the flushable interface.
func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
var opts IterOptions
if o != nil {
@@ -172,10 +170,11 @@ func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
// aren't truly levels in the lsm. Right now, the encoding only supports
// L0 sublevels, and the rest of the levels in the lsm.
return newLevelIter(
opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), nil,
opts, s.comparer, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
)
}
// newFlushIter is part of the flushable interface.
func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
// newFlushIter is only used for writing memtables to disk as sstables.
// Since ingested sstables are already present on disk, they don't need to
@@ -199,41 +198,47 @@ func (s *ingestedFlushable) constructRangeDelIter(
return rangeDelIter, nil
}
// newRangeDelIter is part of the flushable interface.
// TODO(bananabrick): Using a level iter instead of a keyspan level iter to
// surface range deletes is more efficient.
func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp,
keyspan.SpanIterOptions{}, s.comparer.Compare,
s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
manifest.KeyTypePoint,
)
}
// newRangeKeyIter is part of the flushable interface.
func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
if !s.containsRangeKeys() {
return nil
}
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters,
keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters,
s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
)
}
// containsRangeKeys is part of the flushable interface.
func (s *ingestedFlushable) containsRangeKeys() bool {
return s.hasRangeKeys
}
// inuseBytes is part of the flushable interface.
func (s *ingestedFlushable) inuseBytes() uint64 {
// inuseBytes is only used when memtables are flushed to disk as sstables.
panic("pebble: not implemented")
}
// totalBytes is part of the flushable interface.
func (s *ingestedFlushable) totalBytes() uint64 {
// We don't allocate additional bytes for the ingestedFlushable.
return 0
}
// readyForFlush is part of the flushable interface.
func (s *ingestedFlushable) readyForFlush() bool {
// ingestedFlushable should always be ready to flush. However, note that
// memtables before the ingested sstables in the memtable queue must be

68
vendor/github.com/cockroachdb/pebble/format_major_version.go generated vendored
View File

@@ -28,10 +28,13 @@ import (
// FormatVersion that the default corresponds to may change with time.
type FormatMajorVersion uint64
// SafeValue implements redact.SafeValue.
func (v FormatMajorVersion) SafeValue() {}
// String implements fmt.Stringer.
func (v FormatMajorVersion) String() string {
// NB: This must not change. It's used as the value for the the
// on-disk version marker file.
// NB: This must not change. It's used as the value for the on-disk
// version marker file.
//
// Specifically, this value must always parse as a base 10 integer
// that fits in a uint64. We format it as zero-padded, 3-digit
@@ -110,12 +113,12 @@ const (
// 23.1 versions.
// FormatUnusedPrePebblev1MarkedCompacted is an unused format major version.
// formatUnusedPrePebblev1MarkedCompacted is an unused format major version.
// This format major version was originally intended to ship in the 23.1
// release. It was later decided that this should be deferred until a
// subsequent release. The original ordering is preserved so as not to
// introduce breaking changes in Cockroach.
FormatUnusedPrePebblev1MarkedCompacted
formatUnusedPrePebblev1MarkedCompacted
// FormatSSTableValueBlocks is a format major version that adds support for
// storing values in value blocks in the sstable. Value block support is not
@@ -151,20 +154,20 @@ const (
// compactions for files marked for compaction are complete.
FormatPrePebblev1MarkedCompacted
// ExperimentalFormatDeleteSizedAndObsolete is a format major version that adds support
// FormatDeleteSizedAndObsolete is a format major version that adds support
// for deletion tombstones that encode the size of the value they're
// expected to delete. This format major version is required before the
// associated key kind may be committed through batch applications or
// ingests. It also adds support for keys that are marked obsolete (see
// sstable/format.go for details).
ExperimentalFormatDeleteSizedAndObsolete
FormatDeleteSizedAndObsolete
// ExperimentalFormatVirtualSSTables is a format major version that adds support for
// FormatVirtualSSTables is a format major version that adds support for
// virtual sstables that can reference a sub-range of keys in an underlying
// physical sstable. This information is persisted through new,
// backward-incompatible fields in the Manifest, and therefore requires
// a format major version.
ExperimentalFormatVirtualSSTables
FormatVirtualSSTables
// internalFormatNewest holds the newest format major version, including
// experimental ones excluded from the exported FormatNewest constant until
@@ -172,8 +175,7 @@ const (
internalFormatNewest FormatMajorVersion = iota - 1
// FormatNewest always contains the most recent format major version.
// TODO(jackson): restore FormatNewest to internalFormatNewest.
FormatNewest FormatMajorVersion = FormatPrePebblev1MarkedCompacted
FormatNewest FormatMajorVersion = internalFormatNewest
)
// MaxTableFormat returns the maximum sstable.TableFormat that can be used at
@@ -187,11 +189,11 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
FormatSplitUserKeysMarkedCompacted:
return sstable.TableFormatPebblev1
case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatUnusedPrePebblev1MarkedCompacted:
formatUnusedPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev2
case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev3
case ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
return sstable.TableFormatPebblev4
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -208,15 +210,27 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
FormatRangeKeys:
return sstable.TableFormatLevelDB
case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
return sstable.TableFormatPebblev1
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
}
}
// orderingInvariants returns an enum encoding the set of invariants that must
// hold within the receiver format major version. Invariants only get stricter
// as the format major version advances, so it is okay to retrieve the
// invariants from the current format major version and by the time the
// invariants are enforced, the format major version has advanced.
func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants {
if v < FormatSplitUserKeysMarkedCompacted {
return manifest.AllowSplitUserKeys
}
return manifest.ProhibitSplitUserKeys
}
// formatMajorVersionMigrations defines the migrations from one format
// major version to the next. Each migration is defined as a closure
// which will be invoked on the database before the new format major
@@ -324,9 +338,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
},
FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
// Intentional no-op.
return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted)
return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted)
},
FormatSSTableValueBlocks: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
@@ -343,11 +357,11 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
},
ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete)
FormatDeleteSizedAndObsolete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete)
},
ExperimentalFormatVirtualSSTables: func(d *DB) error {
return d.finalizeFormatVersUpgrade(ExperimentalFormatVirtualSSTables)
FormatVirtualSSTables: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
},
}
@@ -382,9 +396,7 @@ func lookupFormatMajorVersion(
// provided in Options when the database was opened if the existing
// database was written with a higher format version.
func (d *DB) FormatMajorVersion() FormatMajorVersion {
d.mu.Lock()
defer d.mu.Unlock()
return d.mu.formatVers.vers
return FormatMajorVersion(d.mu.formatVers.vers.Load())
}
// RatchetFormatMajorVersion ratchets the opened database's format major
@@ -411,9 +423,9 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
// Guard against accidentally forgetting to update internalFormatNewest.
return errors.Errorf("pebble: unknown format version %d", formatVers)
}
if d.mu.formatVers.vers > formatVers {
if currentVers := d.FormatMajorVersion(); currentVers > formatVers {
return errors.Newf("pebble: database already at format major version %d; cannot reduce to %d",
d.mu.formatVers.vers, formatVers)
currentVers, formatVers)
}
if d.mu.formatVers.ratcheting {
return errors.Newf("pebble: database format major version upgrade is in-progress")
@@ -421,7 +433,7 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
d.mu.formatVers.ratcheting = true
defer func() { d.mu.formatVers.ratcheting = false }()
for nextVers := d.mu.formatVers.vers + 1; nextVers <= formatVers; nextVers++ {
for nextVers := d.FormatMajorVersion() + 1; nextVers <= formatVers; nextVers++ {
if err := formatMajorVersionMigrations[nextVers](d); err != nil {
return errors.Wrapf(err, "migrating to version %d", nextVers)
}
@@ -432,7 +444,7 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
// update in-memory state (without ever dropping locks) after
// the upgrade is finalized. Here we assert that the upgrade
// did occur.
if d.mu.formatVers.vers != nextVers {
if d.FormatMajorVersion() != nextVers {
d.opts.Logger.Fatalf("pebble: successful migration to format version %d never finalized the upgrade", nextVers)
}
}
@@ -451,7 +463,7 @@ func (d *DB) finalizeFormatVersUpgrade(formatVers FormatMajorVersion) error {
if err := d.mu.formatVers.marker.Move(formatVers.String()); err != nil {
return err
}
d.mu.formatVers.vers = formatVers
d.mu.formatVers.vers.Store(uint64(formatVers))
d.opts.EventListener.FormatUpgrade(formatVers)
return nil
}

41
vendor/github.com/cockroachdb/pebble/get_iter.go generated vendored
View File

@@ -19,8 +19,7 @@ import (
// lazily.
type getIter struct {
logger Logger
cmp Compare
equal Equal
comparer *Comparer
newIters tableNewIters
snapshot uint64
key []byte
@@ -83,7 +82,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
// key. Every call to levelIter.Next() potentially switches to a new
// table and thus reinitializes rangeDelIter.
if g.rangeDelIter != nil {
g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key)
g.tombstone = keyspan.Get(g.comparer.Compare, g.rangeDelIter, g.key)
if g.err = g.rangeDelIter.Close(); g.err != nil {
return nil, base.LazyValue{}
}
@@ -101,7 +100,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
g.iter = nil
return nil, base.LazyValue{}
}
if g.equal(g.key, key.UserKey) {
if g.comparer.Equal(g.key, key.UserKey) {
if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) {
g.iterKey, g.iterValue = g.iter.Next()
continue
@@ -159,11 +158,24 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
files := g.l0[n-1].Iter()
g.l0 = g.l0[:n-1]
iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
files, manifest.L0Sublevel(n), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
bc := levelIterBoundaryContext{}
g.levelIter.initBoundaryContext(&bc)
g.iter = &g.levelIter
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
// Compute the key prefix for bloom filtering if split function is
// specified, or use the user key as default.
prefix := g.key
if g.comparer.Split != nil {
prefix = g.key[:g.comparer.Split(g.key)]
}
g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
g.iterKey = nil
g.iterValue = base.LazyValue{}
}
continue
}
g.level++
@@ -178,12 +190,25 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
}
iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
bc := levelIterBoundaryContext{}
g.levelIter.initBoundaryContext(&bc)
g.level++
g.iter = &g.levelIter
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
// Compute the key prefix for bloom filtering if split function is
// specified, or use the user key as default.
prefix := g.key
if g.comparer.Split != nil {
prefix = g.key[:g.comparer.Split(g.key)]
}
g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
g.iterKey = nil
g.iterValue = base.LazyValue{}
}
}
}

1024
vendor/github.com/cockroachdb/pebble/ingest.go generated vendored
View File

File diff suppressed because it is too large Load Diff

56
vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go generated vendored
View File

@@ -18,22 +18,21 @@
package arenaskl
import (
"math"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/constants"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Arena is lock-free.
type Arena struct {
n uint64
n atomic.Uint64
buf []byte
}
const (
align4 = 3
)
const nodeAlignment = 4
var (
// ErrArenaFull indicates that the arena is full and cannot perform any more
@@ -44,20 +43,28 @@ var (
// NewArena allocates a new arena using the specified buffer as the backing
// store.
func NewArena(buf []byte) *Arena {
// Don't store data at position 0 in order to reserve offset=0 as a kind
// of nil pointer.
return &Arena{
n: 1,
if len(buf) > constants.MaxUint32OrInt {
if invariants.Enabled {
panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
}
buf = buf[:constants.MaxUint32OrInt]
}
a := &Arena{
buf: buf,
}
// We don't store data at position 0 in order to reserve offset=0 as a kind of
// nil pointer.
a.n.Store(1)
return a
}
// Size returns the number of bytes allocated by the arena.
func (a *Arena) Size() uint32 {
s := atomic.LoadUint64(&a.n)
if s > math.MaxUint32 {
// Saturate at MaxUint32.
return math.MaxUint32
s := a.n.Load()
if s > constants.MaxUint32OrInt {
// The last failed allocation can push the size higher than len(a.buf).
// Saturate at the maximum representable offset.
return constants.MaxUint32OrInt
}
return uint32(s)
}
@@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 {
return uint32(len(a.buf))
}
func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
// alloc allocates a buffer of the given size and with the given alignment
// (which must be a power of 2).
//
// If overflow is not 0, it also ensures that many bytes after the buffer are
// inside the arena (this is used for structures that are larger than the
// requested size but don't use those extra bytes).
func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
if invariants.Enabled && (alignment&(alignment-1)) != 0 {
panic(errors.AssertionFailedf("invalid alignment %d", alignment))
}
// Verify that the arena isn't already full.
origSize := atomic.LoadUint64(&a.n)
origSize := a.n.Load()
if int(origSize) > len(a.buf) {
return 0, 0, ErrArenaFull
}
// Pad the allocation with enough bytes to ensure the requested alignment.
padded := uint32(size) + align
padded := uint64(size) + uint64(alignment) - 1
newSize := atomic.AddUint64(&a.n, uint64(padded))
if int(newSize)+int(overflow) > len(a.buf) {
newSize := a.n.Add(padded)
if newSize+uint64(overflow) > uint64(len(a.buf)) {
return 0, 0, ErrArenaFull
}
// Return the aligned offset.
offset := (uint32(newSize) - padded + align) & ^align
return offset, padded, nil
offset := (uint32(newSize) - size) & ^(alignment - 1)
return offset, uint32(padded), nil
}
func (a *Arena) getBytes(offset uint32, size uint32) []byte {

14
vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go generated vendored
View File

@@ -18,7 +18,6 @@
package arenaskl
import (
"encoding/binary"
"sync"
"github.com/cockroachdb/pebble/internal/base"
@@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
}
func (it *Iterator) decodeKey() {
b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
// This is a manual inline of base.DecodeInternalKey, because the Go compiler
// seems to refuse to automatically inline it currently.
l := len(b) - 8
if l >= 0 {
it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
it.key.UserKey = b[:l:l]
} else {
it.key.Trailer = uint64(base.InternalKeyKindInvalid)
it.key.UserKey = nil
}
it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
it.key.Trailer = it.nd.keyTrailer
}
func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {

36
vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go generated vendored
View File

@@ -29,25 +29,27 @@ import (
// is used here. If a key/value overflows a uint32, it should not be added to
// the skiplist.
func MaxNodeSize(keySize, valueSize uint32) uint64 {
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
const maxPadding = nodeAlignment - 1
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
}
type links struct {
nextOffset uint32
prevOffset uint32
nextOffset atomic.Uint32
prevOffset atomic.Uint32
}
func (l *links) init(prevOffset, nextOffset uint32) {
l.nextOffset = nextOffset
l.prevOffset = prevOffset
l.nextOffset.Store(nextOffset)
l.prevOffset.Store(prevOffset)
}
type node struct {
// Immutable fields, so no need to lock to access key.
keyOffset uint32
keySize uint32
valueSize uint32
allocSize uint32
keyOffset uint32
keySize uint32
keyTrailer uint64
valueSize uint32
allocSize uint32
// Most nodes do not need to use the full height of the tower, since the
// probability of each successive level decreases exponentially. Because
@@ -65,7 +67,7 @@ func newNode(
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
keySize := key.Size()
keySize := len(key.UserKey)
if int64(keySize) > math.MaxUint32 {
panic("key is too large")
}
@@ -81,8 +83,8 @@ func newNode(
if err != nil {
return
}
key.Encode(nd.getKeyBytes(arena))
nd.keyTrailer = key.Trailer
copy(nd.getKeyBytes(arena), key.UserKey)
copy(nd.getValue(arena), value)
return
}
@@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod
unusedSize := uint32((maxHeight - int(height)) * linksSize)
nodeSize := uint32(maxNodeSize) - unusedSize
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
if err != nil {
return
}
@@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte {
}
func (n *node) nextOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].nextOffset)
return n.tower[h].nextOffset.Load()
}
func (n *node) prevOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].prevOffset)
return n.tower[h].prevOffset.Load()
}
func (n *node) casNextOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
return n.tower[h].nextOffset.CompareAndSwap(old, val)
}
func (n *node) casPrevOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
return n.tower[h].prevOffset.CompareAndSwap(old, val)
}

51
vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go generated vendored
View File

@@ -44,7 +44,6 @@ Key differences:
package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
import (
"encoding/binary"
"math"
"runtime"
"sync/atomic"
@@ -80,7 +79,7 @@ type Skiplist struct {
cmp base.Compare
head *node
tail *node
height uint32 // Current height. 1 <= height <= maxHeight. CAS.
height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
// If set to true by tests, then extra delays are added to make it easier to
// detect unusual race conditions.
@@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
headOffset := arena.getPointerOffset(unsafe.Pointer(head))
tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
for i := 0; i < maxHeight; i++ {
head.tower[i].nextOffset = tailOffset
tail.tower[i].prevOffset = headOffset
head.tower[i].nextOffset.Store(tailOffset)
tail.tower[i].prevOffset.Store(headOffset)
}
*s = Skiplist{
arena: arena,
cmp: cmp,
head: head,
tail: tail,
height: 1,
arena: arena,
cmp: cmp,
head: head,
tail: tail,
}
s.height.Store(1)
}
// Height returns the height of the highest tower within any of the nodes that
// have ever been allocated as part of this skiplist.
func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
func (s *Skiplist) Height() uint32 { return s.height.Load() }
// Arena returns the arena backing this skiplist.
func (s *Skiplist) Arena() *Arena { return s.arena }
@@ -327,7 +326,7 @@ func (s *Skiplist) newNode(
// Try to increase s.height via CAS.
listHeight := s.Height()
for height > listHeight {
if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
if s.height.CompareAndSwap(listHeight, height) {
// Successfully increased skiplist.height.
break
}
@@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel(
offset, size := next.keyOffset, next.keySize
nextKey := s.arena.buf[offset : offset+size]
n := int32(size) - 8
cmp := s.cmp(key.UserKey, nextKey[:n])
cmp := s.cmp(key.UserKey, nextKey)
if cmp < 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
if cmp == 0 {
// User-key equality.
var nextTrailer uint64
if n >= 0 {
nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
} else {
nextTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == nextTrailer {
if key.Trailer == next.keyTrailer {
// Internal key equality.
found = true
break
}
if key.Trailer > nextTrailer {
if key.Trailer > next.keyTrailer {
// We are done for this level, since prev.key < key < next.key.
break
}
@@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel(
func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
n := int32(nd.keySize) - 8
cmp := s.cmp(ndKey[:n], key.UserKey)
cmp := s.cmp(ndKey, key.UserKey)
if cmp < 0 {
return true
}
@@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
return false
}
// User-key equality.
var ndTrailer uint64
if n >= 0 {
ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
} else {
ndTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == ndTrailer {
if key.Trailer == nd.keyTrailer {
// Internal key equality.
return false
}
return key.Trailer < ndTrailer
return key.Trailer < nd.keyTrailer
}
func (s *Skiplist) getNext(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
offset := nd.tower[h].nextOffset.Load()
return (*node)(s.arena.getPointer(offset))
}
func (s *Skiplist) getPrev(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
offset := nd.tower[h].prevOffset.Load()
return (*node)(s.arena.getPointer(offset))
}

10
vendor/github.com/cockroachdb/pebble/internal/base/filenames.go generated vendored
View File

@@ -20,6 +20,11 @@ type FileNum uint64
// String returns a string representation of the file number.
func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
// SafeFormat implements redact.SafeFormatter.
func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("%06d", redact.SafeUint(fn))
}
// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
// called if the caller can ensure that the FileNum belongs to a physical file
// on disk. These could be manifests, log files, physical sstables on disk, the
@@ -40,6 +45,11 @@ type DiskFileNum struct {
func (dfn DiskFileNum) String() string { return dfn.fn.String() }
// SafeFormat implements redact.SafeFormatter.
func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
dfn.fn.SafeFormat(w, verb)
}
// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
func (dfn DiskFileNum) FileNum() FileNum {
return dfn.fn

21
vendor/github.com/cockroachdb/pebble/internal/base/internal.go generated vendored
View File

@@ -9,6 +9,17 @@ import (
"fmt"
"strconv"
"strings"
"github.com/cockroachdb/redact"
)
const (
// SeqNumZero is the zero sequence number, set by compactions if they can
// guarantee there are no keys underneath an internal key.
SeqNumZero = uint64(0)
// SeqNumStart is the first sequence number assigned to a key. Sequence
// numbers 1-9 are reserved for potential future use.
SeqNumStart = uint64(10)
)
// InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
@@ -152,6 +163,11 @@ func (k InternalKeyKind) String() string {
return fmt.Sprintf("UNKNOWN:%d", k)
}
// SafeFormat implements redact.SafeFormatter.
func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
w.Print(redact.SafeString(k.String()))
}
// InternalKey is a key used for the in-memory and on-disk partial DBs that
// make up a pebble DB.
//
@@ -365,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
return k.Trailer >> 8
}
// SeqNumFromTrailer returns the sequence number component of a trailer.
func SeqNumFromTrailer(t uint64) uint64 {
return t >> 8
}
// Visible returns true if the key is visible at the specified snapshot
// sequence number.
func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {

35
vendor/github.com/cockroachdb/pebble/internal/base/iterator.go generated vendored
View File

@@ -7,6 +7,9 @@ package base
import (
"fmt"
"time"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/redact"
)
// InternalIterator iterates over a DB's key/value pairs in key order. Unlike
@@ -407,3 +410,35 @@ func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
}
func (s *InternalIteratorStats) String() string {
return redact.StringWithoutMarkers(s)
}
// SafeFormat implements the redact.SafeFormatter interface.
func (s *InternalIteratorStats) SafeFormat(p redact.SafePrinter, verb rune) {
p.Printf("blocks: %s cached",
humanize.Bytes.Uint64(s.BlockBytesInCache),
)
if s.BlockBytes != s.BlockBytesInCache || s.BlockReadDuration != 0 {
p.Printf(", %s not cached (read time: %s)",
humanize.Bytes.Uint64(s.BlockBytes-s.BlockBytesInCache),
humanize.FormattedString(s.BlockReadDuration.String()),
)
}
p.Printf("; points: %s", humanize.Count.Uint64(s.PointCount))
if s.PointsCoveredByRangeTombstones != 0 {
p.Printf("(%s tombstoned)", humanize.Count.Uint64(s.PointsCoveredByRangeTombstones))
}
p.Printf(" (%s keys, %s values)",
humanize.Bytes.Uint64(s.KeyBytes),
humanize.Bytes.Uint64(s.ValueBytes),
)
if s.SeparatedPointValue.Count != 0 {
p.Printf("; separated: %s (%s, %s fetched)",
humanize.Count.Uint64(s.SeparatedPointValue.Count),
humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytes),
humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytesFetched))
}
}

56
vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go generated vendored
View File

@@ -1,56 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "fmt"
// This file defines sequence numbers that are reserved for foreign keys i.e.
// internal keys coming from other Pebble instances and existing in shared
// storage, as those will "slot below" any internal keys added by our own Pebble
// instance. Any keys created by this Pebble instance need to be greater than
// all reserved sequence numbers (i.e. >= SeqNumStart).
const (
// SeqNumZero is the zero sequence number, set by compactions if they can
// guarantee there are no keys underneath an internal key.
SeqNumZero = uint64(0)
// SeqNumL6 is the sequence number reserved for foreign keys in L6. This seqnum
// will be used to expose any range key sets as well as point keys in L6. Range
// deletes do not need to be exposed in L6.
SeqNumL6 = uint64(1)
// SeqNumL5 is the sequence number reserved for foreign keys in L5. This seqnum
// needs to be greater than SeqNumL6, so that range deletes in L5 can be
// exposed at this level and will correctly delete covering points in L6. Also
// range key unsets/dels will be exposed at this seqnum and will need to shadow
// overlapping range keys in L6.
//
// Note that we can use the same sequence number for all exposed keys in L5 as
// range dels do not delete points at the same seqnum, and range key
// unsets/deletes do not coalesce with range key sets at the same seqnum. Range
// key masking does not care about the sequence number of overlapping points
// (rather, it applies to points based on suffixes), so we can use this seqnum
// for all L5 keys exposed.
SeqNumL5 = uint64(2)
// Sequence numbers 3-9 are reserved for future use.
// SeqNumStart is the first sequence number assigned to a key written by
// ourselves.
SeqNumStart = uint64(10)
)
// SeqNumForLevel returns the appropriate reserved sequence number for keys in
// foreign sstables at the specified level.
func SeqNumForLevel(level int) uint64 {
switch level {
case 5:
return SeqNumL5
case 6:
return SeqNumL6
default:
panic(fmt.Sprintf("unexpected foreign sstable at level %d", level))
}
}

15
vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go generated vendored
View File

@@ -63,14 +63,15 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/constants"
"golang.org/x/exp/rand"
)
const (
maxHeight = 20
maxNodeSize = int(unsafe.Sizeof(node{}))
linksSize = int(unsafe.Sizeof(links{}))
maxNodesSize = math.MaxUint32
maxNodeSize = uint64(unsafe.Sizeof(node{}))
linksSize = uint64(unsafe.Sizeof(links{}))
maxNodesSize = constants.MaxUint32OrInt
)
var (
@@ -281,7 +282,7 @@ func (s *Skiplist) newNode(
panic("height cannot be less than one or greater than the max height")
}
unusedSize := (maxHeight - int(height)) * linksSize
unusedSize := uint64(maxHeight-int(height)) * linksSize
nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
if err != nil {
return 0, err
@@ -296,13 +297,13 @@ func (s *Skiplist) newNode(
}
func (s *Skiplist) alloc(size uint32) (uint32, error) {
offset := len(s.nodes)
offset := uint64(len(s.nodes))
// We only have a need for memory up to offset + size, but we never want
// to allocate a node whose tail points into unallocated memory.
minAllocSize := offset + maxNodeSize
if cap(s.nodes) < minAllocSize {
allocSize := cap(s.nodes) * 2
if uint64(cap(s.nodes)) < minAllocSize {
allocSize := uint64(cap(s.nodes)) * 2
if allocSize < minAllocSize {
allocSize = minAllocSize
}

183
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go generated vendored
View File

@@ -71,14 +71,12 @@ func (h Handle) Get() []byte {
// Release releases the reference to the cache entry.
func (h Handle) Release() {
if h.value != nil {
h.value.release()
}
h.value.release()
}
type shard struct {
hits int64
misses int64
hits atomic.Int64
misses atomic.Int64
mu sync.RWMutex
@@ -119,15 +117,15 @@ func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
value = e.acquireValue()
if value != nil {
atomic.StoreInt32(&e.referenced, 1)
e.referenced.Store(true)
}
}
c.mu.RUnlock()
if value == nil {
atomic.AddInt64(&c.misses, 1)
c.misses.Add(1)
return Handle{}
}
atomic.AddInt64(&c.hits, 1)
c.hits.Add(1)
return Handle{value: value}
}
@@ -160,7 +158,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
case e.peekValue() != nil:
// cache entry was a hot or cold page
e.setValue(value)
atomic.StoreInt32(&e.referenced, 1)
e.referenced.Store(true)
delta := int64(len(value.buf)) - e.size
e.size = int64(len(value.buf))
if e.ptype == etHot {
@@ -176,7 +174,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
// cache entry was a test page
c.sizeTest -= e.size
c.countTest--
c.metaDel(e)
c.metaDel(e).release()
c.metaCheck(e)
e.size = int64(len(value.buf))
@@ -185,7 +183,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
c.coldTarget = c.targetSize()
}
atomic.StoreInt32(&e.referenced, 0)
e.referenced.Store(false)
e.setValue(value)
e.ptype = etHot
if c.metaAdd(k, e) {
@@ -233,37 +231,75 @@ func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
return
}
c.mu.Lock()
defer c.mu.Unlock()
var deletedValue *Value
func() {
c.mu.Lock()
defer c.mu.Unlock()
e := c.blocks.Get(k)
if e == nil {
return
}
c.metaEvict(e)
c.checkConsistency()
e := c.blocks.Get(k)
if e == nil {
return
}
deletedValue = c.metaEvict(e)
c.checkConsistency()
}()
// Now that the mutex has been dropped, release the reference which will
// potentially free the memory associated with the previous cached value.
deletedValue.release()
}
// EvictFile evicts all of the cache values for the specified file.
func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
c.mu.Lock()
defer c.mu.Unlock()
fkey := key{fileKey{id, fileNum}, 0}
for c.evictFileRun(fkey) {
// Sched switch to give another goroutine an opportunity to acquire the
// shard mutex.
runtime.Gosched()
}
}
func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
// If most of the file's blocks are held in the block cache, evicting all
// the blocks may take a while. We don't want to block the entire cache
// shard, forcing concurrent readers to wait until we're finished. We drop
// the mutex every [blocksPerMutexAcquisition] blocks to give other
// goroutines an opportunity to make progress.
const blocksPerMutexAcquisition = 5
c.mu.Lock()
// Releasing a value may result in free-ing it back to the memory allocator.
// This can have a nontrivial cost that we'd prefer to not pay while holding
// the shard mutex, so we collect the evicted values in a local slice and
// only release them in a defer after dropping the cache mutex.
var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
obsoleteValues := obsoleteValuesAlloc[:0]
defer func() {
c.mu.Unlock()
for _, v := range obsoleteValues {
v.release()
}
}()
blocks := c.files.Get(fkey)
if blocks == nil {
return
}
for b, n := blocks, (*entry)(nil); ; b = n {
n = b.fileLink.next
c.metaEvict(b)
if b == n {
break
}
// No blocks for this file.
return false
}
c.checkConsistency()
// b is the current head of the doubly linked list, and n is the entry after b.
for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
n = b.fileLink.next
obsoleteValues = append(obsoleteValues, c.metaEvict(b))
if b == n {
// b == n represents the case where b was the last entry remaining
// in the doubly linked list, which is why it pointed at itself. So
// no more entries left.
c.checkConsistency()
return false
}
}
// Exhausted blocksPerMutexAcquisition.
return true
}
func (c *shard) Free() {
@@ -274,7 +310,7 @@ func (c *shard) Free() {
// metaCheck call when the "invariants" build tag is specified.
for c.handHot != nil {
e := c.handHot
c.metaDel(c.handHot)
c.metaDel(c.handHot).release()
e.free()
}
@@ -359,12 +395,14 @@ func (c *shard) metaAdd(key key, e *entry) bool {
// Remove the entry from the cache. This removes the entry from the blocks map,
// the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
// entry.
func (c *shard) metaDel(e *entry) {
// entry. Returns the deleted value that must be released, if any.
func (c *shard) metaDel(e *entry) (deletedValue *Value) {
if value := e.peekValue(); value != nil {
value.ref.trace("metaDel")
}
e.setValue(nil)
// Remove the pointer to the value.
deletedValue = e.val
e.val = nil
c.blocks.Delete(e.key)
if entriesGoAllocated {
@@ -396,6 +434,7 @@ func (c *shard) metaDel(e *entry) {
} else {
c.files.Put(fkey, next)
}
return deletedValue
}
// Check that the specified entry is not referenced by the cache.
@@ -455,7 +494,7 @@ func (c *shard) metaCheck(e *entry) {
}
}
func (c *shard) metaEvict(e *entry) {
func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
switch e.ptype {
case etHot:
c.sizeHot -= e.size
@@ -467,9 +506,10 @@ func (c *shard) metaEvict(e *entry) {
c.sizeTest -= e.size
c.countTest--
}
c.metaDel(e)
evictedValue = c.metaDel(e)
c.metaCheck(e)
e.free()
return evictedValue
}
func (c *shard) evict() {
@@ -491,8 +531,8 @@ func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
e := c.handCold
if e.ptype == etCold {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
if e.referenced.Load() {
e.referenced.Store(false)
e.ptype = etHot
c.sizeCold -= e.size
c.countCold--
@@ -528,8 +568,8 @@ func (c *shard) runHandHot() {
e := c.handHot
if e.ptype == etHot {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
if e.referenced.Load() {
e.referenced.Store(false)
} else {
e.ptype = etCold
c.sizeHot -= e.size
@@ -564,7 +604,7 @@ func (c *shard) runHandTest() {
if c.coldTarget < 0 {
c.coldTarget = 0
}
c.metaDel(e)
c.metaDel(e).release()
c.metaCheck(e)
e.free()
}
@@ -587,7 +627,7 @@ type Metrics struct {
// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
// used for page replacement
// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
// order to provide better concurrency, 2 x NumCPUs shards are created, with
// order to provide better concurrency, 4 x NumCPUs shards are created, with
// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
// is run independently on each shard.
//
@@ -621,9 +661,9 @@ type Metrics struct {
// used in combination by specifying `-tags invariants,tracing`. Note that
// "tracing" produces a significant slowdown, while "invariants" does not.
type Cache struct {
refs int64
refs atomic.Int64
maxSize int64
idAlloc uint64
idAlloc atomic.Uint64
shards []shard
// Traces recorded by Cache.trace. Used for debugging.
@@ -643,17 +683,44 @@ type Cache struct {
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func New(size int64) *Cache {
return newShards(size, 2*runtime.GOMAXPROCS(0))
// How many cache shards should we create?
//
// Note that the probability two processors will try to access the same
// shard at the same time increases superlinearly with the number of
// processors (Eg, consider the brithday problem where each CPU is a person,
// and each shard is a possible birthday).
//
// We could consider growing the number of shards superlinearly, but
// increasing the shard count may reduce the effectiveness of the caching
// algorithm if frequently-accessed blocks are insufficiently distributed
// across shards. If a shard's size is smaller than a single frequently
// scanned sstable, then the shard will be unable to hold the entire
// frequently-scanned table in memory despite other shards still holding
// infrequently accessed blocks.
//
// Experimentally, we've observed contention contributing to tail latencies
// at 2 shards per processor. For now we use 4 shards per processor,
// recognizing this may not be final word.
m := 4 * runtime.GOMAXPROCS(0)
// In tests we can use large CPU machines with small cache sizes and have
// many caches in existence at a time. If sharding into m shards would
// produce too small shards, constrain the number of shards to 4.
const minimumShardSize = 4 << 20 // 4 MiB
if m > 4 && int(size)/m < minimumShardSize {
m = 4
}
return newShards(size, m)
}
func newShards(size int64, shards int) *Cache {
c := &Cache{
refs: 1,
maxSize: size,
idAlloc: 1,
shards: make([]shard, shards),
}
c.trace("alloc", c.refs)
c.refs.Store(1)
c.idAlloc.Store(1)
c.trace("alloc", c.refs.Load())
for i := range c.shards {
c.shards[i] = shard{
maxSize: size / int64(len(c.shards)),
@@ -669,7 +736,7 @@ func newShards(size int64, shards int) *Cache {
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(c, func(obj interface{}) {
c := obj.(*Cache)
if v := atomic.LoadInt64(&c.refs); v != 0 {
if v := c.refs.Load(); v != 0 {
c.tr.Lock()
fmt.Fprintf(os.Stderr,
"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
@@ -716,7 +783,7 @@ func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *sh
// Ref adds a reference to the cache. The cache only remains valid as long a
// reference is maintained to it.
func (c *Cache) Ref() {
v := atomic.AddInt64(&c.refs, 1)
v := c.refs.Add(1)
if v <= 1 {
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
@@ -725,7 +792,7 @@ func (c *Cache) Ref() {
// Unref releases a reference on the cache.
func (c *Cache) Unref() {
v := atomic.AddInt64(&c.refs, -1)
v := c.refs.Add(-1)
c.trace("unref", v)
switch {
case v < 0:
@@ -785,14 +852,14 @@ func (c *Cache) Size() int64 {
// manually managed. The caller MUST either add the value to the cache (via
// Cache.Set), or release the value (via Cache.Free). Failure to do so will
// result in a memory leak.
func (c *Cache) Alloc(n int) *Value {
func Alloc(n int) *Value {
return newValue(n)
}
// Free frees the specified value. The buffer associated with the value will
// possibly be reused, making it invalid to use the buffer after calling
// Free. Do not call Free on a value that has been added to the cache.
func (c *Cache) Free(v *Value) {
func Free(v *Value) {
if n := v.refs(); n > 1 {
panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
}
@@ -829,8 +896,8 @@ func (c *Cache) Metrics() Metrics {
m.Count += int64(s.blocks.Count())
m.Size += s.sizeHot + s.sizeCold
s.mu.RUnlock()
m.Hits += atomic.LoadInt64(&s.hits)
m.Misses += atomic.LoadInt64(&s.misses)
m.Hits += s.hits.Load()
m.Misses += s.misses.Load()
}
return m
}
@@ -838,5 +905,5 @@ func (c *Cache) Metrics() Metrics {
// NewID returns a new ID to be used as a namespace for cached file
// blocks.
func (c *Cache) NewID() uint64 {
return atomic.AddUint64(&c.idAlloc, 1)
return c.idAlloc.Add(1)
}

8
vendor/github.com/cockroachdb/pebble/internal/cache/entry.go generated vendored
View File

@@ -4,6 +4,8 @@
package cache
import "sync/atomic"
type entryType int8
const (
@@ -56,7 +58,7 @@ type entry struct {
ptype entryType
// referenced is atomically set to indicate that this entry has been accessed
// since the last time one of the clock hands swept it.
referenced int32
referenced atomic.Bool
shard *shard
// Reference count for the entry. The entry is freed when the reference count
// drops to zero.
@@ -137,9 +139,7 @@ func (e *entry) setValue(v *Value) {
}
old := e.val
e.val = v
if old != nil {
old.release()
}
old.release()
}
func (e *entry) peekValue() *Value {

18
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_normal.go generated vendored
View File

@@ -10,33 +10,37 @@ package cache
import (
"fmt"
"sync/atomic"
"github.com/cockroachdb/redact"
)
// refcnt provides an atomic reference count. This version is used when the
// "tracing" build tag is not enabled. See refcnt_tracing.go for the "tracing"
// enabled version.
type refcnt int32
type refcnt struct {
val atomic.Int32
}
// initialize the reference count to the specified value.
func (v *refcnt) init(val int32) {
*v = refcnt(val)
v.val.Store(val)
}
func (v *refcnt) refs() int32 {
return atomic.LoadInt32((*int32)(v))
return v.val.Load()
}
func (v *refcnt) acquire() {
switch v := atomic.AddInt32((*int32)(v), 1); {
switch v := v.val.Add(1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
}
}
func (v *refcnt) release() bool {
switch v := atomic.AddInt32((*int32)(v), -1); {
switch v := v.val.Add(-1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
case v == 0:
return true
default:

10
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_tracing.go generated vendored
View File

@@ -19,22 +19,22 @@ import (
// debugging logic errors in manipulating the reference count. This version is
// used when the "tracing" build tag is enabled.
type refcnt struct {
val int32
val atomic.Int32
sync.Mutex
msgs []string
}
func (v *refcnt) init(val int32) {
v.val = val
v.val.Store(val)
v.trace("init")
}
func (v *refcnt) refs() int32 {
return atomic.LoadInt32(&v.val)
return v.val.Load()
}
func (v *refcnt) acquire() {
switch n := atomic.AddInt32(&v.val, 1); {
switch n := v.val.Add(1); {
case n <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
}
@@ -42,7 +42,7 @@ func (v *refcnt) acquire() {
}
func (v *refcnt) release() bool {
n := atomic.AddInt32(&v.val, -1)
n := v.val.Add(-1)
switch {
case n < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))

2
vendor/github.com/cockroachdb/pebble/internal/cache/value.go generated vendored
View File

@@ -40,7 +40,7 @@ func (v *Value) acquire() {
}
func (v *Value) release() {
if v.ref.release() {
if v != nil && v.ref.release() {
v.free()
}
}

17
vendor/github.com/cockroachdb/pebble/internal/constants/constants.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package constants
const (
// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
oneIf64Bit = ^uint(0) >> 63
// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
// - MaxUint32 on 64-bit platforms;
// - MaxInt on 32-bit platforms.
// It is used when slices are limited to Uint32 on 64-bit platforms (the
// length limit for slices is naturally MaxInt on 32-bit platforms).
MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
)

20
vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go generated vendored
View File

@@ -35,11 +35,11 @@ type config struct {
suffix []string
}
// IEC produces human readable representations of integer values in IEC units.
var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}}
// Bytes produces human readable representations of byte values in IEC units.
var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
// SI produces human readable representations of integer values in SI units.
var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}}
// Count produces human readable representations of unitless values in SI units.
var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
// Int64 produces a human readable representation of the value.
func (c *config) Int64(s int64) FormattedString {
@@ -54,18 +54,6 @@ func (c *config) Uint64(s uint64) FormattedString {
return FormattedString(humanate(s, c.base, c.suffix))
}
// Int64 produces a human readable representation of the value in IEC units
// (base 1024).
func Int64(s int64) FormattedString {
return IEC.Int64(s)
}
// Uint64 produces a human readable representation of the value in IEC units
// (base 1024).
func Uint64(s uint64) FormattedString {
return IEC.Uint64(s)
}
// FormattedString represents a human readable representation of a value. It
// implements the redact.SafeValue interface to signal that it represents a
// a string that does not need to be redacted.

162
vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go generated vendored Normal file
View File

@@ -0,0 +1,162 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package invalidating
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/fastrand"
"github.com/cockroachdb/pebble/internal/invariants"
)
// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
// MaybeWrapIfInvariants does nothing in non-invariant builds.
func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
if invariants.Enabled {
if fastrand.Uint32n(10) == 1 {
return NewIter(iter)
}
}
return iter
}
// iter tests unsafe key/value slice reuse by modifying the last
// returned key/value to all 1s.
type iter struct {
iter base.InternalIterator
lastKey *base.InternalKey
lastValue base.LazyValue
ignoreKinds [base.InternalKeyKindMax + 1]bool
err error
}
// Option configures the behavior of an invalidating iterator.
type Option interface {
apply(*iter)
}
type funcOpt func(*iter)
func (f funcOpt) apply(i *iter) { f(i) }
// IgnoreKinds constructs an Option that configures an invalidating iterator to
// skip trashing k/v pairs with the provided key kinds. Some iterators provided
// key stability guarantees for specific key kinds.
func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
return funcOpt(func(i *iter) {
for _, kind := range kinds {
i.ignoreKinds[kind] = true
}
})
}
// NewIter constructs a new invalidating iterator that wraps the provided
// iterator, trashing buffers for previously returned keys.
func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
i := &iter{iter: originalIterator}
for _, opt := range opts {
opt.apply(i)
}
return i
}
func (i *iter) update(
key *base.InternalKey, value base.LazyValue,
) (*base.InternalKey, base.LazyValue) {
i.trashLastKV()
if key == nil {
i.lastKey = nil
i.lastValue = base.LazyValue{}
return nil, base.LazyValue{}
}
i.lastKey = &base.InternalKey{}
*i.lastKey = key.Clone()
i.lastValue = base.LazyValue{
ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
}
if value.Fetcher != nil {
fetcher := new(base.LazyFetcher)
*fetcher = *value.Fetcher
i.lastValue.Fetcher = fetcher
}
return i.lastKey, i.lastValue
}
func (i *iter) trashLastKV() {
if i.lastKey == nil {
return
}
if i.ignoreKinds[i.lastKey.Kind()] {
return
}
if i.lastKey != nil {
for j := range i.lastKey.UserKey {
i.lastKey.UserKey[j] = 0xff
}
i.lastKey.Trailer = 0xffffffffffffffff
}
for j := range i.lastValue.ValueOrHandle {
i.lastValue.ValueOrHandle[j] = 0xff
}
if i.lastValue.Fetcher != nil {
// Not all the LazyFetcher fields are visible, so we zero out the last
// value's Fetcher struct entirely.
*i.lastValue.Fetcher = base.LazyFetcher{}
}
}
func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekGE(key, flags))
}
func (i *iter) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
}
func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekLT(key, flags))
}
func (i *iter) First() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.First())
}
func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Last())
}
func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Next())
}
func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Prev())
}
func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.NextPrefix(succKey))
}
func (i *iter) Error() error {
if err := i.iter.Error(); err != nil {
return err
}
return i.err
}
func (i *iter) Close() error {
return i.iter.Close()
}
func (i *iter) SetBounds(lower, upper []byte) {
i.iter.SetBounds(lower, upper)
}
func (i *iter) String() string {
return i.iter.String()
}

38
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored
View File

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
// fragments before this span. Defragment backward to find the start of the
// defragmented span.
i.defragmentBackward()
// Defragmenting backward may have stopped because it encountered an error.
// If so, we must not continue so that i.iter.Error() (and thus i.Error())
// yields the error.
if i.iterSpan == nil && i.iter.Error() != nil {
return nil
}
if i.iterPos == iterPosPrev {
// Next once back onto the span.
i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
// fragments after this span. Defragment forward to find the end of the
// defragmented span.
i.defragmentForward()
// Defragmenting forward may have stopped because it encountered an error.
// If so, we must not continue so that i.iter.Error() (and thus i.Error())
// yields the error.
if i.iterSpan == nil && i.iter.Error() != nil {
return nil
}
if i.iterPos == iterPosNext {
// Prev once back onto the span.
i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
// Next once to move onto y, defragment forward to land on the first z
// position.
i.iterSpan = i.iter.Next()
if invariants.Enabled && i.iterSpan == nil {
if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
panic("pebble: invariant violation: no next span while switching directions")
}
// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
// Prev once to move onto y, defragment backward to land on the last x
// position.
i.iterSpan = i.iter.Prev()
if invariants.Enabled && i.iterSpan == nil {
if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
panic("pebble: invariant violation: no previous span while switching directions")
}
// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Next()
}
// i.iterSpan == nil
//
// The inner iterator may return nil when it encounters an error. If there
// was an error, we don't know whether there is another span we should
// defragment or not. Return nil so that the caller knows they should check
// Error().
if i.iter.Error() != nil {
return nil
}
i.curr.Keys = i.keysBuf
return &i.curr
}
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Prev()
}
// i.iterSpan == nil
//
// The inner iterator may return nil when it encounters an error. If there
// was an error, we don't know whether there is another span we should
// defragment or not. Return nil so that the caller knows they should check
// Error().
if i.iter.Error() != nil {
return nil
}
i.curr.Keys = i.keysBuf
return &i.curr
}

25
vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go generated vendored
View File

@@ -4,6 +4,8 @@
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// FilterFunc defines a transform from the input Span into the output Span. The
// function returns true if the Span should be returned by the iterator, and
// false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
type filteringIter struct {
iter FragmentIterator
filterFn FilterFunc
cmp base.Compare
// span is a mutable Span passed to the filterFn. The filterFn is free to
// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
// Filter returns a new filteringIter that will filter the Spans from the
// provided child iterator using the provided FilterFunc.
func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter}
func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
}
// SeekGE implements FragmentIterator.
func (i *filteringIter) SeekGE(key []byte) *Span {
return i.filter(i.iter.SeekGE(key), +1)
span := i.filter(i.iter.SeekGE(key), +1)
// i.filter could return a span that's less than key, _if_ the filterFunc
// (which has no knowledge of the seek key) mutated the span to end at a key
// less than or equal to `key`. Detect this case and next/invalidate the iter.
if span != nil && i.cmp(span.End, key) <= 0 {
return i.Next()
}
return span
}
// SeekLT implements FragmentIterator.
func (i *filteringIter) SeekLT(key []byte) *Span {
return i.filter(i.iter.SeekLT(key), -1)
span := i.filter(i.iter.SeekLT(key), -1)
// i.filter could return a span that's >= key, _if_ the filterFunc (which has
// no knowledge of the seek key) mutated the span to start at a key greater
// than or equal to `key`. Detect this case and prev/invalidate the iter.
if span != nil && i.cmp(span.Start, key) >= 0 {
return i.Prev()
}
return span
}
// First implements FragmentIterator.

925
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored
View File

File diff suppressed because it is too large Load Diff

5
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored
View File

@@ -51,6 +51,8 @@ type FragmentIterator interface {
Prev() *Span
// Error returns any accumulated error.
//
// TODO(jackson): Lift errors into return values on the positioning methods.
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
@@ -70,9 +72,6 @@ type SpanIterOptions struct {
// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
// when iterating over range keys.
RangeKeyFilters []base.BlockPropertyFilter
// Level specifies the level where this sstable is being read. Must be
// specified for foreign (i.e. shared not-created-by-this-instance) sstables.
Level manifest.Level
}
// Iter is an iterator over a set of fragmented spans.

99
vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go generated vendored
View File

@@ -48,102 +48,3 @@ func VisibleTransform(snapshot uint64) Transformer {
return nil
})
}
// TransformerIter is a FragmentIterator that applies a Transformer on all
// returned keys. Used for when a caller needs to apply a transformer on an
// iterator but does not otherwise need the mergingiter's merging ability.
type TransformerIter struct {
FragmentIterator
// Transformer is applied on every Span returned by this iterator.
Transformer Transformer
// Comparer in use for this keyspace.
Compare base.Compare
span Span
err error
}
func (t *TransformerIter) applyTransform(span *Span) *Span {
t.span = Span{
Start: t.span.Start[:0],
End: t.span.End[:0],
Keys: t.span.Keys[:0],
}
if err := t.Transformer.Transform(t.Compare, *span, &t.span); err != nil {
t.err = err
return nil
}
return &t.span
}
// SeekGE implements the FragmentIterator interface.
func (t *TransformerIter) SeekGE(key []byte) *Span {
span := t.FragmentIterator.SeekGE(key)
if span == nil {
return nil
}
return t.applyTransform(span)
}
// SeekLT implements the FragmentIterator interface.
func (t *TransformerIter) SeekLT(key []byte) *Span {
span := t.FragmentIterator.SeekLT(key)
if span == nil {
return nil
}
return t.applyTransform(span)
}
// First implements the FragmentIterator interface.
func (t *TransformerIter) First() *Span {
span := t.FragmentIterator.First()
if span == nil {
return nil
}
return t.applyTransform(span)
}
// Last implements the FragmentIterator interface.
func (t *TransformerIter) Last() *Span {
span := t.FragmentIterator.Last()
if span == nil {
return nil
}
return t.applyTransform(span)
}
// Next implements the FragmentIterator interface.
func (t *TransformerIter) Next() *Span {
span := t.FragmentIterator.Next()
if span == nil {
return nil
}
return t.applyTransform(span)
}
// Prev implements the FragmentIterator interface.
func (t *TransformerIter) Prev() *Span {
span := t.FragmentIterator.Prev()
if span == nil {
return nil
}
return t.applyTransform(span)
}
// Error implements the FragmentIterator interface.
func (t *TransformerIter) Error() error {
if t.err != nil {
return t.err
}
return t.FragmentIterator.Error()
}
// Close implements the FragmentIterator interface.
func (t *TransformerIter) Close() error {
err := t.FragmentIterator.Close()
if err != nil {
return err
}
return t.err
}

9
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored
View File

@@ -14,7 +14,7 @@ func Truncate(
iter FragmentIterator,
lower, upper []byte,
start, end *base.InternalKey,
panicOnPartialOverlap bool,
panicOnUpperTruncate bool,
) FragmentIterator {
return Filter(iter, func(in *Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
@@ -57,7 +57,6 @@ func Truncate(
var truncated bool
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
truncated = true
out.Start = lower
}
if cmp(in.End, upper) > 0 {
@@ -65,10 +64,10 @@ func Truncate(
out.End = upper
}
if panicOnPartialOverlap && truncated {
panic("pebble: bounds should not be truncated")
if panicOnUpperTruncate && truncated {
panic("pebble: upper bound should not be truncated")
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}, cmp)
}

26
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored
View File

@@ -108,7 +108,7 @@ type annotation struct {
}
type leafNode struct {
ref int32
ref atomic.Int32
count int16
leaf bool
// subtreeCount holds the count of files in the entire subtree formed by
@@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node {
func newLeafNode() *node {
n := leafToNode(new(leafNode))
n.leaf = true
n.ref = 1
n.ref.Store(1)
return n
}
func newNode() *node {
n := new(node)
n.ref = 1
n.ref.Store(1)
return n
}
@@ -159,7 +159,7 @@ func newNode() *node {
// When a node is cloned, the provided pointer will be redirected to the new
// mutable node.
func mut(n **node) *node {
if atomic.LoadInt32(&(*n).ref) == 1 {
if (*n).ref.Load() == 1 {
// Exclusive ownership. Can mutate in place.
// Whenever a node will be mutated, reset its annotations to be marked
@@ -186,7 +186,7 @@ func mut(n **node) *node {
// incRef acquires a reference to the node.
func (n *node) incRef() {
atomic.AddInt32(&n.ref, 1)
n.ref.Add(1)
}
// decRef releases a reference to the node. If requested, the method will unref
@@ -195,8 +195,8 @@ func (n *node) incRef() {
// new nodes pass contentsToo=false to preserve existing reference counts during
// operations that should yield a net-zero change to descendant refcounts.
// When a node is released, its contained files are dereferenced.
func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if atomic.AddInt32(&n.ref, -1) > 0 {
func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
if n.ref.Add(-1) > 0 {
// Other references remain. Can't free.
return
}
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if obsolete == nil {
panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
}
*obsolete = append(*obsolete, f)
// Reference counting is performed on the FileBacking. In the case
// of a virtual sstable, this reference counting is performed on
// a FileBacking which is shared by every single virtual sstable
// with the same backing sstable. If the reference count hits 0,
// then we know that the FileBacking won't be required by any
// sstable in Pebble, and that the backing sstable can be deleted.
*obsolete = append(*obsolete, f.FileBacking)
}
}
if !n.leaf {
@@ -761,8 +767,8 @@ type btree struct {
// Release dereferences and clears the root node of the btree, removing all
// items from the btree. In doing so, it decrements contained file counts.
// It returns a slice of newly obsolete files, if any.
func (t *btree) Release() (obsolete []*FileMetadata) {
// It returns a slice of newly obsolete backing files, if any.
func (t *btree) Release() (obsolete []*FileBacking) {
if t.root != nil {
t.root.decRef(true /* contentsToo */, &obsolete)
t.root = nil

9
vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go generated vendored
View File

@@ -712,7 +712,7 @@ func (s *L0Sublevels) calculateFlushSplitKeys(flushSplitMaxBytes int64) {
// InitCompactingFileInfo initializes internal flags relating to compacting
// files. Must be called after sublevel initialization.
//
// Requires DB.mu to be held.
// Requires DB.mu *and* the manifest lock to be held.
func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
for i := range s.orderedIntervals {
s.orderedIntervals[i].compactingFileCount = 0
@@ -735,6 +735,13 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
if !f.IsCompacting() {
continue
}
if invariants.Enabled {
if s.cmp(s.orderedIntervals[f.minIntervalIndex].startKey.key, f.Smallest.UserKey) != 0 || s.cmp(s.orderedIntervals[f.maxIntervalIndex+1].startKey.key, f.Largest.UserKey) != 0 {
panic(fmt.Sprintf("file %s has inconsistent L0 Sublevel interval bounds: %s-%s, %s-%s", f.FileNum,
s.orderedIntervals[f.minIntervalIndex].startKey.key, s.orderedIntervals[f.maxIntervalIndex+1].startKey.key,
f.Smallest.UserKey, f.Largest.UserKey))
}
}
for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
interval := &s.orderedIntervals[i]
interval.compactingFileCount++

77
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored
View File

@@ -15,20 +15,28 @@ import (
// LevelMetadata contains metadata for all of the files within
// a level of the LSM.
type LevelMetadata struct {
level int
tree btree
level int
totalSize uint64
// NumVirtual is the number of virtual sstables in the level.
NumVirtual uint64
// VirtualSize is the size of the virtual sstables in the level.
VirtualSize uint64
tree btree
}
// clone makes a copy of the level metadata, implicitly increasing the ref
// count of every file contained within lm.
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
tree: lm.tree.Clone(),
level: lm.level,
totalSize: lm.totalSize,
NumVirtual: lm.NumVirtual,
VirtualSize: lm.VirtualSize,
tree: lm.tree.Clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
return lm.tree.Release()
}
@@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
var lm LevelMetadata
lm.level = level
lm.tree, _ = makeBTree(bcmp, files)
for _, f := range files {
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
}
return lm
}
@@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
return t, newLevelSlice(t.Iter())
}
func (lm *LevelMetadata) insert(f *FileMetadata) error {
if err := lm.tree.Insert(f); err != nil {
return err
}
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
return nil
}
func (lm *LevelMetadata) remove(f *FileMetadata) bool {
lm.totalSize -= f.Size
if f.Virtual {
lm.NumVirtual--
lm.VirtualSize -= f.Size
}
return lm.tree.Delete(f)
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.Count() == 0
@@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int {
return lm.tree.Count()
}
// Size returns the cumulative size of all the files within the level.
func (lm *LevelMetadata) Size() uint64 {
return lm.totalSize
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.Iter()}
@@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
return sum
}
// NumVirtual returns the number of virtual sstables in the level. Its runtime is
// linear in the length of the slice.
func (ls *LevelSlice) NumVirtual() uint64 {
var n uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
n++
}
}
return n
}
// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
// level.
func (ls *LevelSlice) VirtualSizeSum() uint64 {
var sum uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
sum += f.Size
}
}
return sum
}
// Reslice constructs a new slice backed by the same underlying level, with
// new start and end positions. Reslice invokes the provided function, passing
// two LevelIterators: one positioned to i's inclusive start and one

168
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored
View File

@@ -49,6 +49,9 @@ type TableStats struct {
// The number of point and range deletion entries in the table.
NumDeletions uint64
// NumRangeKeySets is the total number of range key sets in the table.
//
// NB: If there's a chance that the sstable contains any range key sets,
// then NumRangeKeySets must be > 0.
NumRangeKeySets uint64
// Estimate of the total disk space that may be dropped by this table's
// point deletions by compacting them.
@@ -169,22 +172,13 @@ type FileMetadata struct {
// FileNum is the file number.
//
// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
//
// TODO(bananabrick): Consider creating separate types for
// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
// an indentifier for the FileMetadata in Pebble, and also as a handle to
// perform reads and writes. We should ensure through types that
// FileMetadata.FileNum isn't used to perform reads, and that
// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
FileNum base.FileNum
// Size is the size of the file, in bytes. Size is an approximate value for
// virtual sstables.
//
// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
//
// TODO(bananabrick): Size is currently used in metrics, and for many key
// Pebble level heuristics. Make sure that the heuristics will still work
// appropriately with an approximate value of size.
// INVARIANTS:
// - When !FileMetadata.Virtual, Size == FileBacking.Size.
// - Size should be non-zero. Size 0 virtual sstables must not be created.
Size uint64
// File creation time in seconds since the epoch (1970-01-01 00:00:00
// UTC). For ingested sstables, this corresponds to the time the file was
@@ -227,10 +221,6 @@ type FileMetadata struct {
// we'd have to write virtual sstable stats to the version edit.
Stats TableStats
// Deleted is set to true if a VersionEdit gets installed that has deleted
// this file. Protected by the manifest lock (see versionSet.logLock()).
Deleted bool
// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
// pick L0 compactions. Only accurate for the most recent Version.
SubLevel int
@@ -292,6 +282,34 @@ type PhysicalFileMeta struct {
// VirtualFileMeta is used by functions which want a guarantee that their input
// belongs to a virtual sst and not a physical sst.
//
// A VirtualFileMeta inherits all the same fields as a FileMetadata. These
// fields have additional invariants imposed on them, and/or slightly varying
// meanings:
// - Smallest and Largest (and their counterparts
// {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a
// key at that exact bound. We make the effort to determine the next smallest
// or largest key in an sstable after virtualizing it, to maintain this
// tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it
// could mean that a rangedel or range key ends at that user key, or has been
// truncated to that user key.
// - One invariant is that if a rangedel or range key is truncated on its
// upper bound, the virtual sstable *must* have a rangedel or range key
// sentinel key as its upper bound. This is because truncation yields
// an exclusive upper bound for the rangedel/rangekey, and if there are
// any points at that exclusive upper bound within the same virtual
// sstable, those could get uncovered by this truncation. We enforce this
// invariant in calls to keyspan.Truncate.
// - Size is an estimate of the size of the virtualized portion of this sstable.
// The underlying file's size is stored in FileBacking.Size, though it could
// also be estimated or could correspond to just the referenced portion of
// a file (eg. if the file originated on another node).
// - Size must be > 0.
// - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
// This means that all keys in the virtual sstable must have seqnums within
// [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
// a key with a seqnum at either of the bounds. Calculating tight seqnum
// bounds would be too expensive and deliver little value.
//
// NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
type VirtualFileMeta struct {
*FileMetadata
@@ -589,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive
return true
}
// ContainedWithinSpan returns true if the file key range completely overlaps with the
// given range ("end" is assumed to exclusive).
func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool {
lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end)
return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel()))
}
// ContainsKeyType returns whether or not the file contains keys of the provided
// type.
func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
@@ -686,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
if !verbose {
return b.String()
}
fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum)
if m.HasPointKeys {
fmt.Fprintf(&b, " points:[%s-%s]",
m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format))
@@ -701,7 +727,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
// representation.
func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
// Split lines of the form:
// 000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
// 000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...]
fields := strings.FieldsFunc(s, func(c rune) bool {
switch c {
case ':', '[', '-', ']':
@@ -716,6 +742,19 @@ func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
m := &FileMetadata{}
for len(fields) > 0 {
prefix := fields[0]
if prefix == "seqnums" {
smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum
fields = fields[3:]
continue
}
smallest := base.ParsePrettyInternalKey(fields[1])
largest := base.ParsePrettyInternalKey(fields[2])
switch prefix {
@@ -983,6 +1022,9 @@ func NewVersion(
} else {
v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp)
}
for _, f := range files[l] {
v.Levels[l].totalSize += f.Size
}
}
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
panic(err)
@@ -1015,7 +1057,7 @@ func NewVersion(
// key in a higher level table that has both the same user key and a higher
// sequence number.
type Version struct {
refs int32
refs atomic.Int32
// The level 0 sstables are organized in a series of sublevels. Similar to
// the seqnum invariant in normal levels, there is no internal key in a
@@ -1145,12 +1187,12 @@ func ParseVersionDebug(
// Refs returns the number of references to the version.
func (v *Version) Refs() int32 {
return atomic.LoadInt32(&v.refs)
return v.refs.Load()
}
// Ref increments the version refcount.
func (v *Version) Ref() {
atomic.AddInt32(&v.refs, 1)
v.refs.Add(1)
}
// Unref decrements the version refcount. If the last reference to the version
@@ -1158,16 +1200,11 @@ func (v *Version) Ref() {
// Deleted callback is invoked. Requires that the VersionList mutex is NOT
// locked.
func (v *Version) Unref() {
if atomic.AddInt32(&v.refs, -1) == 0 {
if v.refs.Add(-1) == 0 {
l := v.list
l.mu.Lock()
l.Remove(v)
obsolete := v.unrefFiles()
fileBacking := make([]*FileBacking, len(obsolete))
for i, f := range obsolete {
fileBacking[i] = f.FileBacking
}
v.Deleted(fileBacking)
v.Deleted(v.unrefFiles())
l.mu.Unlock()
}
}
@@ -1177,19 +1214,14 @@ func (v *Version) Unref() {
// the Deleted callback is invoked. Requires that the VersionList mutex is
// already locked.
func (v *Version) UnrefLocked() {
if atomic.AddInt32(&v.refs, -1) == 0 {
if v.refs.Add(-1) == 0 {
v.list.Remove(v)
obsolete := v.unrefFiles()
fileBacking := make([]*FileBacking, len(obsolete))
for i, f := range obsolete {
fileBacking[i] = f.FileBacking
}
v.Deleted(fileBacking)
v.Deleted(v.unrefFiles())
}
}
func (v *Version) unrefFiles() []*FileMetadata {
var obsolete []*FileMetadata
func (v *Version) unrefFiles() []*FileBacking {
var obsolete []*FileBacking
for _, lm := range v.Levels {
obsolete = append(obsolete, lm.release()...)
}
@@ -1321,16 +1353,20 @@ func (v *Version) Overlaps(
// CheckOrdering checks that the files are consistent with respect to
// increasing file numbers (for level 0 files) and increasing and non-
// overlapping internal key ranges (for level non-0 files).
func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
func (v *Version) CheckOrdering(
cmp Compare, format base.FormatKey, order OrderingInvariants,
) error {
for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
sublevelIter := v.L0SublevelFiles[sublevel].Iter()
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
// Sublevels have NEVER allowed split user keys, so we can pass
// ProhibitSplitUserKeys.
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
for level, lm := range v.Levels {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
@@ -1399,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
v.list = nil // avoid memory leaks
}
// OrderingInvariants dictates the file ordering invariants active.
type OrderingInvariants int8
const (
// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
// contain the same user key.
ProhibitSplitUserKeys OrderingInvariants = iota
// AllowSplitUserKeys indicates that adjacent files within a level may
// contain the same user key. This is only allowed by historical format
// major versions.
//
// TODO(jackson): Remove.
AllowSplitUserKeys
)
// CheckOrdering checks that the files are consistent with respect to
// seqnums (for level 0 files -- see detailed comment below) and increasing and non-
// overlapping internal key ranges (for non-level 0 files).
func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
//
// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
// inclusive of the same user key. Pebble no longer creates version edits
// installing such files, and Pebble databases with sufficiently high format
// major version should no longer have any such files within their LSM.
// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
// earlier format major versions.
func CheckOrdering(
cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
) error {
// The invariants to check for L0 sublevels are the same as the ones to
// check for all other levels. However, if L0 is not organized into
// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1480,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
// What's considered "overlapping" is dependent on the format
// major version. If ordering=ProhibitSplitUserKeys, then both
// files cannot contain keys with the same user keys. If the
// bounds have the same user key, the previous file's boundary
// must have a Trailer indicating that it's exclusive.
switch ordering {
case AllowSplitUserKeys:
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
case ProhibitSplitUserKeys:
if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
default:
panic("unreachable")
}
}
}

49
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored
View File

@@ -450,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
return nil
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
var buf bytes.Buffer
if v.ComparerName != "" {
fmt.Fprintf(&buf, " comparer: %s", v.ComparerName)
@@ -482,7 +481,12 @@ func (v *VersionEdit) String() string {
fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum)
}
for _, nf := range v.NewFiles {
fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String())
fmt.Fprintf(&buf, " added: L%d", nf.Level)
if verbose {
fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
} else {
fmt.Fprintf(&buf, " %s", nf.Meta.String())
}
if nf.Meta.CreationTime != 0 {
fmt.Fprintf(&buf, " (%s)",
time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -492,6 +496,16 @@ func (v *VersionEdit) String() string {
return buf.String()
}
// DebugString is a more verbose version of String(). Use this in tests.
func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
return v.string(true /* verbose */, fmtKey)
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
return v.string(false /* verbose */, base.DefaultFormatter)
}
// Encode encodes an edit to the specified writer.
func (v *VersionEdit) Encode(w io.Writer) error {
e := versionEditEncoder{new(bytes.Buffer)}
@@ -769,6 +783,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
}
for _, fb := range ve.CreatedBackingTables {
if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
// There is already a FileBacking associated with fb.DiskFileNum.
// This should never happen. There must always be only one FileBacking
// associated with a backing sstable.
panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
}
b.AddedFileBacking[fb.DiskFileNum] = fb
}
@@ -832,6 +852,9 @@ func AccumulateIncompleteAndApplySingleVE(
flushSplitBytes int64,
readCompactionRate int64,
backingStateMap map[base.DiskFileNum]*FileBacking,
addBackingFunc func(*FileBacking),
removeBackingFunc func(base.DiskFileNum),
orderingInvariants OrderingInvariants,
) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
if len(ve.RemovedBackingTables) != 0 {
panic("pebble: invalid incomplete version edit")
@@ -843,14 +866,14 @@ func AccumulateIncompleteAndApplySingleVE(
}
zombies = make(map[base.DiskFileNum]uint64)
v, err := b.Apply(
curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
)
if err != nil {
return nil, nil, err
}
for _, s := range b.AddedFileBacking {
backingStateMap[s.DiskFileNum] = s
addBackingFunc(s)
}
for fileNum := range zombies {
@@ -861,10 +884,9 @@ func AccumulateIncompleteAndApplySingleVE(
ve.RemovedBackingTables = append(
ve.RemovedBackingTables, fileNum,
)
delete(backingStateMap, fileNum)
removeBackingFunc(fileNum)
}
}
return v, zombies, nil
}
@@ -885,6 +907,7 @@ func (b *BulkVersionEdit) Apply(
flushSplitBytes int64,
readCompactionRate int64,
zombies map[base.DiskFileNum]uint64,
orderingInvariants OrderingInvariants,
) (*Version, error) {
addZombie := func(state *FileBacking) {
if zombies != nil {
@@ -953,7 +976,7 @@ func (b *BulkVersionEdit) Apply(
// level.
for _, f := range deletedFilesMap {
if obsolete := v.Levels[level].tree.Delete(f); obsolete {
if obsolete := v.Levels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
@@ -962,7 +985,7 @@ func (b *BulkVersionEdit) Apply(
return nil, err
}
if f.HasRangeKeys {
if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
@@ -1015,7 +1038,7 @@ func (b *BulkVersionEdit) Apply(
f.AllowedSeeks.Store(allowedSeeks)
f.InitAllowedSeeks = allowedSeeks
err := lm.tree.Insert(f)
err := lm.insert(f)
// We're adding this file to the new version, so increment the
// latest refs count.
f.LatestRef()
@@ -1023,7 +1046,7 @@ func (b *BulkVersionEdit) Apply(
return nil, errors.Wrap(err, "pebble")
}
if f.HasRangeKeys {
err = lmRange.tree.Insert(f)
err = lmRange.insert(f)
if err != nil {
return nil, errors.Wrap(err, "pebble")
}
@@ -1068,7 +1091,7 @@ func (b *BulkVersionEdit) Apply(
} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
continue
@@ -1089,7 +1112,7 @@ func (b *BulkVersionEdit) Apply(
end.Prev()
}
})
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
}

147
vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go generated vendored
View File

@@ -6,7 +6,6 @@ package rangekey
import (
"bytes"
"fmt"
"math"
"sort"
@@ -53,15 +52,15 @@ import (
// │
// ╰── <?>.Next
type UserIteratorConfig struct {
snapshot uint64
comparer *base.Comparer
miter keyspan.MergingIter
biter keyspan.BoundedIter
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
onlySets bool
bufs *Buffers
snapshot uint64
comparer *base.Comparer
miter keyspan.MergingIter
biter keyspan.BoundedIter
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
internalKeys bool
bufs *Buffers
}
// Buffers holds various buffers used for range key iteration. They're exposed
@@ -80,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
// Init initializes the range key iterator stack for user iteration. The
// resulting fragment iterator applies range key semantics, defragments spans
// according to their user-observable state and, if onlySets = true, removes all
// according to their user-observable state and, if !internalKeys, removes all
// Keys other than RangeKeySets describing the current state of range keys. The
// resulting spans contain Keys sorted by Suffix.
// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
// in which case they remain sorted by trailer descending).
//
// The snapshot sequence number parameter determines which keys are visible. Any
// keys not visible at the provided snapshot are ignored.
@@ -92,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
lower, upper []byte,
hasPrefix *bool,
prefix *[]byte,
onlySets bool,
internalKeys bool,
bufs *Buffers,
iters ...keyspan.FragmentIterator,
) keyspan.FragmentIterator {
ui.snapshot = snapshot
ui.comparer = comparer
ui.onlySets = onlySets
ui.internalKeys = internalKeys
ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
if internalKeys {
ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
} else {
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
}
ui.litersUsed = 0
ui.bufs = bufs
return &ui.diter
@@ -137,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
// of unset keys, removal of keys overwritten by a set at the same suffix, etc)
// and then non-RangeKeySet keys are removed. The resulting transformed spans
// only contain RangeKeySets describing the state visible at the provided
// sequence number, and hold their Keys sorted by Suffix.
// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
// is true, then keys remain sorted by trailer.
func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
// Apply shadowing of keys.
dst.Start = s.Start
@@ -149,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
return err
}
// During user iteration over range keys, unsets and deletes don't matter.
// Remove them if onlySets = true. This step helps logical defragmentation
// during iteration.
if ui.internalKeys {
if s.KeysOrder != keyspan.ByTrailerDesc {
panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
}
dst.Keys = ui.bufs.sortBuf.Keys
keyspan.SortKeysByTrailer(&dst.Keys)
return nil
}
// During user iteration over range keys, unsets and deletes don't matter. This
// step helps logical defragmentation during iteration.
keys := ui.bufs.sortBuf.Keys
dst.Keys = dst.Keys[:0]
for i := range keys {
@@ -165,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
// Skip.
continue
case base.InternalKeyKindRangeKeyDelete:
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
// Skip.
continue
default:
return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
}
@@ -194,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
// sequence numbers). It's intended for use during user iteration, when the
// wrapped keyspan iterator is merging spans across all levels of the LSM.
func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
// This method is not called with internalKeys = true.
if ui.internalKeys {
panic("unexpected call to ShouldDefragment with internalKeys = true")
}
// This implementation must only be used on spans that have transformed by
// ui.Transform. The transform applies shadowing, removes all keys besides
// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -209,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
ret := true
for i := range a.Keys {
if invariants.Enabled {
if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
panic("pebble: unexpected non-RangeKeySet during defragmentation")
}
if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||
@@ -368,78 +378,3 @@ func coalesce(
}
return nil
}
// ForeignSSTTransformer implements a keyspan.Transformer for range keys in
// foreign sstables (i.e. shared sstables not created by us). It is largely
// similar to the Transform function implemented in UserIteratorConfig in that
// it calls coalesce to remove range keys shadowed by other range keys, but also
// retains the range key that does the shadowing. In addition, it elides
// RangeKey unsets/dels in L6 as they are inapplicable when reading from a
// different Pebble instance.
type ForeignSSTTransformer struct {
Comparer *base.Comparer
Level int
sortBuf keyspan.KeysBySuffix
}
// Transform implements the Transformer interface.
func (f *ForeignSSTTransformer) Transform(
cmp base.Compare, s keyspan.Span, dst *keyspan.Span,
) error {
// Apply shadowing of keys.
dst.Start = s.Start
dst.End = s.End
f.sortBuf = keyspan.KeysBySuffix{
Cmp: cmp,
Keys: f.sortBuf.Keys[:0],
}
if err := coalesce(f.Comparer.Equal, &f.sortBuf, math.MaxUint64, s.Keys); err != nil {
return err
}
keys := f.sortBuf.Keys
dst.Keys = dst.Keys[:0]
for i := range keys {
seqNum := keys[i].SeqNum()
switch keys[i].Kind() {
case base.InternalKeyKindRangeKeySet:
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
switch f.Level {
case 5:
fallthrough
case 6:
if seqNum != base.SeqNumForLevel(f.Level) {
panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
}
}
case base.InternalKeyKindRangeKeyUnset:
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
fallthrough
case base.InternalKeyKindRangeKeyDelete:
switch f.Level {
case 5:
// Emit this key.
if seqNum != base.SeqNumForLevel(f.Level) {
panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
}
case 6:
// Skip this key, as foreign sstable in L6 do not need to emit range key
// unsets/dels as they do not apply to any other sstables.
continue
}
default:
return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
}
dst.Keys = append(dst.Keys, keyspan.Key{
Trailer: base.MakeTrailer(seqNum, keys[i].Kind()),
Suffix: keys[i].Suffix,
Value: keys[i].Value,
})
}
// coalesce results in dst.Keys being sorted by Suffix.
dst.KeysOrder = keyspan.BySuffixAsc
return nil
}

387
vendor/github.com/cockroachdb/pebble/internal/rate/rate.go generated vendored
View File

@@ -1,387 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package rate provides a rate limiter.
package rate // import "github.com/cockroachdb/pebble/internal/rate"
import (
"context"
"math"
"sync"
"time"
"github.com/cockroachdb/errors"
)
// Limit defines the maximum frequency of some events.
// Limit is represented as number of events per second.
// A zero Limit allows no events.
type Limit float64
// Inf is the infinite rate limit; it allows all events (even if burst is zero).
const Inf = Limit(math.MaxFloat64)
// Every converts a minimum time interval between events to a Limit.
func Every(interval time.Duration) Limit {
if interval <= 0 {
return Inf
}
return 1 / Limit(interval.Seconds())
}
// A Limiter controls how frequently events are allowed to happen.
// It implements a "token bucket" of size b, initially full and refilled
// at rate r tokens per second.
// Informally, in any large enough time interval, the Limiter limits the
// rate to r tokens per second, with a maximum burst size of b events.
// As a special case, if r == Inf (the infinite rate), b is ignored.
// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
//
// The zero value is a valid Limiter, but it will reject all events.
// Use NewLimiter to create non-zero Limiters.
//
// Limiter has three main methods, Allow, Reserve, and Wait.
// Most callers should use Wait.
//
// Each of the three methods consumes a single token.
// They differ in their behavior when no token is available.
// If no token is available, Allow returns false.
// If no token is available, Reserve returns a reservation for a future token
// and the amount of time the caller must wait before using it.
// If no token is available, Wait blocks until one can be obtained
// or its associated context.Context is canceled.
//
// The methods AllowN, ReserveN, and WaitN consume n tokens.
type Limiter struct {
limit Limit
burst int
mu sync.Mutex
tokens float64
// last is the last time the limiter's tokens field was updated
last time.Time
// lastEvent is the latest time of a rate-limited event (past or future)
lastEvent time.Time
}
// Limit returns the maximum overall event rate.
func (lim *Limiter) Limit() Limit {
lim.mu.Lock()
defer lim.mu.Unlock()
return lim.limit
}
// Burst returns the maximum burst size. Burst is the maximum number of tokens
// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
// Burst values allow more events to happen at once.
// A zero Burst allows no events, unless limit == Inf.
func (lim *Limiter) Burst() int {
return lim.burst
}
// NewLimiter returns a new Limiter that allows events up to rate r and permits
// bursts of at most b tokens.
func NewLimiter(r Limit, b int) *Limiter {
return &Limiter{
limit: r,
burst: b,
}
}
// Allow is shorthand for AllowN(time.Now(), 1).
func (lim *Limiter) Allow() bool {
return lim.AllowN(time.Now(), 1)
}
// AllowN reports whether n events may happen at time now.
// Use this method if you intend to drop / skip events that exceed the rate limit.
// Otherwise use Reserve or Wait.
func (lim *Limiter) AllowN(now time.Time, n int) bool {
return lim.reserveN(now, n, 0).ok
}
// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
// A Reservation may be canceled, which may enable the Limiter to permit additional events.
type Reservation struct {
ok bool
lim *Limiter
tokens int
timeToAct time.Time
// This is the Limit at reservation time, it can change later.
limit Limit
}
// OK returns whether the limiter can provide the requested number of tokens
// within the maximum wait time. If OK is false, Delay returns InfDuration, and
// Cancel does nothing.
func (r *Reservation) OK() bool {
return r.ok
}
// Delay is shorthand for DelayFrom(time.Now()).
func (r *Reservation) Delay() time.Duration {
return r.DelayFrom(time.Now())
}
// InfDuration is the duration returned by Delay when a Reservation is not OK.
const InfDuration = time.Duration(1<<63 - 1)
// DelayFrom returns the duration for which the reservation holder must wait
// before taking the reserved action. Zero duration means act immediately.
// InfDuration means the limiter cannot grant the tokens requested in this
// Reservation within the maximum wait time.
func (r *Reservation) DelayFrom(now time.Time) time.Duration {
if !r.ok {
return InfDuration
}
delay := r.timeToAct.Sub(now)
if delay < 0 {
return 0
}
return delay
}
// Cancel is shorthand for CancelAt(time.Now()).
func (r *Reservation) Cancel() {
r.CancelAt(time.Now())
}
// CancelAt indicates that the reservation holder will not perform the reserved action
// and reverses the effects of this Reservation on the rate limit as much as possible,
// considering that other reservations may have already been made.
func (r *Reservation) CancelAt(now time.Time) {
if !r.ok {
return
}
r.lim.mu.Lock()
defer r.lim.mu.Unlock()
if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
return
}
// calculate tokens to restore
// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
// after r was obtained. These tokens should not be restored.
restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
if restoreTokens <= 0 {
return
}
// advance time to now
now, _, tokens := r.lim.advance(now)
// calculate new number of tokens
tokens += restoreTokens
if burst := float64(r.lim.burst); tokens > burst {
tokens = burst
}
// update state
r.lim.last = now
r.lim.tokens = tokens
if r.timeToAct == r.lim.lastEvent {
prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
if !prevEvent.Before(now) {
r.lim.lastEvent = prevEvent
}
}
}
// Reserve is shorthand for ReserveN(time.Now(), 1).
func (lim *Limiter) Reserve() Reservation {
return lim.ReserveN(time.Now(), 1)
}
// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
// The Limiter takes this Reservation into account when allowing future events.
// ReserveN returns false if n exceeds the Limiter's burst size.
// Usage example:
//
// r := lim.ReserveN(time.Now(), 1)
// if !r.OK() {
// // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
// return
// }
// time.Sleep(r.Delay())
// Act()
//
// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
// If you need to respect a deadline or cancel the delay, use Wait instead.
// To drop or skip events exceeding rate limit, use Allow instead.
func (lim *Limiter) ReserveN(now time.Time, n int) Reservation {
return lim.reserveN(now, n, InfDuration)
}
// Wait is shorthand for WaitN(ctx, 1).
func (lim *Limiter) Wait(ctx context.Context) (err error) {
return lim.WaitN(ctx, 1)
}
// WaitN blocks until lim permits n events to happen.
// It returns an error if n exceeds the Limiter's burst size, the Context is
// canceled, or the expected wait time exceeds the Context's Deadline.
func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
if n > lim.burst && lim.limit != Inf {
return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst))
}
// Check if ctx is already cancelled
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// Determine wait limit
now := time.Now()
waitLimit := InfDuration
if deadline, ok := ctx.Deadline(); ok {
waitLimit = deadline.Sub(now)
}
// Reserve
r := lim.reserveN(now, n, waitLimit)
if !r.ok {
return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n))
}
// Wait
d := r.DelayFrom(now)
if d <= 0 {
return nil
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
// We can proceed.
return nil
case <-ctx.Done():
// Context was canceled before we could proceed. Cancel the
// reservation, which may permit other events to proceed sooner.
r.Cancel()
return ctx.Err()
}
}
// Delay is shorthand for DelayN(time.Now(), 1).
func (lim *Limiter) Delay() time.Duration {
return lim.DelayN(time.Now(), 1)
}
// DelayN returns the delay to wait to permit n events to happen. Zero duration
// means act immediately. InfDuration means the limiter cannot grant the tokens
// requested within the maximum wait time.
func (lim *Limiter) DelayN(now time.Time, n int) time.Duration {
r := lim.reserveN(now, n, InfDuration)
return r.DelayFrom(now)
}
// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
func (lim *Limiter) SetLimit(newLimit Limit) {
lim.SetLimitAt(time.Now(), newLimit)
}
// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
// before SetLimitAt was called.
func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
lim.last = now
lim.tokens = tokens
lim.limit = newLimit
}
// reserveN is a helper method for AllowN, ReserveN, and WaitN.
// maxFutureReserve specifies the maximum reservation wait duration allowed.
// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
lim.mu.Lock()
if lim.limit == Inf {
lim.mu.Unlock()
return Reservation{
ok: true,
lim: lim,
tokens: n,
timeToAct: now,
}
}
now, last, tokens := lim.advance(now)
// Calculate the remaining number of tokens resulting from the request.
tokens -= float64(n)
// Calculate the wait duration
var waitDuration time.Duration
if tokens < 0 {
waitDuration = lim.limit.durationFromTokens(-tokens)
}
// Decide result
ok := n <= lim.burst && waitDuration <= maxFutureReserve
// Prepare reservation
r := Reservation{
ok: ok,
lim: lim,
limit: lim.limit,
}
if ok {
r.tokens = n
r.timeToAct = now.Add(waitDuration)
}
// Update state
if ok {
lim.last = now
lim.tokens = tokens
lim.lastEvent = r.timeToAct
} else {
lim.last = last
}
lim.mu.Unlock()
return r
}
// advance calculates and returns an updated state for lim resulting from the passage of time.
// lim is not changed.
func (lim *Limiter) advance(
now time.Time,
) (newNow time.Time, newLast time.Time, newTokens float64) {
last := lim.last
if now.Before(last) {
last = now
}
// Avoid making delta overflow below when last is very old.
maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
elapsed := now.Sub(last)
if elapsed > maxElapsed {
elapsed = maxElapsed
}
// Calculate the new number of tokens, due to time that passed.
delta := lim.limit.tokensFromDuration(elapsed)
tokens := lim.tokens + delta
if burst := float64(lim.burst); tokens > burst {
tokens = burst
}
return now, last, tokens
}
// durationFromTokens is a unit conversion function from the number of tokens to the duration
// of time it takes to accumulate them at a rate of limit tokens per second.
func (limit Limit) durationFromTokens(tokens float64) time.Duration {
seconds := tokens / float64(limit)
return time.Nanosecond * time.Duration(1e9*seconds)
}
// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
// which could be accumulated during that duration at a rate of limit tokens per second.
func (limit Limit) tokensFromDuration(d time.Duration) float64 {
return d.Seconds() * float64(limit)
}

229
vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go generated vendored
View File

@@ -20,17 +20,27 @@ import (
"strings"
"github.com/cockroachdb/pebble/internal/base"
"golang.org/x/exp/constraints"
"golang.org/x/exp/rand"
)
const alpha = "abcdefghijklmnopqrstuvwxyz"
const suffixDelim = '@'
var inverseAlphabet = make(map[byte]int64, len(alpha))
func init() {
for i := range alpha {
inverseAlphabet[alpha[i]] = int64(i)
}
}
// MaxSuffixLen is the maximum length of a suffix generated by this package.
var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64))
var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64)))
// Comparer is the comparer for test keys generated by this package.
var Comparer *base.Comparer = &base.Comparer{
var Comparer = &base.Comparer{
Compare: compare,
Equal: func(a, b []byte) bool { return compare(a, b) == 0 },
AbbreviatedKey: func(k []byte) uint64 {
@@ -136,7 +146,7 @@ func compareTimestamps(a, b []byte) int {
// Keyspace describes a finite keyspace of unsuffixed test keys.
type Keyspace interface {
// Count returns the number of keys that exist within this keyspace.
Count() int
Count() int64
// MaxLen returns the maximum length, in bytes, of a key within this
// keyspace. This is only guaranteed to return an upper bound.
@@ -144,20 +154,21 @@ type Keyspace interface {
// Slice returns the sub-keyspace from index i, inclusive, to index j,
// exclusive. The receiver is unmodified.
Slice(i, j int) Keyspace
Slice(i, j int64) Keyspace
// EveryN returns a key space that includes 1 key for every N keys in the
// original keyspace. The receiver is unmodified.
EveryN(n int) Keyspace
EveryN(n int64) Keyspace
key(buf []byte, i int) int
// key writes the i-th key to the buffer and returns the length.
key(buf []byte, i int64) int
}
// Divvy divides the provided keyspace into N equal portions, containing
// disjoint keys evenly distributed across the keyspace.
func Divvy(ks Keyspace, n int) []Keyspace {
func Divvy(ks Keyspace, n int64) []Keyspace {
ret := make([]Keyspace, n)
for i := 0; i < n; i++ {
for i := int64(0); i < n; i++ {
ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
}
return ret
@@ -175,7 +186,7 @@ func Alpha(maxLength int) Keyspace {
// KeyAt returns the i-th key within the keyspace with a suffix encoding the
// timestamp t.
func KeyAt(k Keyspace, i int, t int) []byte {
func KeyAt(k Keyspace, i int64, t int64) []byte {
b := make([]byte, k.MaxLen()+MaxSuffixLen)
return b[:WriteKeyAt(b, k, i, t)]
}
@@ -183,20 +194,20 @@ func KeyAt(k Keyspace, i int, t int) []byte {
// WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
// suffix encoding the timestamp t suffix. It returns the number of bytes
// written.
func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int {
func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int {
n := WriteKey(dst, k, i)
n += WriteSuffix(dst[n:], t)
return n
}
// Suffix returns the test keys suffix representation of timestamp t.
func Suffix(t int) []byte {
func Suffix(t int64) []byte {
b := make([]byte, MaxSuffixLen)
return b[:WriteSuffix(b, t)]
}
// SuffixLen returns the exact length of the given suffix when encoded.
func SuffixLen(t int) int {
func SuffixLen(t int64) int {
// Begin at 1 for the '@' delimiter, 1 for a single digit.
n := 2
t /= 10
@@ -208,42 +219,42 @@ func SuffixLen(t int) int {
}
// ParseSuffix returns the integer representation of the encoded suffix.
func ParseSuffix(s []byte) (int, error) {
return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
func ParseSuffix(s []byte) (int64, error) {
return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64)
}
// WriteSuffix writes the test keys suffix representation of timestamp t to dst,
// returning the number of bytes written.
func WriteSuffix(dst []byte, t int) int {
func WriteSuffix(dst []byte, t int64) int {
dst[0] = suffixDelim
n := 1
n += len(strconv.AppendInt(dst[n:n], int64(t), 10))
n += len(strconv.AppendInt(dst[n:n], t, 10))
return n
}
// Key returns the i-th unsuffixed key within the keyspace.
func Key(k Keyspace, i int) []byte {
func Key(k Keyspace, i int64) []byte {
b := make([]byte, k.MaxLen())
return b[:k.key(b, i)]
}
// WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
// returns the number of bytes written.
func WriteKey(dst []byte, k Keyspace, i int) int {
func WriteKey(dst []byte, k Keyspace, i int64) int {
return k.key(dst, i)
}
type alphabet struct {
alphabet []byte
maxLength int
headSkip int
tailSkip int
increment int
headSkip int64
tailSkip int64
increment int64
}
func (a alphabet) Count() int {
func (a alphabet) Count() int64 {
// Calculate the total number of keys, ignoring the increment.
total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip)
total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip
// The increment dictates that we take every N keys, where N = a.increment.
// Consider a total containing the 5 keys:
@@ -263,20 +274,25 @@ func (a alphabet) MaxLen() int {
return a.maxLength
}
func (a alphabet) Slice(i, j int) Keyspace {
func (a alphabet) Slice(i, j int64) Keyspace {
s := a
s.headSkip += i
s.tailSkip += a.Count() - j
return s
}
func (a alphabet) EveryN(n int) Keyspace {
func (a alphabet) EveryN(n int64) Keyspace {
s := a
s.increment *= n
return s
}
func keyCount(n, l int) int {
func keyCount(n, l int) int64 {
if n == 0 {
return 0
} else if n == 1 {
return int64(l)
}
// The number of representable keys in the keyspace is a function of the
// length of the alphabet n and the max key length l. Consider how the
// number of representable keys grows as l increases:
@@ -286,10 +302,10 @@ func keyCount(n, l int) int {
// l = 3: n + n^2 + n^3
// ...
// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1)
return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1)
}
func (a alphabet) key(buf []byte, idx int) int {
func (a alphabet) key(buf []byte, idx int64) int {
// This function generates keys of length 1..maxKeyLength, pulling
// characters from the alphabet. The idx determines which key to generate,
// generating the i-th lexicographically next key.
@@ -308,14 +324,14 @@ func (a alphabet) key(buf []byte, idx int) int {
keyCount(len(a.alphabet), a.maxLength))
}
func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int {
if keyCount == 0 || i > keyCount || i < 0 {
return 0
}
// Of the keyCount keys in the generative keyspace, how many are there
// starting with a particular character?
keysPerCharacter := keyCount / len(alphabet)
keysPerCharacter := keyCount / int64(len(alphabet))
// Find the character that the key at index i starts with and set it.
characterIdx := i / keysPerCharacter
@@ -347,3 +363,156 @@ func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
i = i - keysPerCharacter*characterIdx - 1
return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
}
// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey,
// returning the index of a particular key, given the provided alphabet and max
// length of a key.
//
// len(key) must be ≥ 1.
func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
i, ok := alphabet[key[0]]
if !ok {
panic(fmt.Sprintf("unrecognized alphabet character %v", key[0]))
}
// How many keys exist that start with the preceding i characters? Each of
// the i characters themselves are a key, plus the count of all the keys
// with one less character for each.
ret := i + i*keyCount(len(alphabet), n-1)
if len(key) > 1 {
ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1)
}
return ret
}
func abs(a int64) int64 {
if a < 0 {
return -a
}
return a
}
// RandomSeparator returns a random alphabetic key k such that a < k < b,
// pulling randomness from the provided random number generator. If dst is
// provided and the generated key fits within dst's capacity, the returned slice
// will use dst's memory.
//
// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
// will consist of the prefix P appended with the provided suffix. A zero suffix
// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b,
// but the generated key will not use the provided suffix. Note that it's
// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case
// RandomSeparator returns nil.
//
// If RandomSeparator generates a new prefix, the generated prefix will have
// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))).
//
// RandomSeparator panics if a or b fails to decode.
func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
if Comparer.Compare(a, b) >= 0 {
return nil
}
// Determine both keys' logical prefixes and suffixes.
ai := Comparer.Split(a)
bi := Comparer.Split(b)
ap := a[:ai]
bp := b[:bi]
maxLength = max[int](maxLength, max[int](len(ap), len(bp)))
var as, bs int64
var err error
if ai != len(a) {
as, err = ParseSuffix(a[ai:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", a))
}
}
if bi != len(b) {
bs, err = ParseSuffix(b[bi:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", b))
}
}
apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength)
bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength)
diff := bpIdx - apIdx
generatedIdx := bpIdx
if diff > 0 {
var add int64 = diff + 1
var start int64 = apIdx
if as == 1 {
// There's no expressible key with prefix a greater than a@1. So,
// exclude ap.
start = apIdx + 1
add = diff
}
if bs == 0 {
// No key with prefix b can sort before b@0. We don't want to pick b.
add--
}
// We're allowing generated id to be in the range [start, start + add - 1].
if start > start+add-1 {
return nil
}
// If we can generate a key which is actually in the middle of apIdx
// and bpIdx use it so that we don't have to bother about timestamps.
generatedIdx = rng.Int63n(add) + start
for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
generatedIdx = rng.Int63n(add) + start
}
}
switch {
case generatedIdx == apIdx && generatedIdx == bpIdx:
if abs(bs-as) <= 1 {
// There's no expressible suffix between the two, and there's no
// possible separator key.
return nil
}
// The key b is >= key a, but has the same prefix, so b must have the
// smaller timestamp, unless a has timestamp of 0.
//
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if as == 0 {
// bs > as
suffix = bs + rng.Int63n(10) + 1
} else {
// bs < as.
// Generate suffix in range [bs + 1, as - 1]
suffix = bs + 1 + rng.Int63n(as-bs-1)
}
case generatedIdx == apIdx:
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if as == 0 && suffix == 0 {
suffix++
} else if as != 0 && suffix >= as {
suffix = rng.Int63n(as)
}
case generatedIdx == bpIdx:
if suffix <= bs {
suffix = bs + rng.Int63n(10) + 1
}
}
if sz := maxLength + SuffixLen(suffix); cap(dst) < sz {
dst = make([]byte, sz)
} else {
dst = dst[:cap(dst)]
}
var w int
if suffix == 0 {
w = WriteKey(dst, Alpha(maxLength), generatedIdx)
} else {
w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
}
return dst[:w]
}
func max[I constraints.Ordered](a, b I) I {
if b > a {
return b
}
return a
}

410
vendor/github.com/cockroachdb/pebble/iterator.go generated vendored
View File

@@ -154,6 +154,18 @@ func (s *RangeKeyIteratorStats) Merge(o RangeKeyIteratorStats) {
s.SkippedPoints += o.SkippedPoints
}
func (s *RangeKeyIteratorStats) String() string {
return redact.StringWithoutMarkers(s)
}
// SafeFormat implements the redact.SafeFormatter interface.
func (s *RangeKeyIteratorStats) SafeFormat(p redact.SafePrinter, verb rune) {
p.Printf("range keys: %s, contained points: %s (%s skipped)",
humanize.Count.Uint64(uint64(s.Count)),
humanize.Count.Uint64(uint64(s.ContainedPoints)),
humanize.Count.Uint64(uint64(s.SkippedPoints)))
}
// LazyValue is a lazy value. See the long comment in base.LazyValue.
type LazyValue = base.LazyValue
@@ -187,7 +199,9 @@ type Iterator struct {
comparer base.Comparer
iter internalIterator
pointIter internalIterator
// Either readState or version is set, but not both.
readState *readState
version *version
// rangeKey holds iteration state specific to iteration over range keys.
// The range key field may be nil if the Iterator has never been configured
// to iterate over range keys. Its non-nilness cannot be used to determine
@@ -445,6 +459,11 @@ const (
seekPrefixGELastPositioningOp
seekGELastPositioningOp
seekLTLastPositioningOp
// internalNextOp is a special internal iterator positioning operation used
// by CanDeterministicallySingleDelete. It exists for enforcing requirements
// around calling CanDeterministicallySingleDelete at most once per external
// iterator position.
internalNextOp
// invalidatedLastPositionOp is similar to unknownLastPositionOp and the
// only reason to distinguish this is for the wider set of SeekGE
// optimizations we permit for the external iterator Iterator.forwardOnly
@@ -548,6 +567,21 @@ func (i *Iterator) findNextEntry(limit []byte) {
return
}
// If the user has configured a SkipPoint function, invoke it to see
// whether we should skip over the current user key.
if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(i.iterKey.UserKey) {
// NB: We could call nextUserKey, but in some cases the SkipPoint
// predicate function might be cheaper than nextUserKey's key copy
// and key comparison. This should be the case for MVCC suffix
// comparisons, for example. In the future, we could expand the
// SkipPoint interface to give the implementor more control over
// whether we skip over just the internal key, the user key, or even
// the key prefix.
i.stats.ForwardStepCount[InternalIterCall]++
i.iterKey, i.iterValue = i.iter.Next()
continue
}
switch key.Kind() {
case InternalKeyKindRangeKeySet:
// Save the current key.
@@ -617,6 +651,13 @@ func (i *Iterator) findNextEntry(limit []byte) {
}
func (i *Iterator) nextPointCurrentUserKey() bool {
// If the user has configured a SkipPoint function and the current user key
// would be skipped by it, there's no need to step forward looking for a
// point key. If we were to find one, it should be skipped anyways.
if i.opts.SkipPoint != nil && i.opts.SkipPoint(i.key) {
return false
}
i.pos = iterPosCurForward
i.iterKey, i.iterValue = i.iter.Next()
@@ -778,39 +819,41 @@ func (i *Iterator) maybeSampleRead() {
func (i *Iterator) sampleRead() {
var topFile *manifest.FileMetadata
topLevel, numOverlappingLevels := numLevels, 0
if mi, ok := i.iter.(*mergingIter); ok {
if len(mi.levels) > 1 {
mi.ForEachLevelIter(func(li *levelIter) bool {
l := manifest.LevelToInt(li.level)
if f := li.iterFile; f != nil {
var containsKey bool
if i.pos == iterPosNext || i.pos == iterPosCurForward ||
i.pos == iterPosCurForwardPaused {
containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
i.pos == iterPosCurReversePaused {
containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
}
// Do nothing if the current key is not contained in f's
// bounds. We could seek the LevelIterator at this level
// to find the right file, but the performance impacts of
// doing that are significant enough to negate the benefits
// of read sampling in the first place. See the discussion
// at:
// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
if containsKey {
numOverlappingLevels++
if numOverlappingLevels >= 2 {
// Terminate the loop early if at least 2 overlapping levels are found.
return true
}
topLevel = l
topFile = f
}
mi := i.merging
if mi == nil {
return
}
if len(mi.levels) > 1 {
mi.ForEachLevelIter(func(li *levelIter) bool {
l := manifest.LevelToInt(li.level)
if f := li.iterFile; f != nil {
var containsKey bool
if i.pos == iterPosNext || i.pos == iterPosCurForward ||
i.pos == iterPosCurForwardPaused {
containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
i.pos == iterPosCurReversePaused {
containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
}
return false
})
}
// Do nothing if the current key is not contained in f's
// bounds. We could seek the LevelIterator at this level
// to find the right file, but the performance impacts of
// doing that are significant enough to negate the benefits
// of read sampling in the first place. See the discussion
// at:
// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
if containsKey {
numOverlappingLevels++
if numOverlappingLevels >= 2 {
// Terminate the loop early if at least 2 overlapping levels are found.
return true
}
topLevel = l
topFile = f
}
}
return false
})
}
if topFile == nil || topLevel >= numLevels {
return
@@ -907,6 +950,26 @@ func (i *Iterator) findPrevEntry(limit []byte) {
}
}
// If the user has configured a SkipPoint function, invoke it to see
// whether we should skip over the current user key.
if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(key.UserKey) {
// NB: We could call prevUserKey, but in some cases the SkipPoint
// predicate function might be cheaper than prevUserKey's key copy
// and key comparison. This should be the case for MVCC suffix
// comparisons, for example. In the future, we could expand the
// SkipPoint interface to give the implementor more control over
// whether we skip over just the internal key, the user key, or even
// the key prefix.
i.stats.ReverseStepCount[InternalIterCall]++
i.iterKey, i.iterValue = i.iter.Prev()
if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
i.iterValidityState = IterAtLimit
i.pos = iterPosCurReversePaused
return
}
continue
}
switch key.Kind() {
case InternalKeyKindRangeKeySet:
// Range key start boundary markers are interleaved with the maximum
@@ -944,12 +1007,12 @@ func (i *Iterator) findPrevEntry(limit []byte) {
// Compare with the limit. We could optimize by only checking when
// we step to the previous user key, but detecting that requires a
// comparison too. Note that this position may already passed a
// number of versions of this user key, but they are all deleted,
// so the fact that a subsequent Prev*() call will not see them is
// number of versions of this user key, but they are all deleted, so
// the fact that a subsequent Prev*() call will not see them is
// harmless. Also note that this is the only place in the loop,
// other than the firstLoopIter case above, where we could step
// to a different user key and start processing it for returning
// to the caller.
// other than the firstLoopIter and SkipPoint cases above, where we
// could step to a different user key and start processing it for
// returning to the caller.
if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
i.iterValidityState = IterAtLimit
i.pos = iterPosCurReversePaused
@@ -1760,7 +1823,8 @@ func (i *Iterator) internalNextPrefix(currKeyPrefixLen int) {
i.iterKey, i.iterValue = i.iter.NextPrefix(i.prefixOrFullSeekKey)
if invariants.Enabled && i.iterKey != nil {
if iterKeyPrefixLen := i.split(i.iterKey.UserKey); i.cmp(i.iterKey.UserKey[:iterKeyPrefixLen], i.prefixOrFullSeekKey) < 0 {
panic("pebble: iter.NextPrefix did not advance beyond the current prefix")
panic(errors.AssertionFailedf("pebble: iter.NextPrefix did not advance beyond the current prefix: now at %q; expected to be geq %q",
i.iterKey, i.prefixOrFullSeekKey))
}
}
}
@@ -2129,6 +2193,9 @@ func (i *Iterator) Value() []byte {
// ValueAndErr returns the value, and any error encountered in extracting the value.
// REQUIRES: i.Error()==nil and HasPointAndRange() returns true for hasPoint.
//
// The caller should not modify the contents of the returned slice, and its
// contents may change on the next call to Next.
func (i *Iterator) ValueAndErr() ([]byte, error) {
val, callerOwned, err := i.value.Value(i.lazyValueBuf)
if err != nil {
@@ -2159,7 +2226,13 @@ func (i *Iterator) RangeKeys() []RangeKeyData {
// Valid returns true if the iterator is positioned at a valid key/value pair
// and false otherwise.
func (i *Iterator) Valid() bool {
return i.iterValidityState == IterValid && !i.requiresReposition
valid := i.iterValidityState == IterValid && !i.requiresReposition
if invariants.Enabled {
if err := i.Error(); valid && err != nil {
panic(errors.WithSecondaryError(errors.AssertionFailedf("pebble: iterator is valid with non-nil Error"), err))
}
}
return valid
}
// Error returns any accumulated error.
@@ -2224,6 +2297,10 @@ func (i *Iterator) Close() error {
i.readState = nil
}
if i.version != nil {
i.version.Unref()
}
for _, readers := range i.externalReaders {
for _, r := range readers {
err = firstError(err, r.Close())
@@ -2416,7 +2493,8 @@ func (i *Iterator) SetOptions(o *IterOptions) {
// If either options specify block property filters for an iterator stack,
// reconstruct it.
if i.pointIter != nil && (closeBoth || len(o.PointKeyFilters) > 0 || len(i.opts.PointKeyFilters) > 0 ||
o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil) {
o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil || o.SkipPoint != nil ||
i.opts.SkipPoint != nil) {
i.err = firstError(i.err, i.pointIter.Close())
i.pointIter = nil
}
@@ -2655,11 +2733,22 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
}
readState := i.readState
if readState == nil {
vers := i.version
if readState == nil && vers == nil {
return nil, errors.Errorf("cannot Clone a closed Iterator")
}
// i is already holding a ref, so there is no race with unref here.
readState.ref()
//
// TODO(bilal): If the underlying iterator was created on a snapshot, we could
// grab a reference to the current readState instead of reffing the original
// readState. This allows us to release references to some zombie sstables
// and memtables.
if readState != nil {
readState.ref()
}
if vers != nil {
vers.Ref()
}
// Bundle various structures under a single umbrella in order to allocate
// them together.
buf := iterAllocPool.Get().(*iterAlloc)
@@ -2671,6 +2760,7 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
merge: i.merge,
comparer: i.comparer,
readState: readState,
version: vers,
keyBuf: buf.keyBuf,
prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
boundsBuf: buf.boundsBuf,
@@ -2710,43 +2800,209 @@ func (stats *IteratorStats) String() string {
// SafeFormat implements the redact.SafeFormatter interface.
func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
for i := range stats.ForwardStepCount {
switch IteratorStatsKind(i) {
case InterfaceCall:
s.SafeString("(interface (dir, seek, step): ")
case InternalIterCall:
s.SafeString(", (internal (dir, seek, step): ")
}
s.Printf("(fwd, %d, %d), (rev, %d, %d))",
redact.Safe(stats.ForwardSeekCount[i]), redact.Safe(stats.ForwardStepCount[i]),
redact.Safe(stats.ReverseSeekCount[i]), redact.Safe(stats.ReverseStepCount[i]))
}
if stats.InternalStats != (InternalIteratorStats{}) {
s.SafeString(",\n(internal-stats: ")
s.Printf("(block-bytes: (total %s, cached %s, read-time %s)), "+
"(points: (count %s, key-bytes %s, value-bytes %s, tombstoned %s))",
humanize.IEC.Uint64(stats.InternalStats.BlockBytes),
humanize.IEC.Uint64(stats.InternalStats.BlockBytesInCache),
humanize.FormattedString(stats.InternalStats.BlockReadDuration.String()),
humanize.SI.Uint64(stats.InternalStats.PointCount),
humanize.SI.Uint64(stats.InternalStats.KeyBytes),
humanize.SI.Uint64(stats.InternalStats.ValueBytes),
humanize.SI.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones),
if stats.ReverseSeekCount[InterfaceCall] == 0 && stats.ReverseSeekCount[InternalIterCall] == 0 {
s.Printf("seeked %s times (%s internal)",
humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InternalIterCall])),
)
if stats.InternalStats.SeparatedPointValue.Count != 0 {
s.Printf(", (separated: (count %s, bytes %s, fetched %s)))",
humanize.SI.Uint64(stats.InternalStats.SeparatedPointValue.Count),
humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes),
humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched))
} else {
s.Printf(")")
}
} else {
s.Printf("seeked %s times (%s fwd/%s rev, internal: %s fwd/%s rev)",
humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InterfaceCall]+stats.ReverseSeekCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ReverseSeekCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InternalIterCall])),
humanize.Count.Uint64(uint64(stats.ReverseSeekCount[InternalIterCall])),
)
}
s.SafeString("; ")
if stats.ReverseStepCount[InterfaceCall] == 0 && stats.ReverseStepCount[InternalIterCall] == 0 {
s.Printf("stepped %s times (%s internal)",
humanize.Count.Uint64(uint64(stats.ForwardStepCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ForwardStepCount[InternalIterCall])),
)
} else {
s.Printf("stepped %s times (%s fwd/%s rev, internal: %s fwd/%s rev)",
humanize.Count.Uint64(uint64(stats.ForwardStepCount[InterfaceCall]+stats.ReverseStepCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ForwardStepCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ReverseStepCount[InterfaceCall])),
humanize.Count.Uint64(uint64(stats.ForwardStepCount[InternalIterCall])),
humanize.Count.Uint64(uint64(stats.ReverseStepCount[InternalIterCall])),
)
}
if stats.InternalStats != (InternalIteratorStats{}) {
s.SafeString("; ")
stats.InternalStats.SafeFormat(s, verb)
}
if stats.RangeKeyStats != (RangeKeyIteratorStats{}) {
s.SafeString(",\n(range-key-stats: ")
s.Printf("(count %d), (contained points: (count %d, skipped %d)))",
stats.RangeKeyStats.Count,
stats.RangeKeyStats.ContainedPoints,
stats.RangeKeyStats.SkippedPoints)
s.SafeString(", ")
stats.RangeKeyStats.SafeFormat(s, verb)
}
}
// CanDeterministicallySingleDelete takes a valid iterator and examines internal
// state to determine if a SingleDelete deleting Iterator.Key() would
// deterministically delete the key. CanDeterministicallySingleDelete requires
// the iterator to be oriented in the forward direction (eg, the last
// positioning operation must've been a First, a Seek[Prefix]GE, or a
// Next[Prefix][WithLimit]).
//
// This function does not change the external position of the iterator, and all
// positioning methods should behave the same as if it was never called. This
// function will only return a meaningful result the first time it's invoked at
// an iterator position. This function invalidates the iterator Value's memory,
// and the caller must not rely on the memory safety of the previous Iterator
// position.
//
// If CanDeterministicallySingleDelete returns true AND the key at the iterator
// position is not modified between the creation of the Iterator and the commit
// of a batch containing a SingleDelete over the key, then the caller can be
// assured that SingleDelete is equivalent to Delete on the local engine, but it
// may not be true on another engine that received the same writes and with
// logically equivalent state since this engine may have collapsed multiple SETs
// into one.
func CanDeterministicallySingleDelete(it *Iterator) (bool, error) {
// This function may only be called once per external iterator position. We
// can validate this by checking the last positioning operation.
if it.lastPositioningOp == internalNextOp {
return false, errors.New("pebble: CanDeterministicallySingleDelete called twice")
}
validity, kind := it.internalNext()
var shadowedBySingleDelete bool
for validity == internalNextValid {
switch kind {
case InternalKeyKindDelete, InternalKeyKindDeleteSized:
// A DEL or DELSIZED tombstone is okay. An internal key
// sequence like SINGLEDEL; SET; DEL; SET can be handled
// deterministically. If there are SETs further down, we
// don't care about them.
return true, nil
case InternalKeyKindSingleDelete:
// A SingleDelete is okay as long as when that SingleDelete was
// written, it was written deterministically (eg, with its own
// CanDeterministicallySingleDelete check). Validate that it was
// written deterministically. We'll allow one set to appear after
// the SingleDelete.
shadowedBySingleDelete = true
validity, kind = it.internalNext()
continue
case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
// If we observed a single delete, it's allowed to delete 1 key.
// We'll keep looping to validate that the internal keys beneath the
// already-written single delete are copacetic.
if shadowedBySingleDelete {
shadowedBySingleDelete = false
validity, kind = it.internalNext()
continue
}
// We encountered a shadowed SET, SETWITHDEL, MERGE. A SINGLEDEL
// that deleted the KV at the original iterator position could
// result in this key becoming visible.
return false, nil
case InternalKeyKindRangeDelete:
// RangeDeletes are handled by the merging iterator and should never
// be observed by the top-level Iterator.
panic(errors.AssertionFailedf("pebble: unexpected range delete"))
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
// Range keys are interleaved at the maximal sequence number and
// should never be observed within a user key.
panic(errors.AssertionFailedf("pebble: unexpected range key"))
default:
panic(errors.AssertionFailedf("pebble: unexpected key kind: %s", errors.Safe(kind)))
}
}
if validity == internalNextError {
return false, it.Error()
}
return true, nil
}
// internalNextValidity enumerates the potential outcomes of a call to
// internalNext.
type internalNextValidity int8
const (
// internalNextError is returned by internalNext when an error occurred and
// the caller is responsible for checking iter.Error().
internalNextError internalNextValidity = iota
// internalNextExhausted is returned by internalNext when the next internal
// key is an internal key with a different user key than Iterator.Key().
internalNextExhausted
// internalNextValid is returned by internalNext when the internal next
// found a shadowed internal key with a user key equal to Iterator.Key().
internalNextValid
)
// internalNext advances internal Iterator state forward to expose the
// InternalKeyKind of the next internal key with a user key equal to Key().
//
// internalNext is a highly specialized operation and is unlikely to be
// generally useful. See Iterator.Next for how to reposition the iterator to the
// next key. internalNext requires the Iterator to be at a valid position in the
// forward direction (the last positioning operation must've been a First, a
// Seek[Prefix]GE, or a Next[Prefix][WithLimit] and Valid() must return true).
//
// internalNext, unlike all other Iterator methods, exposes internal LSM state.
// internalNext advances the Iterator's internal iterator to the next shadowed
// key with a user key equal to Key(). When a key is overwritten or deleted, its
// removal from the LSM occurs lazily as a part of compactions. internalNext
// allows the caller to see whether an obsolete internal key exists with the
// current Key(), and what it's key kind is. Note that the existence of an
// internal key is nondeterministic and dependent on internal LSM state. These
// semantics are unlikely to be applicable to almost all use cases.
//
// If internalNext finds a key that shares the same user key as Key(), it
// returns internalNextValid and the internal key's kind. If internalNext
// encounters an error, it returns internalNextError and the caller is expected
// to call Iterator.Error() to retrieve it. In all other circumstances,
// internalNext returns internalNextExhausted, indicating that there are no more
// additional internal keys with the user key Key().
//
// internalNext does not change the external position of the iterator, and a
// Next operation should behave the same as if internalNext was never called.
// internalNext does invalidate the iterator Value's memory, and the caller must
// not rely on the memory safety of the previous Iterator position.
func (i *Iterator) internalNext() (internalNextValidity, base.InternalKeyKind) {
i.stats.ForwardStepCount[InterfaceCall]++
if i.err != nil {
return internalNextError, base.InternalKeyKindInvalid
} else if i.iterValidityState != IterValid {
return internalNextExhausted, base.InternalKeyKindInvalid
}
i.lastPositioningOp = internalNextOp
switch i.pos {
case iterPosCurForward:
i.iterKey, i.iterValue = i.iter.Next()
if i.iterKey == nil {
// We check i.iter.Error() here and return an internalNextError enum
// variant so that the caller does not need to check i.iter.Error()
// in the common case that the next internal key has a new user key.
if i.err = i.iter.Error(); i.err != nil {
return internalNextError, base.InternalKeyKindInvalid
}
i.pos = iterPosNext
return internalNextExhausted, base.InternalKeyKindInvalid
} else if i.comparer.Equal(i.iterKey.UserKey, i.key) {
return internalNextValid, i.iterKey.Kind()
}
i.pos = iterPosNext
return internalNextExhausted, base.InternalKeyKindInvalid
case iterPosCurReverse, iterPosCurReversePaused, iterPosPrev:
i.err = errors.New("switching from reverse to forward via internalNext is prohibited")
i.iterValidityState = IterExhausted
return internalNextError, base.InternalKeyKindInvalid
case iterPosNext, iterPosCurForwardPaused:
// The previous method already moved onto the next user key. This is
// only possible if
// - the last positioning method was a call to internalNext, and we
// advanced to a new user key.
// - the previous non-internalNext iterator operation encountered a
// range key or merge, forcing an internal Next that found a new
// user key that's not equal to i.Iterator.Key().
return internalNextExhausted, base.InternalKeyKindInvalid
default:
panic("unreachable")
}
}

11
vendor/github.com/cockroachdb/pebble/level_checker.go generated vendored
View File

@@ -345,7 +345,7 @@ func iterateAndCheckTombstones(
type checkConfig struct {
logger Logger
cmp Compare
comparer *Comparer
readState *readState
newIters tableNewIters
seqNum uint64
@@ -354,6 +354,9 @@ type checkConfig struct {
formatKey base.FormatKey
}
// cmp is shorthand for comparer.Compare.
func (c *checkConfig) cmp(a, b []byte) int { return c.comparer.Compare(a, b) }
func checkRangeTombstones(c *checkConfig) error {
var level int
var tombstones []tombstoneWithLevel
@@ -571,7 +574,7 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
checkConfig := &checkConfig{
logger: d.opts.Logger,
cmp: d.cmp,
comparer: d.opts.Comparer,
readState: readState,
newIters: d.newIters,
seqNum: seqNum,
@@ -639,7 +642,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
manifestIter := current.L0SublevelFiles[sublevel].Iter()
iterOpts := IterOptions{logger: c.logger}
li := &levelIter{}
li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
li.init(context.Background(), iterOpts, c.comparer, c.newIters, manifestIter,
manifest.L0Sublevel(sublevel), internalIterOpts{})
li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)
@@ -653,7 +656,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
iterOpts := IterOptions{logger: c.logger}
li := &levelIter{}
li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters,
li.init(context.Background(), iterOpts, c.comparer, c.newIters,
current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{})
li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)

51
vendor/github.com/cockroachdb/pebble/level_iter.go generated vendored
View File

@@ -37,8 +37,7 @@ type tableNewIters func(
// for the rangedel iterator returned by tableNewIters.
func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter {
return func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
iter, rangeDelIter, err := newIters(
ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters, level: iterOptions.Level}, internalIterOpts{})
iter, rangeDelIter, err := newIters(ctx, file, nil, internalIterOpts{})
if iter != nil {
_ = iter.Close()
}
@@ -51,6 +50,7 @@ func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.T
type internalIterOpts struct {
bytesIterated *uint64
bufferPool *sstable.BufferPool
stats *base.InternalIteratorStats
boundLimitedFilter sstable.BoundLimitedBlockPropertyFilter
}
@@ -82,10 +82,11 @@ type levelIter struct {
// short-lived (since they pin sstables), (b) plumbing a context into every
// method is very painful, (c) they do not (yet) respect context
// cancellation and are only used for tracing.
ctx context.Context
logger Logger
cmp Compare
split Split
ctx context.Context
logger Logger
comparer *Comparer
cmp Compare
split Split
// The lower/upper bounds for iteration as specified at creation or the most
// recent call to SetBounds.
lower []byte
@@ -241,24 +242,22 @@ var _ base.InternalIterator = (*levelIter)(nil)
// parameter if the caller is never going to call SeekPrefixGE.
func newLevelIter(
opts IterOptions,
cmp Compare,
split Split,
comparer *Comparer,
newIters tableNewIters,
files manifest.LevelIterator,
level manifest.Level,
bytesIterated *uint64,
internalOpts internalIterOpts,
) *levelIter {
l := &levelIter{}
l.init(context.Background(), opts, cmp, split, newIters, files, level,
internalIterOpts{bytesIterated: bytesIterated})
l.init(context.Background(), opts, comparer, newIters, files, level,
internalOpts)
return l
}
func (l *levelIter) init(
ctx context.Context,
opts IterOptions,
cmp Compare,
split Split,
comparer *Comparer,
newIters tableNewIters,
files manifest.LevelIterator,
level manifest.Level,
@@ -278,8 +277,9 @@ func (l *levelIter) init(
l.tableOpts.UseL6Filters = opts.UseL6Filters
l.tableOpts.level = l.level
l.tableOpts.snapshotForHideObsoletePoints = opts.snapshotForHideObsoletePoints
l.cmp = cmp
l.split = split
l.comparer = comparer
l.cmp = comparer.Compare
l.split = comparer.Split
l.iterFile = nil
l.newIters = newIters
l.files = files
@@ -797,7 +797,7 @@ func (l *levelIter) SeekPrefixGE(
}
return l.verify(l.largestBoundary, base.LazyValue{})
}
// It is possible that we are here because bloom filter matching failed. In
// It is possible that we are here because bloom filter matching failed. In
// that case it is likely that all keys matching the prefix are wholly
// within the current file and cannot be in the subsequent file. In that
// case we don't want to go to the next file, since loading and seeking in
@@ -805,7 +805,16 @@ func (l *levelIter) SeekPrefixGE(
// next file will defeat the optimization for the next SeekPrefixGE that is
// called with flags.TrySeekUsingNext(), since for sparse key spaces it is
// likely that the next key will also be contained in the current file.
if n := l.split(l.iterFile.LargestPointKey.UserKey); l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
var n int
if l.split != nil {
// If the split function is specified, calculate the prefix length accordingly.
n = l.split(l.iterFile.LargestPointKey.UserKey)
} else {
// If the split function is not specified, the entire key is used as the
// prefix. This case can occur when getIter uses SeekPrefixGE.
n = len(l.iterFile.LargestPointKey.UserKey)
}
if l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
return nil, base.LazyValue{}
}
return l.verify(l.skipEmptyFileForward())
@@ -1075,7 +1084,9 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, base.LazyValue) {
if *l.rangeDelIterPtr != nil && l.filteredIter != nil &&
l.filteredIter.MaybeFilteredKeys() {
l.largestBoundary = &l.iterFile.Largest
l.boundaryContext.isIgnorableBoundaryKey = true
if l.boundaryContext != nil {
l.boundaryContext.isIgnorableBoundaryKey = true
}
return l.largestBoundary, base.LazyValue{}
}
}
@@ -1164,7 +1175,9 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, base.LazyValue) {
// the next file.
if *l.rangeDelIterPtr != nil && l.filteredIter != nil && l.filteredIter.MaybeFilteredKeys() {
l.smallestBoundary = &l.iterFile.Smallest
l.boundaryContext.isIgnorableBoundaryKey = true
if l.boundaryContext != nil {
l.boundaryContext.isIgnorableBoundaryKey = true
}
return l.smallestBoundary, base.LazyValue{}
}
}

92
vendor/github.com/cockroachdb/pebble/mem_table.go generated vendored
View File

@@ -10,12 +10,12 @@ import (
"os"
"sync"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/arenaskl"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manual"
"github.com/cockroachdb/pebble/internal/rangedel"
"github.com/cockroachdb/pebble/internal/rangekey"
)
@@ -85,7 +85,16 @@ type memTable struct {
rangeKeys keySpanCache
// The current logSeqNum at the time the memtable was created. This is
// guaranteed to be less than or equal to any seqnum stored in the memtable.
logSeqNum uint64
logSeqNum uint64
releaseAccountingReservation func()
}
func (m *memTable) free() {
if m != nil {
m.releaseAccountingReservation()
manual.Free(m.arenaBuf)
m.arenaBuf = nil
}
}
// memTableOptions holds configuration used when creating a memTable. All of
@@ -93,9 +102,10 @@ type memTable struct {
// which is used by tests.
type memTableOptions struct {
*Options
arenaBuf []byte
size int
logSeqNum uint64
arenaBuf []byte
size int
logSeqNum uint64
releaseAccountingReservation func()
}
func checkMemTable(obj interface{}) {
@@ -110,16 +120,22 @@ func checkMemTable(obj interface{}) {
// Options.MemTableSize is used instead.
func newMemTable(opts memTableOptions) *memTable {
opts.Options = opts.Options.EnsureDefaults()
if opts.size == 0 {
opts.size = opts.MemTableSize
}
m := new(memTable)
m.init(opts)
return m
}
m := &memTable{
cmp: opts.Comparer.Compare,
formatKey: opts.Comparer.FormatKey,
equal: opts.Comparer.Equal,
arenaBuf: opts.arenaBuf,
logSeqNum: opts.logSeqNum,
func (m *memTable) init(opts memTableOptions) {
if opts.size == 0 {
opts.size = int(opts.MemTableSize)
}
*m = memTable{
cmp: opts.Comparer.Compare,
formatKey: opts.Comparer.FormatKey,
equal: opts.Comparer.Equal,
arenaBuf: opts.arenaBuf,
logSeqNum: opts.logSeqNum,
releaseAccountingReservation: opts.releaseAccountingReservation,
}
m.writerRefs.Store(1)
m.tombstones = keySpanCache{
@@ -144,7 +160,6 @@ func newMemTable(opts memTableOptions) *memTable {
m.rangeDelSkl.Reset(arena, m.cmp)
m.rangeKeySkl.Reset(arena, m.cmp)
m.reserved = arena.Size()
return m
}
func (m *memTable) writerRef() {
@@ -154,7 +169,8 @@ func (m *memTable) writerRef() {
}
}
func (m *memTable) writerUnref() bool {
// writerUnref drops a ref on the memtable. Returns true if this was the last ref.
func (m *memTable) writerUnref() (wasLastRef bool) {
switch v := m.writerRefs.Add(-1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
@@ -165,6 +181,7 @@ func (m *memTable) writerUnref() bool {
}
}
// readyForFlush is part of the flushable interface.
func (m *memTable) readyForFlush() bool {
return m.writerRefs.Load() == 0
}
@@ -194,11 +211,13 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
var tombstoneCount, rangeKeyCount uint32
startSeqNum := seqNum
for r := batch.Reader(); ; seqNum++ {
kind, ukey, value, ok := r.Next()
kind, ukey, value, ok, err := r.Next()
if !ok {
if err != nil {
return err
}
break
}
var err error
ikey := base.MakeInternalKey(ukey, seqNum, kind)
switch kind {
case InternalKeyKindRangeDelete:
@@ -233,17 +252,19 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
return nil
}
// newIter returns an iterator that is unpositioned (Iterator.Valid() will
// return false). The iterator can be positioned via a call to SeekGE,
// SeekLT, First or Last.
// newIter is part of the flushable interface. It returns an iterator that is
// unpositioned (Iterator.Valid() will return false). The iterator can be
// positioned via a call to SeekGE, SeekLT, First or Last.
func (m *memTable) newIter(o *IterOptions) internalIterator {
return m.skl.NewIter(o.GetLowerBound(), o.GetUpperBound())
}
// newFlushIter is part of the flushable interface.
func (m *memTable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
return m.skl.NewFlushIter(bytesFlushed)
}
// newRangeDelIter is part of the flushable interface.
func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
tombstones := m.tombstones.get()
if tombstones == nil {
@@ -252,6 +273,7 @@ func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
return keyspan.NewIter(m.cmp, tombstones)
}
// newRangeKeyIter is part of the flushable interface.
func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
rangeKeys := m.rangeKeys.get()
if rangeKeys == nil {
@@ -260,8 +282,9 @@ func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
return keyspan.NewIter(m.cmp, rangeKeys)
}
// containsRangeKeys is part of the flushable interface.
func (m *memTable) containsRangeKeys() bool {
return atomic.LoadUint32(&m.rangeKeys.atomicCount) > 0
return m.rangeKeys.count.Load() > 0
}
func (m *memTable) availBytes() uint32 {
@@ -275,10 +298,12 @@ func (m *memTable) availBytes() uint32 {
return a.Capacity() - m.reserved
}
// inuseBytes is part of the flushable interface.
func (m *memTable) inuseBytes() uint64 {
return uint64(m.skl.Size() - memTableEmptySize)
}
// totalBytes is part of the flushable interface.
func (m *memTable) totalBytes() uint64 {
return uint64(m.skl.Arena().Capacity())
}
@@ -355,8 +380,8 @@ func (f *keySpanFrags) get(
// invalidated whenever a key of the same kind is added to a memTable, and
// populated when empty when a span iterator of that key kind is created.
type keySpanCache struct {
atomicCount uint32
frags unsafe.Pointer
count atomic.Uint32
frags atomic.Pointer[keySpanFrags]
cmp Compare
formatKey base.FormatKey
constructSpan constructSpan
@@ -366,23 +391,20 @@ type keySpanCache struct {
// Invalidate the current set of cached spans, indicating the number of
// spans that were added.
func (c *keySpanCache) invalidate(count uint32) {
newCount := atomic.AddUint32(&c.atomicCount, count)
newCount := c.count.Add(count)
var frags *keySpanFrags
for {
oldPtr := atomic.LoadPointer(&c.frags)
if oldPtr != nil {
oldFrags := (*keySpanFrags)(oldPtr)
if oldFrags.count >= newCount {
// Someone else invalidated the cache before us and their invalidation
// subsumes ours.
break
}
oldFrags := c.frags.Load()
if oldFrags != nil && oldFrags.count >= newCount {
// Someone else invalidated the cache before us and their invalidation
// subsumes ours.
break
}
if frags == nil {
frags = &keySpanFrags{count: newCount}
}
if atomic.CompareAndSwapPointer(&c.frags, oldPtr, unsafe.Pointer(frags)) {
if c.frags.CompareAndSwap(oldFrags, frags) {
// We successfully invalidated the cache.
break
}
@@ -391,7 +413,7 @@ func (c *keySpanCache) invalidate(count uint32) {
}
func (c *keySpanCache) get() []keyspan.Span {
frags := (*keySpanFrags)(atomic.LoadPointer(&c.frags))
frags := c.frags.Load()
if frags == nil {
return nil
}

10
vendor/github.com/cockroachdb/pebble/merging_iter.go generated vendored
View File

@@ -27,6 +27,10 @@ type mergingIterLevel struct {
// iterKey and iterValue cache the current key and value iter are pointed at.
iterKey *InternalKey
iterValue base.LazyValue
// levelIter is non-nil if this level's iter is ultimately backed by a
// *levelIter. The handle in iter may have wrapped the levelIter with
// intermediary internalIterator implementations.
levelIter *levelIter
// levelIterBoundaryContext's fields are set when using levelIter, in order
// to surface sstable boundary keys and file-level context. See levelIter
@@ -1371,9 +1375,9 @@ func (m *mergingIter) DebugString() string {
}
func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) {
for _, iter := range m.levels {
if li, ok := iter.iter.(*levelIter); ok {
if done := fn(li); done {
for _, ml := range m.levels {
if ml.levelIter != nil {
if done := fn(ml.levelIter); done {
break
}
}

401
vendor/github.com/cockroachdb/pebble/metrics.go generated vendored
View File

@@ -6,12 +6,13 @@ package pebble
import (
"fmt"
"strings"
"math"
"time"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/redact"
@@ -28,13 +29,10 @@ type FilterMetrics = sstable.FilterMetrics
// comment in base.
type ThroughputMetric = base.ThroughputMetric
func formatCacheMetrics(w redact.SafePrinter, m *CacheMetrics, name redact.SafeString) {
w.Printf("%7s %9s %7s %6.1f%% (score == hit-rate)\n",
name,
humanize.SI.Int64(m.Count),
humanize.IEC.Int64(m.Size),
redact.Safe(hitRate(m.Hits, m.Misses)))
}
// SecondaryCacheMetrics holds metrics for the persistent secondary cache
// that caches commonly accessed blocks from blob storage on a local
// file system.
type SecondaryCacheMetrics = sharedcache.Metrics
// LevelMetrics holds per-level metrics such as the number of files and total
// size of the files, and compaction related metrics.
@@ -46,9 +44,14 @@ type LevelMetrics struct {
Sublevels int32
// The total number of files in the level.
NumFiles int64
// The total number of virtual sstables in the level.
NumVirtualFiles uint64
// The total size in bytes of the files in the level.
Size int64
// The level's compaction score.
// The total size of the virtual sstables in the level.
VirtualSize uint64
// The level's compaction score. This is the compensatedScoreRatio in the
// candidateLevelInfo.
Score float64
// The number of incoming bytes from other levels read during
// compactions. This excludes bytes moved and bytes ingested. For L0 this is
@@ -79,6 +82,18 @@ type LevelMetrics struct {
TablesIngested uint64
// The number of sstables moved to this level by a "move" compaction.
TablesMoved uint64
MultiLevel struct {
// BytesInTop are the total bytes in a multilevel compaction coming from the top level.
BytesInTop uint64
// BytesIn, exclusively for multiLevel compactions.
BytesIn uint64
// BytesRead, exclusively for multilevel compactions.
BytesRead uint64
}
// Additional contains misc additional metrics that are not always printed.
Additional struct {
// The sum of Properties.ValueBlocksSize for all the sstables in this
@@ -96,6 +111,8 @@ type LevelMetrics struct {
// Add updates the counter metrics for the level.
func (m *LevelMetrics) Add(u *LevelMetrics) {
m.NumFiles += u.NumFiles
m.NumVirtualFiles += u.NumVirtualFiles
m.VirtualSize += u.VirtualSize
m.Size += u.Size
m.BytesIn += u.BytesIn
m.BytesIngested += u.BytesIngested
@@ -107,6 +124,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) {
m.TablesFlushed += u.TablesFlushed
m.TablesIngested += u.TablesIngested
m.TablesMoved += u.TablesMoved
m.MultiLevel.BytesInTop += u.MultiLevel.BytesInTop
m.MultiLevel.BytesRead += u.MultiLevel.BytesRead
m.MultiLevel.BytesIn += u.MultiLevel.BytesIn
m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks
m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks
m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize
@@ -121,32 +141,6 @@ func (m *LevelMetrics) WriteAmp() float64 {
return float64(m.BytesFlushed+m.BytesCompacted) / float64(m.BytesIn)
}
// format generates a string of the receiver's metrics, formatting it into the
// supplied buffer.
func (m *LevelMetrics) format(
w redact.SafePrinter, score redact.SafeValue, includeValueBlocksSize bool,
) {
w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f",
redact.Safe(m.NumFiles),
humanize.IEC.Int64(m.Size),
score,
humanize.IEC.Uint64(m.BytesIn),
humanize.IEC.Uint64(m.BytesIngested),
humanize.SI.Uint64(m.TablesIngested),
humanize.IEC.Uint64(m.BytesMoved),
humanize.SI.Uint64(m.TablesMoved),
humanize.IEC.Uint64(m.BytesFlushed+m.BytesCompacted),
humanize.SI.Uint64(m.TablesFlushed+m.TablesCompacted),
humanize.IEC.Uint64(m.BytesRead),
redact.Safe(m.Sublevels),
redact.Safe(m.WriteAmp()))
if includeValueBlocksSize {
w.Printf(" %7s\n", humanize.IEC.Uint64(m.Additional.ValueBlocksSize))
} else {
w.SafeString("\n")
}
}
// Metrics holds metrics for various subsystems of the DB such as the Cache,
// Compactions, WAL, and per-Level metrics.
//
@@ -158,14 +152,15 @@ type Metrics struct {
Compact struct {
// The total number of compactions, and per-compaction type counts.
Count int64
DefaultCount int64
DeleteOnlyCount int64
ElisionOnlyCount int64
MoveCount int64
ReadCount int64
RewriteCount int64
MultiLevelCount int64
Count int64
DefaultCount int64
DeleteOnlyCount int64
ElisionOnlyCount int64
MoveCount int64
ReadCount int64
RewriteCount int64
MultiLevelCount int64
CounterLevelCount int64
// An estimate of the number of bytes that need to be compacted for the LSM
// to reach a stable state.
EstimatedDebt uint64
@@ -218,7 +213,10 @@ type Metrics struct {
// The count of memtables.
Count int64
// The number of bytes present in zombie memtables which are no longer
// referenced by the current DB state but are still in use by an iterator.
// referenced by the current DB state. An unbounded number of memtables
// may be zombie if they're still in use by an iterator. One additional
// memtable may be zombie if it's no longer in use and waiting to be
// recycled.
ZombieSize uint64
// The count of zombie memtables.
ZombieCount int64
@@ -261,6 +259,10 @@ type Metrics struct {
ZombieSize uint64
// The count of zombie tables.
ZombieCount int64
// The count of the backing sstables.
BackingTableCount uint64
// The sum of the sizes of the all of the backing sstables.
BackingTableSize uint64
}
TableCache CacheMetrics
@@ -294,6 +296,8 @@ type Metrics struct {
record.LogWriterMetrics
}
SecondaryCacheMetrics SecondaryCacheMetrics
private struct {
optionsFileSize uint64
manifestFileSize uint64
@@ -307,6 +311,13 @@ var (
prometheus.LinearBuckets(0.0, float64(time.Microsecond*100), 50),
prometheus.ExponentialBucketsRange(float64(time.Millisecond*5), float64(10*time.Second), 50)...,
)
// SecondaryCacheIOBuckets exported to enable exporting from package pebble to
// enable exporting metrics with below buckets in CRDB.
SecondaryCacheIOBuckets = sharedcache.IOBuckets
// SecondaryCacheChannelWriteBuckets exported to enable exporting from package
// pebble to enable exporting metrics with below buckets in CRDB.
SecondaryCacheChannelWriteBuckets = sharedcache.ChannelWriteBuckets
)
// DiskSpaceUsage returns the total disk space used by the database in bytes,
@@ -326,12 +337,25 @@ func (m *Metrics) DiskSpaceUsage() uint64 {
return usageBytes
}
func (m *Metrics) levelSizes() [numLevels]int64 {
var sizes [numLevels]int64
for i := 0; i < len(sizes); i++ {
sizes[i] = m.Levels[i].Size
// NumVirtual is the number of virtual sstables in the latest version
// summed over every level in the lsm.
func (m *Metrics) NumVirtual() uint64 {
var n uint64
for _, level := range m.Levels {
n += level.NumVirtualFiles
}
return sizes
return n
}
// VirtualSize is the sum of the sizes of the virtual sstables in the
// latest version. BackingTableSize - VirtualSize gives an estimate for
// the space amplification caused by not compacting virtual sstables.
func (m *Metrics) VirtualSize() uint64 {
var size uint64
for _, level := range m.Levels {
size += level.VirtualSize
}
return size
}
// ReadAmp returns the current read amplification of the database.
@@ -362,59 +386,34 @@ func (m *Metrics) Total() LevelMetrics {
return total
}
const notApplicable = redact.SafeString("-")
func (m *Metrics) formatWAL(w redact.SafePrinter) {
var writeAmp float64
if m.WAL.BytesIn > 0 {
writeAmp = float64(m.WAL.BytesWritten) / float64(m.WAL.BytesIn)
}
w.Printf(" WAL %9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7.1f\n",
redact.Safe(m.WAL.Files),
humanize.Uint64(m.WAL.Size),
notApplicable,
humanize.Uint64(m.WAL.BytesIn),
notApplicable,
notApplicable,
notApplicable,
notApplicable,
humanize.Uint64(m.WAL.BytesWritten),
notApplicable,
notApplicable,
notApplicable,
redact.Safe(writeAmp))
}
// String pretty-prints the metrics, showing a line for the WAL, a line per-level, and
// a total:
// String pretty-prints the metrics as below:
//
// __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp
// WAL 1 28 B - 17 B - - - - 56 B - - - 3.3
// 0 1 770 B 0.25 28 B 0 B 0 0 B 0 770 B 1 0 B 1 27.5
// 1 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 2 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 3 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 4 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 5 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 6 0 0 B - 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// total 1 770 B - 56 B 0 B 0 0 B 0 826 B 1 0 B 1 14.8
// flush 1 0 B 0 0 (ingest = ingested-as-flushable, move = tables-ingested)
// compact 0 0 B 0 B 0 (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)
// ctype 0 0 0 0 0 0 0 (default, delete, elision, move, read, rewrite, multi-level)
// memtbl 1 256 K
// zmemtbl 1 256 K
// ztbl 0 0 B
// bcache 4 697 B 0.0% (score == hit-rate)
// tcache 1 696 B 0.0% (score == hit-rate)
// snaps 0 - 0 (score == earliest seq num)
// titers 1
// filter - - 0.0% (score == utility)
//
// The WAL "in" metric is the size of the batches written to the WAL. The WAL
// "write" metric is the size of the physical data written to the WAL which
// includes record fragment overhead. Write amplification is computed as
// bytes-written / bytes-in, except for the total row where bytes-in is
// replaced with WAL-bytes-written + bytes-ingested.
// | | | | ingested | moved | written | | amp
// level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w
// ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------
// 0 | 101 102B 0B 0 | 103.0 | 104B | 112 104B | 113 106B | 221 217B | 107B | 1 2.1
// 1 | 201 202B 0B 0 | 203.0 | 204B | 212 204B | 213 206B | 421 417B | 207B | 2 2.0
// 2 | 301 302B 0B 0 | 303.0 | 304B | 312 304B | 313 306B | 621 617B | 307B | 3 2.0
// 3 | 401 402B 0B 0 | 403.0 | 404B | 412 404B | 413 406B | 821 817B | 407B | 4 2.0
// 4 | 501 502B 0B 0 | 503.0 | 504B | 512 504B | 513 506B | 1.0K 1017B | 507B | 5 2.0
// 5 | 601 602B 0B 0 | 603.0 | 604B | 612 604B | 613 606B | 1.2K 1.2KB | 607B | 6 2.0
// 6 | 701 702B 0B 0 | - | 704B | 712 704B | 713 706B | 1.4K 1.4KB | 707B | 7 2.0
// total | 2.8K 2.7KB 0B 0 | - | 2.8KB | 2.9K 2.8KB | 2.9K 2.8KB | 5.7K 8.4KB | 2.8KB | 28 3.0
// -------------------------------------------------------------------------------------------------------------------
// WAL: 22 files (24B) in: 25B written: 26B (4% overhead)
// Flushes: 8
// Compactions: 5 estimated debt: 6B in progress: 2 (7B)
// default: 27 delete: 28 elision: 29 move: 30 read: 31 rewrite: 32 multi-level: 33
// MemTables: 12 (11B) zombie: 14 (13B)
// Zombie tables: 16 (15B)
// Backing tables: 0 (0B)
// Block cache: 2 entries (1B) hit rate: 42.9%
// Table cache: 18 entries (17B) hit rate: 48.7%
// Secondary cache: 40 entries (40B) hit rate: 49.9%
// Snapshots: 4 earliest seq num: 1024
// Table iters: 21
// Filter utility: 47.4%
// Ingestions: 27 as flushable: 36 (34B in 35 tables)
func (m *Metrics) String() string {
return redact.StringWithoutMarkers(m)
}
@@ -435,29 +434,84 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
// width specifiers. When the issue is fixed, we can convert these to
// RedactableStrings. https://github.com/cockroachdb/redact/issues/17
haveValueBlocks := false
var valueBlocksHeading redact.SafeString
for level := 0; level < numLevels; level++ {
if m.Levels[level].Additional.ValueBlocksSize > 0 {
haveValueBlocks = true
valueBlocksHeading = "__val-bl"
break
multiExists := m.Compact.MultiLevelCount > 0
appendIfMulti := func(line redact.SafeString) {
if multiExists {
w.SafeString(line)
}
}
newline := func() {
w.SafeString("\n")
}
w.SafeString(" | | | | ingested | moved | written | | amp")
appendIfMulti(" | multilevel")
newline()
w.SafeString("level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w")
appendIfMulti(" | top in read")
newline()
w.SafeString("------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------")
appendIfMulti("-+------------------")
newline()
// formatRow prints out a row of the table.
formatRow := func(m *LevelMetrics, score float64) {
scoreStr := "-"
if !math.IsNaN(score) {
// Try to keep the string no longer than 5 characters.
switch {
case score < 99.995:
scoreStr = fmt.Sprintf("%.2f", score)
case score < 999.95:
scoreStr = fmt.Sprintf("%.1f", score)
default:
scoreStr = fmt.Sprintf("%.0f", score)
}
}
var wampStr string
if wamp := m.WriteAmp(); wamp > 99.5 {
wampStr = fmt.Sprintf("%.0f", wamp)
} else {
wampStr = fmt.Sprintf("%.1f", wamp)
}
w.Printf("| %5s %6s %6s %7s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s",
humanize.Count.Int64(m.NumFiles),
humanize.Bytes.Int64(m.Size),
humanize.Bytes.Uint64(m.Additional.ValueBlocksSize),
humanize.Count.Uint64(m.NumVirtualFiles),
redact.Safe(scoreStr),
humanize.Bytes.Uint64(m.BytesIn),
humanize.Count.Uint64(m.TablesIngested),
humanize.Bytes.Uint64(m.BytesIngested),
humanize.Count.Uint64(m.TablesMoved),
humanize.Bytes.Uint64(m.BytesMoved),
humanize.Count.Uint64(m.TablesFlushed+m.TablesCompacted),
humanize.Bytes.Uint64(m.BytesFlushed+m.BytesCompacted),
humanize.Bytes.Uint64(m.BytesRead),
redact.Safe(m.Sublevels),
redact.Safe(wampStr))
if multiExists {
w.Printf(" | %5s %5s %5s",
humanize.Bytes.Uint64(m.MultiLevel.BytesInTop),
humanize.Bytes.Uint64(m.MultiLevel.BytesIn),
humanize.Bytes.Uint64(m.MultiLevel.BytesRead))
}
newline()
}
var total LevelMetrics
w.Printf("__level_____count____size___score______in__ingest(sz_cnt)"+
"____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp%s\n", valueBlocksHeading)
m.formatWAL(w)
for level := 0; level < numLevels; level++ {
l := &m.Levels[level]
w.Printf("%7d ", redact.Safe(level))
w.Printf("%5d ", redact.Safe(level))
// Format the score.
var score redact.SafeValue = notApplicable
score := math.NaN()
if level < numLevels-1 {
score = redact.Safe(fmt.Sprintf("%0.2f", l.Score))
score = l.Score
}
l.format(w, score, haveValueBlocks)
formatRow(l, score)
total.Add(l)
total.Sublevels += l.Sublevels
}
@@ -467,60 +521,105 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
// the bytes written to the log and bytes written externally and then
// ingested.
total.BytesFlushed += total.BytesIn
w.SafeString(" total ")
total.format(w, notApplicable, haveValueBlocks)
w.SafeString("total ")
formatRow(&total, math.NaN())
w.Printf(" flush %9d %31s %7d %7d %s\n",
redact.Safe(m.Flush.Count),
humanize.IEC.Uint64(m.Flush.AsIngestBytes),
redact.Safe(m.Flush.AsIngestTableCount),
redact.Safe(m.Flush.AsIngestCount),
redact.SafeString(`(ingest = tables-ingested, move = ingested-as-flushable)`))
w.Printf("compact %9d %7s %7s %7d %s %s\n",
w.SafeString("-------------------------------------------------------------------------------------------------------------------")
appendIfMulti("--------------------")
newline()
w.Printf("WAL: %d files (%s) in: %s written: %s (%.0f%% overhead)\n",
redact.Safe(m.WAL.Files),
humanize.Bytes.Uint64(m.WAL.Size),
humanize.Bytes.Uint64(m.WAL.BytesIn),
humanize.Bytes.Uint64(m.WAL.BytesWritten),
redact.Safe(percent(int64(m.WAL.BytesWritten)-int64(m.WAL.BytesIn), int64(m.WAL.BytesIn))))
w.Printf("Flushes: %d\n", redact.Safe(m.Flush.Count))
w.Printf("Compactions: %d estimated debt: %s in progress: %d (%s)\n",
redact.Safe(m.Compact.Count),
humanize.IEC.Uint64(m.Compact.EstimatedDebt),
humanize.IEC.Int64(m.Compact.InProgressBytes),
humanize.Bytes.Uint64(m.Compact.EstimatedDebt),
redact.Safe(m.Compact.NumInProgress),
redact.SafeString(strings.Repeat(" ", 24)),
redact.SafeString(`(size == estimated-debt, score = in-progress-bytes, in = num-in-progress)`))
w.Printf(" ctype %9d %7d %7d %7d %7d %7d %7d %s\n",
humanize.Bytes.Int64(m.Compact.InProgressBytes))
w.Printf(" default: %d delete: %d elision: %d move: %d read: %d rewrite: %d multi-level: %d\n",
redact.Safe(m.Compact.DefaultCount),
redact.Safe(m.Compact.DeleteOnlyCount),
redact.Safe(m.Compact.ElisionOnlyCount),
redact.Safe(m.Compact.MoveCount),
redact.Safe(m.Compact.ReadCount),
redact.Safe(m.Compact.RewriteCount),
redact.Safe(m.Compact.MultiLevelCount),
redact.SafeString(`(default, delete, elision, move, read, rewrite, multi-level)`))
w.Printf(" memtbl %9d %7s\n",
redact.Safe(m.Compact.MultiLevelCount))
w.Printf("MemTables: %d (%s) zombie: %d (%s)\n",
redact.Safe(m.MemTable.Count),
humanize.IEC.Uint64(m.MemTable.Size))
w.Printf("zmemtbl %9d %7s\n",
humanize.Bytes.Uint64(m.MemTable.Size),
redact.Safe(m.MemTable.ZombieCount),
humanize.IEC.Uint64(m.MemTable.ZombieSize))
w.Printf(" ztbl %9d %7s\n",
humanize.Bytes.Uint64(m.MemTable.ZombieSize))
w.Printf("Zombie tables: %d (%s)\n",
redact.Safe(m.Table.ZombieCount),
humanize.IEC.Uint64(m.Table.ZombieSize))
formatCacheMetrics(w, &m.BlockCache, "bcache")
formatCacheMetrics(w, &m.TableCache, "tcache")
w.Printf(" snaps %9d %7s %7d (score == earliest seq num)\n",
humanize.Bytes.Uint64(m.Table.ZombieSize))
w.Printf("Backing tables: %d (%s)\n",
redact.Safe(m.Table.BackingTableCount),
humanize.Bytes.Uint64(m.Table.BackingTableSize))
w.Printf("Virtual tables: %d (%s)\n",
redact.Safe(m.NumVirtual()),
humanize.Bytes.Uint64(m.VirtualSize()))
formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
w.Printf("%s: %s entries (%s) hit rate: %.1f%%\n",
name,
humanize.Count.Int64(m.Count),
humanize.Bytes.Int64(m.Size),
redact.Safe(hitRate(m.Hits, m.Misses)))
}
formatCacheMetrics(&m.BlockCache, "Block cache")
formatCacheMetrics(&m.TableCache, "Table cache")
formatSharedCacheMetrics := func(w redact.SafePrinter, m *SecondaryCacheMetrics, name redact.SafeString) {
w.Printf("%s: %s entries (%s) hit rate: %.1f%%\n",
name,
humanize.Count.Int64(m.Count),
humanize.Bytes.Int64(m.Size),
redact.Safe(hitRate(m.ReadsWithFullHit, m.ReadsWithPartialHit+m.ReadsWithNoHit)))
}
formatSharedCacheMetrics(w, &m.SecondaryCacheMetrics, "Secondary cache")
w.Printf("Snapshots: %d earliest seq num: %d\n",
redact.Safe(m.Snapshots.Count),
notApplicable,
redact.Safe(m.Snapshots.EarliestSeqNum))
w.Printf(" titers %9d\n", redact.Safe(m.TableIters))
w.Printf(" filter %9s %7s %6.1f%% (score == utility)\n",
notApplicable,
notApplicable,
redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
w.Printf(" ingest %9d\n",
w.Printf("Table iters: %d\n", redact.Safe(m.TableIters))
w.Printf("Filter utility: %.1f%%\n", redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
w.Printf("Ingestions: %d as flushable: %d (%s in %d tables)\n",
redact.Safe(m.Ingest.Count),
)
redact.Safe(m.Flush.AsIngestCount),
humanize.Bytes.Uint64(m.Flush.AsIngestBytes),
redact.Safe(m.Flush.AsIngestTableCount))
}
func hitRate(hits, misses int64) float64 {
sum := hits + misses
if sum == 0 {
return percent(hits, hits+misses)
}
func percent(numerator, denominator int64) float64 {
if denominator == 0 {
return 0
}
return 100 * float64(hits) / float64(sum)
return 100 * float64(numerator) / float64(denominator)
}
// StringForTests is identical to m.String() on 64-bit platforms. It is used to
// provide a platform-independent result for tests.
func (m *Metrics) StringForTests() string {
mCopy := *m
if math.MaxInt == math.MaxInt32 {
// This is the difference in Sizeof(sstable.Reader{})) between 64 and 32 bit
// platforms.
const tableCacheSizeAdjustment = 212
mCopy.TableCache.Size += mCopy.TableCache.Count * tableCacheSizeAdjustment
}
return redact.StringWithoutMarkers(&mCopy)
}

112
vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go generated vendored
View File

@@ -8,7 +8,10 @@ import (
"context"
"fmt"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
)
@@ -87,21 +90,71 @@ type ObjectMetadata struct {
DiskFileNum base.DiskFileNum
FileType base.FileType
// The fields below are only set if the object is on shared storage.
Shared struct {
// The fields below are only set if the object is on remote storage.
Remote struct {
// CreatorID identifies the DB instance that originally created the object.
//
// Only used when CustomObjectName is not set.
CreatorID CreatorID
// CreatorFileNum is the identifier for the object within the context of the
// DB instance that originally created the object.
//
// Only used when CustomObjectName is not set.
CreatorFileNum base.DiskFileNum
// CustomObjectName (if it is set) overrides the object name that is normally
// derived from the CreatorID and CreatorFileNum.
CustomObjectName string
// CleanupMethod indicates the method for cleaning up unused shared objects.
CleanupMethod SharedCleanupMethod
// Locator identifies the remote.Storage implementation for this object.
Locator remote.Locator
// Storage is the remote.Storage object corresponding to the Locator. Used
// to avoid lookups in hot paths.
Storage remote.Storage
}
}
// IsShared returns true if the object is on shared storage.
// IsRemote returns true if the object is on remote storage.
func (meta *ObjectMetadata) IsRemote() bool {
return meta.IsShared() || meta.IsExternal()
}
// IsExternal returns true if the object is on remote storage but is not owned
// by any Pebble instances in the cluster.
func (meta *ObjectMetadata) IsExternal() bool {
return meta.Remote.CustomObjectName != ""
}
// IsShared returns true if the object is on remote storage and is owned by a
// Pebble instance in the cluster (potentially shared between multiple
// instances).
func (meta *ObjectMetadata) IsShared() bool {
return meta.Shared.CreatorID.IsSet()
return meta.Remote.CreatorID.IsSet()
}
// AssertValid checks that the metadata is sane.
func (meta *ObjectMetadata) AssertValid() {
if !meta.IsRemote() {
// Verify all Remote fields are empty.
if meta.Remote != (ObjectMetadata{}).Remote {
panic(errors.AssertionFailedf("meta.Remote not empty: %#v", meta.Remote))
}
} else {
if meta.Remote.CustomObjectName != "" {
if meta.Remote.CreatorID == 0 {
panic(errors.AssertionFailedf("CreatorID not set"))
}
if meta.Remote.CreatorFileNum == base.FileNum(0).DiskFileNum() {
panic(errors.AssertionFailedf("CreatorFileNum not set"))
}
}
if meta.Remote.CleanupMethod != SharedNoCleanup && meta.Remote.CleanupMethod != SharedRefTracking {
panic(errors.AssertionFailedf("invalid CleanupMethod %d", meta.Remote.CleanupMethod))
}
if meta.Remote.Storage == nil {
panic(errors.AssertionFailedf("Storage not set"))
}
}
}
// CreatorID identifies the DB instance that originally created a shared object.
@@ -122,7 +175,7 @@ const (
// keep track of references via reference marker objects.
SharedRefTracking SharedCleanupMethod = iota
// SharedNoCleanup is used for shared objects that are managed externally; the
// SharedNoCleanup is used for remote objects that are managed externally; the
// objstorage provider never deletes such objects.
SharedNoCleanup
)
@@ -155,7 +208,7 @@ type CreateOptions struct {
// created by the provider, or existing objects the Provider was informed about
// via AddObjects.
//
// Objects are currently backed by a vfs.File or a shared.Storage object.
// Objects are currently backed by a vfs.File or a remote.Storage object.
type Provider interface {
// OpenForReading opens an existing object.
OpenForReading(
@@ -210,55 +263,62 @@ type Provider interface {
List() []ObjectMetadata
// SetCreatorID sets the CreatorID which is needed in order to use shared
// objects. Shared object usage is disabled until this method is called the
// objects. Remote object usage is disabled until this method is called the
// first time. Once set, the Creator ID is persisted and cannot change.
//
// Cannot be called if shared storage is not configured for the provider.
SetCreatorID(creatorID CreatorID) error
// IsForeign returns whether this object is owned by a different node. Return
// value undefined if creator ID is not set yet, or if this object does not
// exist in this provider.
IsForeign(meta ObjectMetadata) bool
// IsSharedForeign returns whether this object is owned by a different node.
IsSharedForeign(meta ObjectMetadata) bool
// SharedObjectBacking encodes the shared object metadata.
SharedObjectBacking(meta *ObjectMetadata) (SharedObjectBackingHandle, error)
// RemoteObjectBacking encodes the remote object metadata for the given object.
RemoteObjectBacking(meta *ObjectMetadata) (RemoteObjectBackingHandle, error)
// AttachSharedObjects registers existing shared objects with this provider.
AttachSharedObjects(objs []SharedObjectToAttach) ([]ObjectMetadata, error)
// CreateExternalObjectBacking creates a backing for an existing object with a
// custom object name. The object is considered to be managed outside of
// Pebble and will never be removed by Pebble.
CreateExternalObjectBacking(locator remote.Locator, objName string) (RemoteObjectBacking, error)
// AttachRemoteObjects registers existing remote objects with this provider.
AttachRemoteObjects(objs []RemoteObjectToAttach) ([]ObjectMetadata, error)
Close() error
// IsNotExistError indicates whether the error is known to report that a file or
// directory does not exist.
IsNotExistError(err error) bool
// Metrics returns metrics about objstorage. Currently, it only returns metrics
// about the shared cache.
Metrics() sharedcache.Metrics
}
// SharedObjectBacking encodes the metadata necessary to incorporate a shared
// RemoteObjectBacking encodes the metadata necessary to incorporate a shared
// object into a different Pebble instance. The encoding is specific to a given
// Provider implementation.
type SharedObjectBacking []byte
type RemoteObjectBacking []byte
// SharedObjectBackingHandle is a container for a SharedObjectBacking which
// RemoteObjectBackingHandle is a container for a RemoteObjectBacking which
// ensures that the backing stays valid. A backing can otherwise become invalid
// if this provider unrefs the shared object. The SharedObjectBackingHandle
// if this provider unrefs the shared object. The RemoteObjectBackingHandle
// delays any unref until Close.
type SharedObjectBackingHandle interface {
type RemoteObjectBackingHandle interface {
// Get returns the backing. The backing is only guaranteed to be valid until
// Close is called (or until the Provider is closed). If Close was already
// called, returns an error.
Get() (SharedObjectBacking, error)
Get() (RemoteObjectBacking, error)
Close()
}
// SharedObjectToAttach contains the arguments needed to attach an existing shared object.
type SharedObjectToAttach struct {
// RemoteObjectToAttach contains the arguments needed to attach an existing remote object.
type RemoteObjectToAttach struct {
// FileNum is the file number that will be used to refer to this object (in
// the context of this instance).
FileNum base.DiskFileNum
FileType base.FileType
// Backing contains the metadata for the shared object backing (normally
// Backing contains the metadata for the remote object backing (normally
// generated from a different instance, but using the same Provider
// implementation).
Backing SharedObjectBacking
Backing RemoteObjectBacking
}

185
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go generated vendored
View File

@@ -17,8 +17,9 @@ import (
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
)
@@ -30,18 +31,20 @@ type provider struct {
tracer *objiotracing.Tracer
shared sharedSubsystem
remote remoteSubsystem
mu struct {
sync.RWMutex
shared struct {
// catalogBatch accumulates shared object creations and deletions until
remote struct {
// catalogBatch accumulates remote object creations and deletions until
// Sync is called.
catalogBatch sharedobjcat.Batch
catalogBatch remoteobjcat.Batch
storageObjects map[remote.Locator]remote.Storage
}
// localObjectsChanged is set if non-shared objects were created or deleted
// localObjectsChanged is set if non-remote objects were created or deleted
// but Sync was not yet called.
localObjectsChanged bool
@@ -89,18 +92,44 @@ type Settings struct {
// out a large chunk of dirty filesystem buffers.
BytesPerSync int
// Fields here are set only if the provider is to support shared objects
// Local contains fields that are only relevant for files stored on the local
// filesystem.
Local struct {
// TODO(radu): move FSCleaner, NoSyncOnClose, BytesPerSync here.
// ReadaheadConfigFn is a function used to retrieve the current readahead
// mode. This function is run whenever a local object is open for reading.
// If it is nil, DefaultReadaheadConfig is used.
ReadaheadConfigFn func() ReadaheadConfig
}
// Fields here are set only if the provider is to support remote objects
// (experimental).
Shared struct {
Storage shared.Storage
Remote struct {
StorageFactory remote.StorageFactory
// If CreateOnShared is non-zero, sstables are created on remote storage using
// the CreateOnSharedLocator (when the PreferSharedStorage create option is
// true).
CreateOnShared remote.CreateOnSharedStrategy
CreateOnSharedLocator remote.Locator
// CacheSizeBytes is the size of the on-disk block cache for objects
// on shared storage. If it is 0, no cache is used.
// on remote storage. If it is 0, no cache is used.
CacheSizeBytes int64
// CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used.
CacheBlockSize int
// ShardingBlockSize is the size of a shard block. The cache is split into contiguous
// ShardingBlockSize units. The units are distributed across multiple independent shards
// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
// policies operate at the level of shard, not whole cache. This is done to reduce lock
// contention.
//
// If ShardingBlockSize is 0, the default of 1 MB is used.
ShardingBlockSize int64
// The number of independent shards the cache leverages. Each shard is the same size,
// and a hash of filenum & offset map a read to a certain shard. If set to 0,
// 2*runtime.GOMAXPROCS is used as the shard count.
@@ -111,7 +140,44 @@ type Settings struct {
}
}
// DefaultSettings initializes default settings (with no shared storage),
// ReadaheadConfig controls the use of read-ahead.
type ReadaheadConfig struct {
// Informed is the type of read-ahead for operations that are known to read a
// large consecutive chunk of a file.
Informed ReadaheadMode
// Speculative is the type of read-ahead used automatically, when consecutive
// reads are detected.
Speculative ReadaheadMode
}
// DefaultReadaheadConfig is the readahead config used when ReadaheadConfigFn is
// not specified.
var DefaultReadaheadConfig = ReadaheadConfig{
Informed: FadviseSequential,
Speculative: FadviseSequential,
}
// ReadaheadMode indicates the type of read-ahead to use, either for informed
// read-ahead (e.g. compactions) or speculative read-ahead.
type ReadaheadMode uint8
const (
// NoReadahead disables readahead altogether.
NoReadahead ReadaheadMode = iota
// SysReadahead enables the use of SYS_READAHEAD call to prefetch data.
// The prefetch window grows dynamically as consecutive writes are detected.
SysReadahead
// FadviseSequential enables to use of FADV_SEQUENTIAL. For informed
// read-ahead, FADV_SEQUENTIAL is used from the beginning. For speculative
// read-ahead SYS_READAHEAD is first used until the window reaches the maximum
// size, then we siwtch to FADV_SEQUENTIAL.
FadviseSequential
)
// DefaultSettings initializes default settings (with no remote storage),
// suitable for tests and tools.
func DefaultSettings(fs vfs.FS, dirName string) Settings {
return Settings{
@@ -126,7 +192,13 @@ func DefaultSettings(fs vfs.FS, dirName string) Settings {
// Open creates the provider.
func Open(settings Settings) (objstorage.Provider, error) {
return open(settings)
// Note: we can't just `return open(settings)` because in an error case we
// would return (*provider)(nil) which is not objstorage.Provider(nil).
p, err := open(settings)
if err != nil {
return nil, err
}
return p, nil
}
func open(settings Settings) (p *provider, _ error) {
@@ -157,8 +229,8 @@ func open(settings Settings) (p *provider, _ error) {
return nil, err
}
// Initialize shared subsystem (if configured) and add shared objects.
if err := p.sharedInit(); err != nil {
// Initialize remote subsystem (if configured) and add remote objects.
if err := p.remoteInit(); err != nil {
return nil, err
}
@@ -167,9 +239,9 @@ func open(settings Settings) (p *provider, _ error) {
// Close is part of the objstorage.Provider interface.
func (p *provider) Close() error {
var err error
err := p.sharedClose()
if p.fsDir != nil {
err = p.fsDir.Close()
err = firstError(err, p.fsDir.Close())
p.fsDir = nil
}
if objiotracing.Enabled {
@@ -197,10 +269,14 @@ func (p *provider) OpenForReading(
}
var r objstorage.Readable
if !meta.IsShared() {
if !meta.IsRemote() {
r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts)
} else {
r, err = p.sharedOpenForReading(ctx, meta, opts)
r, err = p.remoteOpenForReading(ctx, meta, opts)
if err != nil && p.isNotExistError(meta, err) {
// Wrap the error so that IsNotExistError functions properly.
err = errors.Mark(err, os.ErrNotExist)
}
}
if err != nil {
return nil, err
@@ -221,8 +297,8 @@ func (p *provider) Create(
fileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
if opts.PreferSharedStorage && p.st.Shared.Storage != nil {
w, meta, err = p.sharedCreate(ctx, fileType, fileNum, opts)
if opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone {
w, meta, err = p.sharedCreate(ctx, fileType, fileNum, p.st.Remote.CreateOnSharedLocator, opts)
} else {
w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
}
@@ -239,8 +315,8 @@ func (p *provider) Create(
// Remove removes an object.
//
// Note that if the object is shared, the object is only (conceptually) removed
// from this provider. If other providers have references on the shared object,
// Note that if the object is remote, the object is only (conceptually) removed
// from this provider. If other providers have references on the remote object,
// it will not be removed.
//
// The object is not guaranteed to be durably removed until Sync is called.
@@ -250,11 +326,15 @@ func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) erro
return err
}
if !meta.IsShared() {
if !meta.IsRemote() {
err = p.vfsRemove(fileType, fileNum)
} else {
// TODO(radu): implement shared object removal (i.e. deref).
// TODO(radu): implement remote object removal (i.e. deref).
err = p.sharedUnref(meta)
if err != nil && p.isNotExistError(meta, err) {
// Wrap the error so that IsNotExistError functions properly.
err = errors.Mark(err, os.ErrNotExist)
}
}
if err != nil && !p.IsNotExistError(err) {
// We want to be able to retry a Remove, so we keep the object in our list.
@@ -267,15 +347,18 @@ func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) erro
return err
}
func (p *provider) isNotExistError(meta objstorage.ObjectMetadata, err error) bool {
if meta.Remote.Storage != nil {
return meta.Remote.Storage.IsNotExistError(err)
}
return oserror.IsNotExist(err)
}
// IsNotExistError is part of the objstorage.Provider interface.
func (p *provider) IsNotExistError(err error) bool {
if oserror.IsNotExist(err) {
return true
}
if p.sharedStorage() != nil && p.sharedStorage().IsNotExistError(err) {
return true
}
return false
// We use errors.Mark(err, os.ErrNotExist) for not-exist errors coming from
// remote.Storage.
return oserror.IsNotExist(err)
}
// Sync flushes the metadata from creation or removal of objects since the last Sync.
@@ -303,7 +386,7 @@ func (p *provider) LinkOrCopyFromLocal(
dstFileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (objstorage.ObjectMetadata, error) {
shared := opts.PreferSharedStorage && p.st.Shared.Storage != nil
shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone
if !shared && srcFS == p.st.FS {
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
@@ -383,18 +466,18 @@ func (p *provider) Lookup(
// Path is part of the objstorage.Provider interface.
func (p *provider) Path(meta objstorage.ObjectMetadata) string {
if !meta.IsShared() {
if !meta.IsRemote() {
return p.vfsPath(meta.FileType, meta.DiskFileNum)
}
return p.sharedPath(meta)
return p.remotePath(meta)
}
// Size returns the size of the object.
func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
if !meta.IsShared() {
if !meta.IsRemote() {
return p.vfsSize(meta.FileType, meta.DiskFileNum)
}
return p.sharedSize(meta)
return p.remoteSize(meta)
}
// List is part of the objstorage.Provider interface.
@@ -411,17 +494,29 @@ func (p *provider) List() []objstorage.ObjectMetadata {
return res
}
// Metrics is part of the objstorage.Provider interface.
func (p *provider) Metrics() sharedcache.Metrics {
if p.remote.cache != nil {
return p.remote.cache.Metrics()
}
return sharedcache.Metrics{}
}
func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
if invariants.Enabled {
meta.AssertValid()
}
p.mu.Lock()
defer p.mu.Unlock()
p.mu.knownObjects[meta.DiskFileNum] = meta
if meta.IsShared() {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
if meta.IsRemote() {
p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: meta.DiskFileNum,
FileType: meta.FileType,
CreatorID: meta.Shared.CreatorID,
CreatorFileNum: meta.Shared.CreatorFileNum,
CleanupMethod: meta.Shared.CleanupMethod,
CreatorID: meta.Remote.CreatorID,
CreatorFileNum: meta.Remote.CreatorFileNum,
Locator: meta.Remote.Locator,
CleanupMethod: meta.Remote.CleanupMethod,
})
} else {
p.mu.localObjectsChanged = true
@@ -437,14 +532,14 @@ func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
return
}
delete(p.mu.knownObjects, fileNum)
if meta.IsShared() {
p.mu.shared.catalogBatch.DeleteObject(fileNum)
if meta.IsRemote() {
p.mu.remote.catalogBatch.DeleteObject(fileNum)
} else {
p.mu.localObjectsChanged = true
}
}
// protectObject prevents the unreferencing of a shared object until
// protectObject prevents the unreferencing of a remote object until
// unprotectObject is called.
func (p *provider) protectObject(fileNum base.DiskFileNum) {
p.mu.Lock()

5
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go generated vendored
View File

@@ -4,6 +4,8 @@
package objstorageprovider
import "github.com/cockroachdb/pebble/internal/invariants"
const (
// Constants for dynamic readahead of data blocks. Note that the size values
// make sense as some multiple of the default block size; and they should
@@ -81,6 +83,9 @@ func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
// Returns a size value (greater than 0) that should be prefetched if readahead
// would be beneficial.
func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
if invariants.Enabled && rs.maxReadaheadSize == 0 {
panic("readaheadState not initialized")
}
currentReadEnd := offset + blockLength
if rs.numReads >= minFileReadsForReadahead {
// The minimum threshold of sequential reads to justify reading ahead

376
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go generated vendored Normal file
View File

@@ -0,0 +1,376 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"fmt"
"runtime"
"sync"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/remote"
)
// remoteSubsystem contains the provider fields related to remote storage.
// All fields remain unset if remote storage is not configured.
type remoteSubsystem struct {
catalog *remoteobjcat.Catalog
// catalogSyncMutex is used to correctly serialize two sharedSync operations.
// It must be acquired before the provider mutex.
catalogSyncMutex sync.Mutex
cache *sharedcache.Cache
// shared contains the fields relevant to shared objects, i.e. objects that
// are created by Pebble and potentially shared between Pebble instances.
shared struct {
// initialized guards access to the creatorID field.
initialized atomic.Bool
creatorID objstorage.CreatorID
initOnce sync.Once
// checkRefsOnOpen controls whether we check the ref marker file when opening
// an object. Normally this is true when invariants are enabled (but the provider
// test tweaks this field).
checkRefsOnOpen bool
}
}
// remoteInit initializes the remote object subsystem (if configured) and finds
// any remote objects.
func (p *provider) remoteInit() error {
if p.st.Remote.StorageFactory == nil {
return nil
}
catalog, contents, err := remoteobjcat.Open(p.st.FS, p.st.FSDirName)
if err != nil {
return errors.Wrapf(err, "pebble: could not open remote object catalog")
}
p.remote.catalog = catalog
p.remote.shared.checkRefsOnOpen = invariants.Enabled
// The creator ID may or may not be initialized yet.
if contents.CreatorID.IsSet() {
p.remote.initShared(contents.CreatorID)
p.st.Logger.Infof("remote storage configured; creatorID = %s", contents.CreatorID)
} else {
p.st.Logger.Infof("remote storage configured; no creatorID yet")
}
if p.st.Remote.CacheSizeBytes > 0 {
const defaultBlockSize = 32 * 1024
blockSize := p.st.Remote.CacheBlockSize
if blockSize == 0 {
blockSize = defaultBlockSize
}
const defaultShardingBlockSize = 1024 * 1024
shardingBlockSize := p.st.Remote.ShardingBlockSize
if shardingBlockSize == 0 {
shardingBlockSize = defaultShardingBlockSize
}
numShards := p.st.Remote.CacheShardCount
if numShards == 0 {
numShards = 2 * runtime.GOMAXPROCS(0)
}
p.remote.cache, err = sharedcache.Open(
p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, shardingBlockSize, p.st.Remote.CacheSizeBytes, numShards)
if err != nil {
return errors.Wrapf(err, "pebble: could not open remote object cache")
}
}
for _, meta := range contents.Objects {
o := objstorage.ObjectMetadata{
DiskFileNum: meta.FileNum,
FileType: meta.FileType,
}
o.Remote.CreatorID = meta.CreatorID
o.Remote.CreatorFileNum = meta.CreatorFileNum
o.Remote.CleanupMethod = meta.CleanupMethod
o.Remote.Locator = meta.Locator
o.Remote.CustomObjectName = meta.CustomObjectName
o.Remote.Storage, err = p.ensureStorageLocked(o.Remote.Locator)
if err != nil {
return errors.Wrapf(err, "creating remote.Storage object for locator '%s'", o.Remote.Locator)
}
if invariants.Enabled {
o.AssertValid()
}
p.mu.knownObjects[o.DiskFileNum] = o
}
return nil
}
// initShared initializes the creator ID, allowing use of shared objects.
func (ss *remoteSubsystem) initShared(creatorID objstorage.CreatorID) {
ss.shared.initOnce.Do(func() {
ss.shared.creatorID = creatorID
ss.shared.initialized.Store(true)
})
}
func (p *provider) sharedClose() error {
if p.st.Remote.StorageFactory == nil {
return nil
}
var err error
if p.remote.cache != nil {
err = p.remote.cache.Close()
p.remote.cache = nil
}
if p.remote.catalog != nil {
err = firstError(err, p.remote.catalog.Close())
p.remote.catalog = nil
}
return err
}
// SetCreatorID is part of the objstorage.Provider interface.
func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
if p.st.Remote.StorageFactory == nil {
return errors.AssertionFailedf("attempt to set CreatorID but remote storage not enabled")
}
// Note: this call is a cheap no-op if the creator ID was already set. This
// call also checks if we are trying to change the ID.
if err := p.remote.catalog.SetCreatorID(creatorID); err != nil {
return err
}
if !p.remote.shared.initialized.Load() {
p.st.Logger.Infof("remote storage creatorID set to %s", creatorID)
p.remote.initShared(creatorID)
}
return nil
}
// IsSharedForeign is part of the objstorage.Provider interface.
func (p *provider) IsSharedForeign(meta objstorage.ObjectMetadata) bool {
if !p.remote.shared.initialized.Load() {
return false
}
return meta.IsShared() && (meta.Remote.CreatorID != p.remote.shared.creatorID)
}
func (p *provider) remoteCheckInitialized() error {
if p.st.Remote.StorageFactory == nil {
return errors.Errorf("remote object support not configured")
}
return nil
}
func (p *provider) sharedCheckInitialized() error {
if err := p.remoteCheckInitialized(); err != nil {
return err
}
if !p.remote.shared.initialized.Load() {
return errors.Errorf("remote object support not available: remote creator ID not yet set")
}
return nil
}
func (p *provider) sharedSync() error {
// Serialize parallel sync operations. Note that ApplyBatch is already
// serialized internally, but we want to make sure they get called with
// batches in the right order.
p.remote.catalogSyncMutex.Lock()
defer p.remote.catalogSyncMutex.Unlock()
batch := func() remoteobjcat.Batch {
p.mu.Lock()
defer p.mu.Unlock()
res := p.mu.remote.catalogBatch.Copy()
p.mu.remote.catalogBatch.Reset()
return res
}()
if batch.IsEmpty() {
return nil
}
if err := p.remote.catalog.ApplyBatch(batch); err != nil {
// Put back the batch (for the next Sync), appending any operations that
// happened in the meantime.
p.mu.Lock()
defer p.mu.Unlock()
batch.Append(p.mu.remote.catalogBatch)
p.mu.remote.catalogBatch = batch
return err
}
return nil
}
func (p *provider) remotePath(meta objstorage.ObjectMetadata) string {
if meta.Remote.Locator != "" {
return fmt.Sprintf("remote-%s://%s", meta.Remote.Locator, remoteObjectName(meta))
}
return "remote://" + remoteObjectName(meta)
}
// sharedCreateRef creates a reference marker object.
func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
if err := p.sharedCheckInitialized(); err != nil {
return err
}
if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
return nil
}
refName := p.sharedObjectRefName(meta)
writer, err := meta.Remote.Storage.CreateObject(refName)
if err == nil {
// The object is empty, just close the writer.
err = writer.Close()
}
if err != nil {
return errors.Wrapf(err, "creating marker object %q", refName)
}
return nil
}
func (p *provider) sharedCreate(
_ context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
locator remote.Locator,
opts objstorage.CreateOptions,
) (objstorage.Writable, objstorage.ObjectMetadata, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, objstorage.ObjectMetadata{}, err
}
storage, err := p.ensureStorage(locator)
if err != nil {
return nil, objstorage.ObjectMetadata{}, err
}
meta := objstorage.ObjectMetadata{
DiskFileNum: fileNum,
FileType: fileType,
}
meta.Remote.CreatorID = p.remote.shared.creatorID
meta.Remote.CreatorFileNum = fileNum
meta.Remote.CleanupMethod = opts.SharedCleanupMethod
meta.Remote.Locator = locator
meta.Remote.Storage = storage
objName := remoteObjectName(meta)
writer, err := storage.CreateObject(objName)
if err != nil {
return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
}
return &sharedWritable{
p: p,
meta: meta,
storageWriter: writer,
}, meta, nil
}
func (p *provider) remoteOpenForReading(
ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
if err := p.remoteCheckInitialized(); err != nil {
return nil, err
}
// Verify we have a reference on this object; for performance reasons, we only
// do this in testing scenarios.
if p.remote.shared.checkRefsOnOpen && meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
if err := p.sharedCheckInitialized(); err != nil {
return nil, err
}
refName := p.sharedObjectRefName(meta)
if _, err := meta.Remote.Storage.Size(refName); err != nil {
if meta.Remote.Storage.IsNotExistError(err) {
if opts.MustExist {
p.st.Logger.Fatalf("marker object %q does not exist", refName)
// TODO(radu): maybe list references for the object.
}
return nil, errors.Errorf("marker object %q does not exist", refName)
}
return nil, errors.Wrapf(err, "checking marker object %q", refName)
}
}
objName := remoteObjectName(meta)
reader, size, err := meta.Remote.Storage.ReadObject(ctx, objName)
if err != nil {
if opts.MustExist && meta.Remote.Storage.IsNotExistError(err) {
p.st.Logger.Fatalf("object %q does not exist", objName)
// TODO(radu): maybe list references for the object.
}
return nil, err
}
return p.newRemoteReadable(reader, size, meta.DiskFileNum), nil
}
func (p *provider) remoteSize(meta objstorage.ObjectMetadata) (int64, error) {
if err := p.remoteCheckInitialized(); err != nil {
return 0, err
}
objName := remoteObjectName(meta)
return meta.Remote.Storage.Size(objName)
}
// sharedUnref implements object "removal" with the remote backend. The ref
// marker object is removed and the backing object is removed only if there are
// no other ref markers.
func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup {
// Never delete objects in this mode.
return nil
}
if p.isProtected(meta.DiskFileNum) {
// TODO(radu): we need a mechanism to unref the object when it becomes
// unprotected.
return nil
}
refName := p.sharedObjectRefName(meta)
// Tolerate a not-exists error.
if err := meta.Remote.Storage.Delete(refName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
return err
}
otherRefs, err := meta.Remote.Storage.List(sharedObjectRefPrefix(meta), "" /* delimiter */)
if err != nil {
return err
}
if len(otherRefs) == 0 {
objName := remoteObjectName(meta)
if err := meta.Remote.Storage.Delete(objName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
return err
}
}
return nil
}
// ensureStorageLocked populates the remote.Storage object for the given
// locator, if necessary. p.mu must be held.
func (p *provider) ensureStorageLocked(locator remote.Locator) (remote.Storage, error) {
if p.mu.remote.storageObjects == nil {
p.mu.remote.storageObjects = make(map[remote.Locator]remote.Storage)
}
if res, ok := p.mu.remote.storageObjects[locator]; ok {
return res, nil
}
res, err := p.st.Remote.StorageFactory.CreateStorage(locator)
if err != nil {
return nil, err
}
p.mu.remote.storageObjects[locator] = res
return res, nil
}
// ensureStorage populates the remote.Storage object for the given locator, if necessary.
func (p *provider) ensureStorage(locator remote.Locator) (remote.Storage, error) {
p.mu.Lock()
defer p.mu.Unlock()
return p.ensureStorageLocked(locator)
}

159
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_backing.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go generated vendored
View File

@@ -12,7 +12,8 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/remote"
)
const (
@@ -25,6 +26,12 @@ const (
// allows the "target" provider to check that the "source" provider kept its
// reference on the object alive.
tagRefCheckID = 4
// tagLocator encodes the remote.Locator; if absent the locator is "". It is
// followed by the locator string length and the locator string.
tagLocator = 5
// tagLocator encodes a custom object name (if present). It is followed by the
// custom name string length and the string.
tagCustomObjectName = 6
// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
// followed by the length of the data (so they can be skipped).
@@ -34,83 +41,103 @@ const (
tagNotSafeToIgnoreMask = 64
)
func (p *provider) encodeSharedObjectBacking(
func (p *provider) encodeRemoteObjectBacking(
meta *objstorage.ObjectMetadata,
) (objstorage.SharedObjectBacking, error) {
if !meta.IsShared() {
return nil, errors.AssertionFailedf("object %s not on shared storage", meta.DiskFileNum)
) (objstorage.RemoteObjectBacking, error) {
if !meta.IsRemote() {
return nil, errors.AssertionFailedf("object %s not on remote storage", meta.DiskFileNum)
}
buf := make([]byte, 0, binary.MaxVarintLen64*4)
buf = binary.AppendUvarint(buf, tagCreatorID)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID))
buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorID))
// TODO(radu): encode file type as well?
buf = binary.AppendUvarint(buf, tagCreatorFileNum)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, tagCleanupMethod)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CleanupMethod))
if meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
buf = binary.AppendUvarint(buf, uint64(meta.Remote.CleanupMethod))
if meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
buf = binary.AppendUvarint(buf, tagRefCheckID)
buf = binary.AppendUvarint(buf, uint64(p.shared.creatorID))
buf = binary.AppendUvarint(buf, uint64(p.remote.shared.creatorID))
buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum()))
}
if meta.Remote.Locator != "" {
buf = binary.AppendUvarint(buf, tagLocator)
buf = encodeString(buf, string(meta.Remote.Locator))
}
if meta.Remote.CustomObjectName != "" {
buf = binary.AppendUvarint(buf, tagCustomObjectName)
buf = encodeString(buf, meta.Remote.CustomObjectName)
}
return buf, nil
}
type sharedObjectBackingHandle struct {
backing objstorage.SharedObjectBacking
type remoteObjectBackingHandle struct {
backing objstorage.RemoteObjectBacking
fileNum base.DiskFileNum
p *provider
}
func (s *sharedObjectBackingHandle) Get() (objstorage.SharedObjectBacking, error) {
func (s *remoteObjectBackingHandle) Get() (objstorage.RemoteObjectBacking, error) {
if s.backing == nil {
return nil, errors.Errorf("SharedObjectBackingHandle.Get() called after Close()")
return nil, errors.Errorf("RemoteObjectBackingHandle.Get() called after Close()")
}
return s.backing, nil
}
func (s *sharedObjectBackingHandle) Close() {
func (s *remoteObjectBackingHandle) Close() {
if s.backing != nil {
s.backing = nil
s.p.unprotectObject(s.fileNum)
}
}
var _ objstorage.SharedObjectBackingHandle = (*sharedObjectBackingHandle)(nil)
var _ objstorage.RemoteObjectBackingHandle = (*remoteObjectBackingHandle)(nil)
// SharedObjectBacking is part of the objstorage.Provider interface.
func (p *provider) SharedObjectBacking(
// RemoteObjectBacking is part of the objstorage.Provider interface.
func (p *provider) RemoteObjectBacking(
meta *objstorage.ObjectMetadata,
) (objstorage.SharedObjectBackingHandle, error) {
backing, err := p.encodeSharedObjectBacking(meta)
) (objstorage.RemoteObjectBackingHandle, error) {
backing, err := p.encodeRemoteObjectBacking(meta)
if err != nil {
return nil, err
}
p.protectObject(meta.DiskFileNum)
return &sharedObjectBackingHandle{
return &remoteObjectBackingHandle{
backing: backing,
fileNum: meta.DiskFileNum,
p: p,
}, nil
}
// CreateExternalObjectBacking is part of the objstorage.Provider interface.
func (p *provider) CreateExternalObjectBacking(
locator remote.Locator, objName string,
) (objstorage.RemoteObjectBacking, error) {
var meta objstorage.ObjectMetadata
meta.Remote.Locator = locator
meta.Remote.CustomObjectName = objName
meta.Remote.CleanupMethod = objstorage.SharedNoCleanup
return p.encodeRemoteObjectBacking(&meta)
}
type decodedBacking struct {
meta objstorage.ObjectMetadata
// refToCheck is set only when meta.Shared.CleanupMethod is RefTracking
// refToCheck is set only when meta.Remote.CleanupMethod is RefTracking
refToCheck struct {
creatorID objstorage.CreatorID
fileNum base.DiskFileNum
}
}
// decodeSharedObjectBacking decodes the shared object metadata.
// Returns the object metadata and (optionally) the creator ID of the provider
// that encoded the backing whose ref marker needs to be checked.
func decodeSharedObjectBacking(
fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.SharedObjectBacking,
// decodeRemoteObjectBacking decodes the remote object metadata.
//
// Note that the meta.Remote.Storage field is not set.
func decodeRemoteObjectBacking(
fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.RemoteObjectBacking,
) (decodedBacking, error) {
var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64
var locator, customObjName string
br := bytes.NewReader(buf)
for {
tag, err := binary.ReadUvarint(br)
@@ -136,6 +163,12 @@ func decodeSharedObjectBacking(
refCheckFileNum, err = binary.ReadUvarint(br)
}
case tagLocator:
locator, err = decodeString(br)
case tagCustomObjectName:
customObjName, err = decodeString(br)
default:
// Ignore unknown tags, unless they're not safe to ignore.
if tag&tagNotSafeToIgnoreMask != 0 {
@@ -151,37 +184,65 @@ func decodeSharedObjectBacking(
return decodedBacking{}, err
}
}
if creatorID == 0 {
return decodedBacking{}, errors.Newf("shared object backing missing creator ID")
}
if creatorFileNum == 0 {
return decodedBacking{}, errors.Newf("shared object backing missing creator file num")
if customObjName == "" {
if creatorID == 0 {
return decodedBacking{}, errors.Newf("remote object backing missing creator ID")
}
if creatorFileNum == 0 {
return decodedBacking{}, errors.Newf("remote object backing missing creator file num")
}
}
var res decodedBacking
res.meta.DiskFileNum = fileNum
res.meta.FileType = fileType
res.meta.Shared.CreatorID = objstorage.CreatorID(creatorID)
res.meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
res.meta.Shared.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
if res.meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
res.meta.Remote.CreatorID = objstorage.CreatorID(creatorID)
res.meta.Remote.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
res.meta.Remote.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
if res.meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
if refCheckCreatorID == 0 || refCheckFileNum == 0 {
return decodedBacking{}, errors.Newf("shared object backing missing ref to check")
return decodedBacking{}, errors.Newf("remote object backing missing ref to check")
}
res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID)
res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum()
}
res.meta.Remote.Locator = remote.Locator(locator)
res.meta.Remote.CustomObjectName = customObjName
return res, nil
}
// AttachSharedObjects is part of the objstorage.Provider interface.
func (p *provider) AttachSharedObjects(
objs []objstorage.SharedObjectToAttach,
func encodeString(buf []byte, s string) []byte {
buf = binary.AppendUvarint(buf, uint64(len(s)))
buf = append(buf, []byte(s)...)
return buf
}
func decodeString(br io.ByteReader) (string, error) {
length, err := binary.ReadUvarint(br)
if err != nil || length == 0 {
return "", err
}
buf := make([]byte, length)
for i := range buf {
buf[i], err = br.ReadByte()
if err != nil {
return "", err
}
}
return string(buf), nil
}
// AttachRemoteObjects is part of the objstorage.Provider interface.
func (p *provider) AttachRemoteObjects(
objs []objstorage.RemoteObjectToAttach,
) ([]objstorage.ObjectMetadata, error) {
decoded := make([]decodedBacking, len(objs))
for i, o := range objs {
var err error
decoded[i], err = decodeSharedObjectBacking(o.FileType, o.FileNum, o.Backing)
decoded[i], err = decodeRemoteObjectBacking(o.FileType, o.FileNum, o.Backing)
if err != nil {
return nil, err
}
decoded[i].meta.Remote.Storage, err = p.ensureStorage(decoded[i].meta.Remote.Locator)
if err != nil {
return nil, err
}
@@ -190,7 +251,7 @@ func (p *provider) AttachSharedObjects(
// Create the reference marker objects.
// TODO(radu): parallelize this.
for _, d := range decoded {
if d.meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
if d.meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
continue
}
if err := p.sharedCreateRef(d.meta); err != nil {
@@ -199,10 +260,10 @@ func (p *provider) AttachSharedObjects(
}
// Check the "origin"'s reference.
refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum)
if _, err := p.sharedStorage().Size(refName); err != nil {
if _, err := d.meta.Remote.Storage.Size(refName); err != nil {
_ = p.sharedUnref(d.meta)
// TODO(radu): clean up references previously created in this loop.
if p.sharedStorage().IsNotExistError(err) {
if d.meta.Remote.Storage.IsNotExistError(err) {
return nil, errors.Errorf("origin marker object %q does not exist;"+
" object probably removed from the provider which created the backing", refName)
}
@@ -214,11 +275,13 @@ func (p *provider) AttachSharedObjects(
p.mu.Lock()
defer p.mu.Unlock()
for _, d := range decoded {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: d.meta.DiskFileNum,
FileType: d.meta.FileType,
CreatorID: d.meta.Shared.CreatorID,
CreatorFileNum: d.meta.Shared.CreatorFileNum,
CreatorID: d.meta.Remote.CreatorID,
CreatorFileNum: d.meta.Remote.CreatorFileNum,
CleanupMethod: d.meta.Remote.CleanupMethod,
Locator: d.meta.Remote.Locator,
})
}
}()

29
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_obj_name.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go generated vendored
View File

@@ -11,16 +11,19 @@ import (
"github.com/cockroachdb/pebble/objstorage"
)
// sharedObjectName returns the name of an object on shared storage.
// remoteObjectName returns the name of an object on remote storage.
//
// For sstables, the format is: <hash>-<creator-id>-<file-num>.sst
// For example: 1a3f-2-000001.sst
func sharedObjectName(meta objstorage.ObjectMetadata) string {
func remoteObjectName(meta objstorage.ObjectMetadata) string {
if meta.Remote.CustomObjectName != "" {
return meta.Remote.CustomObjectName
}
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst",
objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
)
}
panic("unknown FileType")
@@ -34,25 +37,33 @@ func sharedObjectName(meta objstorage.ObjectMetadata) string {
func sharedObjectRefName(
meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum,
) string {
if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
panic("ref object used when ref tracking disabled")
}
if meta.Remote.CustomObjectName != "" {
return fmt.Sprintf(
"%s.ref.%d.%06d", meta.Remote.CustomObjectName, refCreatorID, refFileNum.FileNum(),
)
}
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst.ref.%d.%06d",
objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
)
}
panic("unknown FileType")
}
func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
if meta.Remote.CustomObjectName != "" {
return meta.Remote.CustomObjectName + ".ref."
}
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst.ref.",
objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
)
}
panic("unknown FileType")
@@ -64,10 +75,10 @@ func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
//
// For example: 1a3f-2-000001.sst.ref.5.000008
func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
panic("ref object used when ref tracking disabled")
}
return sharedObjectRefName(meta, p.shared.creatorID, meta.DiskFileNum)
return sharedObjectRefName(meta, p.remote.shared.creatorID, meta.DiskFileNum)
}
// objHash returns a 16-bit hash value derived from the creator ID and creator
@@ -76,5 +87,5 @@ func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
func objHash(meta objstorage.ObjectMetadata) uint16 {
const prime1 = 7459
const prime2 = 17539
return uint16(uint64(meta.Shared.CreatorID)*prime1 + uint64(meta.Shared.CreatorFileNum.FileNum())*prime2)
return uint16(uint64(meta.Remote.CreatorID)*prime1 + uint64(meta.Remote.CreatorFileNum.FileNum())*prime2)
}

58
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go generated vendored
View File

@@ -11,26 +11,26 @@ import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/objstorage/remote"
)
const sharedMaxReadaheadSize = 1024 * 1024 /* 1MB */
const remoteMaxReadaheadSize = 1024 * 1024 /* 1MB */
// sharedReadable is a very simple implementation of Readable on top of the
// ReadCloser returned by shared.Storage.CreateObject.
type sharedReadable struct {
objReader shared.ObjectReader
// remoteReadable is a very simple implementation of Readable on top of the
// ReadCloser returned by remote.Storage.CreateObject.
type remoteReadable struct {
objReader remote.ObjectReader
size int64
fileNum base.DiskFileNum
provider *provider
}
var _ objstorage.Readable = (*sharedReadable)(nil)
var _ objstorage.Readable = (*remoteReadable)(nil)
func (p *provider) newSharedReadable(
objReader shared.ObjectReader, size int64, fileNum base.DiskFileNum,
) *sharedReadable {
return &sharedReadable{
func (p *provider) newRemoteReadable(
objReader remote.ObjectReader, size int64, fileNum base.DiskFileNum,
) *remoteReadable {
return &remoteReadable{
objReader: objReader,
size: size,
fileNum: fileNum,
@@ -39,43 +39,43 @@ func (p *provider) newSharedReadable(
}
// ReadAt is part of the objstorage.Readable interface.
func (r *sharedReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
func (r *remoteReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
return r.readInternal(ctx, p, offset, false /* forCompaction */)
}
// readInternal performs a read for the object, using the cache when
// appropriate.
func (r *sharedReadable) readInternal(
func (r *remoteReadable) readInternal(
ctx context.Context, p []byte, offset int64, forCompaction bool,
) error {
if cache := r.provider.shared.cache; cache != nil {
if cache := r.provider.remote.cache; cache != nil {
flags := sharedcache.ReadFlags{
// Don't add data to the cache if this read is for a compaction.
ReadOnly: forCompaction,
}
return r.provider.shared.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
return r.provider.remote.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
}
return r.objReader.ReadAt(ctx, p, offset)
}
func (r *sharedReadable) Close() error {
func (r *remoteReadable) Close() error {
defer func() { r.objReader = nil }()
return r.objReader.Close()
}
func (r *sharedReadable) Size() int64 {
func (r *remoteReadable) Size() int64 {
return r.size
}
func (r *sharedReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
func (r *remoteReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
// TODO(radu): use a pool.
rh := &sharedReadHandle{readable: r}
rh.readahead.state = makeReadaheadState(sharedMaxReadaheadSize)
rh := &remoteReadHandle{readable: r}
rh.readahead.state = makeReadaheadState(remoteMaxReadaheadSize)
return rh
}
type sharedReadHandle struct {
readable *sharedReadable
type remoteReadHandle struct {
readable *remoteReadable
readahead struct {
state readaheadState
data []byte
@@ -84,10 +84,10 @@ type sharedReadHandle struct {
forCompaction bool
}
var _ objstorage.ReadHandle = (*sharedReadHandle)(nil)
var _ objstorage.ReadHandle = (*remoteReadHandle)(nil)
// ReadAt is part of the objstorage.ReadHandle interface.
func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
readaheadSize := r.maybeReadahead(offset, len(p))
// Check if we already have the data from a previous read-ahead.
@@ -135,27 +135,27 @@ func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) e
return r.readable.readInternal(ctx, p, offset, r.forCompaction)
}
func (r *sharedReadHandle) maybeReadahead(offset int64, len int) int {
func (r *remoteReadHandle) maybeReadahead(offset int64, len int) int {
if r.forCompaction {
return sharedMaxReadaheadSize
return remoteMaxReadaheadSize
}
return int(r.readahead.state.maybeReadahead(offset, int64(len)))
}
// Close is part of the objstorage.ReadHandle interface.
func (r *sharedReadHandle) Close() error {
func (r *remoteReadHandle) Close() error {
r.readable = nil
r.readahead.data = nil
return nil
}
// SetupForCompaction is part of the objstorage.ReadHandle interface.
func (r *sharedReadHandle) SetupForCompaction() {
func (r *remoteReadHandle) SetupForCompaction() {
r.forCompaction = true
}
// RecordCacheHit is part of the objstorage.ReadHandle interface.
func (r *sharedReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
func (r *remoteReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
if !r.forCompaction {
r.readahead.state.recordCacheHit(offset, size)
}

107
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/catalog.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go generated vendored
View File

@@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sharedobjcat
package remoteobjcat
import (
"fmt"
@@ -13,15 +13,16 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
// Catalog is used to manage the on-disk shared object catalog.
// Catalog is used to manage the on-disk remote object catalog.
//
// The catalog file is a log of records, where each record is an encoded
// versionEdit.
// VersionEdit.
type Catalog struct {
fs vfs.FS
dirname string
@@ -29,7 +30,7 @@ type Catalog struct {
sync.Mutex
creatorID objstorage.CreatorID
objects map[base.DiskFileNum]SharedObjectMetadata
objects map[base.DiskFileNum]RemoteObjectMetadata
marker *atomicfs.Marker
@@ -44,8 +45,8 @@ type Catalog struct {
}
}
// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
type SharedObjectMetadata struct {
// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object.
type RemoteObjectMetadata struct {
// FileNum is the identifier for the object within the context of a single DB
// instance.
FileNum base.DiskFileNum
@@ -56,24 +57,29 @@ type SharedObjectMetadata struct {
// CreatorFileNum is the identifier for the object within the context of the
// DB instance that originally created the object.
CreatorFileNum base.DiskFileNum
// CleanupMethod indicates the method for cleaning up unused shared objects.
CleanupMethod objstorage.SharedCleanupMethod
// Locator identifies a remote.Storage implementation.
Locator remote.Locator
// CustomObjectName (if it is set) overrides the object name that is normally
// derived from the CreatorID and CreatorFileNum.
CustomObjectName string
}
const (
catalogFilenameBase = "SHARED-CATALOG"
catalogMarkerName = "shared-catalog"
catalogFilenameBase = "REMOTE-OBJ-CATALOG"
catalogMarkerName = "remote-obj-catalog"
// We create a new file when the size exceeds 1MB (and some other conditions
// hold; see record.RotationHelper).
rotateFileSize = 1024 * 1024 // 1MB
)
// CatalogContents contains the shared objects in the catalog.
// CatalogContents contains the remote objects in the catalog.
type CatalogContents struct {
// CreatorID, if it is set.
CreatorID objstorage.CreatorID
Objects []SharedObjectMetadata
Objects []RemoteObjectMetadata
}
// Open creates a Catalog and loads any existing catalog file, returning the
@@ -83,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
fs: fs,
dirname: dirname,
}
c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata)
c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata)
var err error
c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
@@ -102,7 +108,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
}
res := CatalogContents{
CreatorID: c.mu.creatorID,
Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)),
Objects: make([]RemoteObjectMetadata, 0, len(c.mu.objects)),
}
for _, meta := range c.mu.objects {
res.Objects = append(res.Objects, meta)
@@ -130,9 +136,9 @@ func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {
return nil
}
ve := versionEdit{CreatorID: id}
ve := VersionEdit{CreatorID: id}
if err := c.writeToCatalogFileLocked(&ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}
c.mu.creatorID = id
return nil
@@ -159,14 +165,14 @@ func (c *Catalog) closeCatalogFile() error {
// Batch is used to perform multiple object additions/deletions at once.
type Batch struct {
ve versionEdit
ve VersionEdit
}
// AddObject adds a new object to the batch.
//
// The given FileNum must be new - it must not match that of any object that was
// ever in the catalog.
func (b *Batch) AddObject(meta SharedObjectMetadata) {
func (b *Batch) AddObject(meta RemoteObjectMetadata) {
b.ve.NewObjects = append(b.ve.NewObjects, meta)
}
@@ -190,7 +196,7 @@ func (b *Batch) IsEmpty() bool {
func (b *Batch) Copy() Batch {
var res Batch
if len(b.ve.NewObjects) > 0 {
res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects))
copy(res.ve.NewObjects, b.ve.NewObjects)
}
if len(b.ve.DeletedObjects) > 0 {
@@ -212,29 +218,40 @@ func (c *Catalog) ApplyBatch(b Batch) error {
c.mu.Lock()
defer c.mu.Unlock()
for _, n := range b.ve.DeletedObjects {
if _, exists := c.mu.objects[n]; !exists {
return errors.AssertionFailedf("deleting non-existent object %s", n)
// Sanity checks.
toAdd := make(map[base.DiskFileNum]struct{}, len(b.ve.NewObjects))
exists := func(n base.DiskFileNum) bool {
_, ok := c.mu.objects[n]
if !ok {
_, ok = toAdd[n]
}
return ok
}
for _, meta := range b.ve.NewObjects {
if _, exists := c.mu.objects[meta.FileNum]; exists {
if exists(meta.FileNum) {
return errors.AssertionFailedf("adding existing object %s", meta.FileNum)
}
toAdd[meta.FileNum] = struct{}{}
}
for _, n := range b.ve.DeletedObjects {
if !exists(n) {
return errors.AssertionFailedf("deleting non-existent object %s", n)
}
}
if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}
// Apply the batch to our current state.
for _, n := range b.ve.DeletedObjects {
delete(c.mu.objects, n)
}
// Add new objects before deleting any objects. This allows for cases where
// the same batch adds and deletes an object.
for _, meta := range b.ve.NewObjects {
c.mu.objects[meta.FileNum] = meta
}
b.Reset()
for _, n := range b.ve.DeletedObjects {
delete(c.mu.objects, n)
}
return nil
}
@@ -243,7 +260,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
f, err := c.fs.Open(catalogPath)
if err != nil {
return errors.Wrapf(
err, "pebble: could not open shared object catalog file %q for DB %q",
err, "pebble: could not open remote object catalog file %q for DB %q",
errors.Safe(filename), c.dirname,
)
}
@@ -255,32 +272,26 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
break
}
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
var ve versionEdit
err = ve.Decode(r)
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
var ve VersionEdit
if err := ve.Decode(r); err != nil {
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
// Apply the version edit to the current state.
if ve.CreatorID.IsSet() {
c.mu.creatorID = ve.CreatorID
}
for _, fileNum := range ve.DeletedObjects {
delete(c.mu.objects, fileNum)
}
for _, meta := range ve.NewObjects {
c.mu.objects[meta.FileNum] = meta
if err := ve.Apply(&c.mu.creatorID, c.mu.objects); err != nil {
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
}
return nil
}
// writeToCatalogFileLocked writes a versionEdit to the catalog file.
// writeToCatalogFileLocked writes a VersionEdit to the catalog file.
// Creates a new file if this is the first write.
func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error {
func (c *Catalog) writeToCatalogFileLocked(ve *VersionEdit) error {
c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects)))
snapshotSize := int64(len(c.mu.objects))
@@ -323,10 +334,10 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
}
recWriter := record.NewWriter(file)
err = func() error {
// Create a versionEdit that gets us from an empty catalog to the current state.
var ve versionEdit
// Create a VersionEdit that gets us from an empty catalog to the current state.
var ve VersionEdit
ve.CreatorID = c.mu.creatorID
ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects))
for _, meta := range c.mu.objects {
ve.NewObjects = append(ve.NewObjects, meta)
}
@@ -361,7 +372,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
return nil
}
func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error {
func writeRecord(ve *VersionEdit, file vfs.File, recWriter *record.Writer) error {
w, err := recWriter.Next()
if err != nil {
return err

123
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/version_edit.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go generated vendored
View File

@@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sharedobjcat
package remoteobjcat
import (
"bufio"
@@ -11,28 +11,37 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/remote"
)
// versionEdit is a modification to the shared object state which can be encoded
// VersionEdit is a modification to the remote object state which can be encoded
// into a record.
//
// TODO(radu): consider adding creation and deletion time for debugging purposes.
type versionEdit struct {
NewObjects []SharedObjectMetadata
type VersionEdit struct {
NewObjects []RemoteObjectMetadata
DeletedObjects []base.DiskFileNum
CreatorID objstorage.CreatorID
}
const (
// tagNewObject is followed by the FileNum, creator ID, creator FileNum, and
// cleanup method.
// tagNewObject is followed by the FileNum, creator ID, creator FileNum,
// cleanup method, optional new object tags, and ending with a 0 byte.
tagNewObject = 1
// tagDeletedObject is followed by the FileNum.
tagDeletedObject = 2
// tagCreatorID is followed by the Creator ID for this store. This ID can
// never change.
tagCreatorID = 3
// tagNewObjectLocator is an optional tag inside the tagNewObject payload. It
// is followed by the encoded length of the locator string and the string.
tagNewObjectLocator = 4
// tagNewObjectCustomName is an optional tag inside the tagNewObject payload.
// It is followed by the encoded length of the custom object name string
// followed by the string.
tagNewObjectCustomName = 5
)
// Object type values. We don't want to encode FileType directly because it is
@@ -61,8 +70,8 @@ func fileTypeToObjType(fileType base.FileType) (uint64, error) {
}
// Encode encodes an edit to the specified writer.
func (v *versionEdit) Encode(w io.Writer) error {
buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2))
func (v *VersionEdit) Encode(w io.Writer) error {
buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*10+len(v.DeletedObjects)*2+2))
for _, meta := range v.NewObjects {
objType, err := fileTypeToObjType(meta.FileType)
if err != nil {
@@ -74,6 +83,16 @@ func (v *versionEdit) Encode(w io.Writer) error {
buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
if meta.Locator != "" {
buf = binary.AppendUvarint(buf, uint64(tagNewObjectLocator))
buf = encodeString(buf, string(meta.Locator))
}
if meta.CustomObjectName != "" {
buf = binary.AppendUvarint(buf, uint64(tagNewObjectCustomName))
buf = encodeString(buf, meta.CustomObjectName)
}
// Append 0 as the terminator for optional new object tags.
buf = binary.AppendUvarint(buf, 0)
}
for _, dfn := range v.DeletedObjects {
@@ -89,7 +108,7 @@ func (v *versionEdit) Encode(w io.Writer) error {
}
// Decode decodes an edit from the specified reader.
func (v *versionEdit) Decode(r io.Reader) error {
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(io.ByteReader)
if !ok {
br = bufio.NewReader(r)
@@ -107,6 +126,7 @@ func (v *versionEdit) Decode(r io.Reader) error {
switch tag {
case tagNewObject:
var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
var locator, customName string
var fileType base.FileType
fileNum, err = binary.ReadUvarint(br)
if err == nil {
@@ -125,13 +145,34 @@ func (v *versionEdit) Decode(r io.Reader) error {
if err == nil {
cleanupMethod, err = binary.ReadUvarint(br)
}
for err == nil {
var optionalTag uint64
optionalTag, err = binary.ReadUvarint(br)
if err != nil || optionalTag == 0 {
break
}
switch optionalTag {
case tagNewObjectLocator:
locator, err = decodeString(br)
case tagNewObjectCustomName:
customName, err = decodeString(br)
default:
err = errors.Newf("unknown newObject tag %d", optionalTag)
}
}
if err == nil {
v.NewObjects = append(v.NewObjects, SharedObjectMetadata{
FileNum: base.FileNum(fileNum).DiskFileNum(),
FileType: fileType,
CreatorID: objstorage.CreatorID(creatorID),
CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(),
CleanupMethod: objstorage.SharedCleanupMethod(cleanupMethod),
v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{
FileNum: base.FileNum(fileNum).DiskFileNum(),
FileType: fileType,
CreatorID: objstorage.CreatorID(creatorID),
CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(),
CleanupMethod: objstorage.SharedCleanupMethod(cleanupMethod),
Locator: remote.Locator(locator),
CustomObjectName: customName,
})
}
@@ -150,8 +191,9 @@ func (v *versionEdit) Decode(r io.Reader) error {
}
default:
// Ignore unknown tags.
err = errors.Newf("unknown tag %d", tag)
}
if err != nil {
if err == io.EOF {
return errCorruptCatalog
@@ -162,4 +204,51 @@ func (v *versionEdit) Decode(r io.Reader) error {
return nil
}
var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog")
func encodeString(buf []byte, s string) []byte {
buf = binary.AppendUvarint(buf, uint64(len(s)))
buf = append(buf, []byte(s)...)
return buf
}
func decodeString(br io.ByteReader) (string, error) {
length, err := binary.ReadUvarint(br)
if err != nil || length == 0 {
return "", err
}
buf := make([]byte, length)
for i := range buf {
buf[i], err = br.ReadByte()
if err != nil {
return "", err
}
}
return string(buf), nil
}
var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog")
// Apply the version edit to a creator ID and a map of objects.
func (v *VersionEdit) Apply(
creatorID *objstorage.CreatorID, objects map[base.DiskFileNum]RemoteObjectMetadata,
) error {
if v.CreatorID.IsSet() {
*creatorID = v.CreatorID
}
for _, meta := range v.NewObjects {
if invariants.Enabled {
if _, exists := objects[meta.FileNum]; exists {
return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
}
}
objects[meta.FileNum] = meta
}
for _, fileNum := range v.DeletedObjects {
if invariants.Enabled {
if _, exists := objects[fileNum]; !exists {
return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
}
}
delete(objects, fileNum)
}
return nil
}

284
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go generated vendored
View File

@@ -1,284 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"runtime"
"sync"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
"github.com/cockroachdb/pebble/objstorage/shared"
)
// sharedSubsystem contains the provider fields related to shared storage.
// All fields remain unset if shared storage is not configured.
type sharedSubsystem struct {
catalog *sharedobjcat.Catalog
cache *sharedcache.Cache
// checkRefsOnOpen controls whether we check the ref marker file when opening
// an object. Normally this is true when invariants are enabled (but the provider
// test tweaks this field).
checkRefsOnOpen bool
// initialized guards access to the creatorID field.
initialized atomic.Bool
creatorID objstorage.CreatorID
initOnce sync.Once
}
func (ss *sharedSubsystem) init(creatorID objstorage.CreatorID) {
ss.initOnce.Do(func() {
ss.creatorID = creatorID
ss.initialized.Store(true)
})
}
func (p *provider) sharedStorage() shared.Storage {
return p.st.Shared.Storage
}
// sharedInit initializes the shared object subsystem (if configured) and finds
// any shared objects.
func (p *provider) sharedInit() error {
if p.sharedStorage() == nil {
return nil
}
catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName)
if err != nil {
return errors.Wrapf(err, "pebble: could not open shared object catalog")
}
p.shared.catalog = catalog
p.shared.checkRefsOnOpen = invariants.Enabled
// The creator ID may or may not be initialized yet.
if contents.CreatorID.IsSet() {
p.shared.init(contents.CreatorID)
p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID)
} else {
p.st.Logger.Infof("shared storage configured; no creatorID yet")
}
if p.st.Shared.CacheSizeBytes > 0 {
const defaultBlockSize = 32 * 1024
blockSize := p.st.Shared.CacheBlockSize
if blockSize == 0 {
blockSize = defaultBlockSize
}
numShards := p.st.Shared.CacheShardCount
if numShards == 0 {
numShards = 2 * runtime.GOMAXPROCS(0)
}
p.shared.cache, err = sharedcache.Open(p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, p.st.Shared.CacheSizeBytes, numShards)
if err != nil {
return errors.Wrapf(err, "pebble: could not open shared object cache")
}
}
for _, meta := range contents.Objects {
o := objstorage.ObjectMetadata{
DiskFileNum: meta.FileNum,
FileType: meta.FileType,
}
o.Shared.CreatorID = meta.CreatorID
o.Shared.CreatorFileNum = meta.CreatorFileNum
o.Shared.CleanupMethod = meta.CleanupMethod
p.mu.knownObjects[o.DiskFileNum] = o
}
return nil
}
// SetCreatorID is part of the objstorage.Provider interface.
func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
if p.sharedStorage() == nil {
return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled")
}
// Note: this call is a cheap no-op if the creator ID was already set. This
// call also checks if we are trying to change the ID.
if err := p.shared.catalog.SetCreatorID(creatorID); err != nil {
return err
}
if !p.shared.initialized.Load() {
p.st.Logger.Infof("shared storage creatorID set to %s", creatorID)
p.shared.init(creatorID)
}
return nil
}
// IsForeign is part of the objstorage.Provider interface.
func (p *provider) IsForeign(meta objstorage.ObjectMetadata) bool {
if !p.shared.initialized.Load() {
return false
}
return meta.IsShared() && p.shared.creatorID != meta.Shared.CreatorID
}
func (p *provider) sharedCheckInitialized() error {
if p.sharedStorage() == nil {
return errors.Errorf("shared object support not configured")
}
if !p.shared.initialized.Load() {
return errors.Errorf("shared object support not available: shared creator ID not yet set")
}
return nil
}
func (p *provider) sharedSync() error {
batch := func() sharedobjcat.Batch {
p.mu.Lock()
defer p.mu.Unlock()
res := p.mu.shared.catalogBatch.Copy()
p.mu.shared.catalogBatch.Reset()
return res
}()
if batch.IsEmpty() {
return nil
}
err := p.shared.catalog.ApplyBatch(batch)
if err != nil {
// We have to put back the batch (for the next Sync).
p.mu.Lock()
defer p.mu.Unlock()
batch.Append(p.mu.shared.catalogBatch)
p.mu.shared.catalogBatch = batch
return err
}
return nil
}
func (p *provider) sharedPath(meta objstorage.ObjectMetadata) string {
return "shared://" + sharedObjectName(meta)
}
// sharedCreateRef creates a reference marker object.
func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
return nil
}
refName := p.sharedObjectRefName(meta)
writer, err := p.sharedStorage().CreateObject(refName)
if err == nil {
// The object is empty, just close the writer.
err = writer.Close()
}
if err != nil {
return errors.Wrapf(err, "creating marker object %q", refName)
}
return nil
}
func (p *provider) sharedCreate(
_ context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (objstorage.Writable, objstorage.ObjectMetadata, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, objstorage.ObjectMetadata{}, err
}
meta := objstorage.ObjectMetadata{
DiskFileNum: fileNum,
FileType: fileType,
}
meta.Shared.CreatorID = p.shared.creatorID
meta.Shared.CreatorFileNum = fileNum
meta.Shared.CleanupMethod = opts.SharedCleanupMethod
objName := sharedObjectName(meta)
writer, err := p.sharedStorage().CreateObject(objName)
if err != nil {
return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
}
return &sharedWritable{
p: p,
meta: meta,
storageWriter: writer,
}, meta, nil
}
func (p *provider) sharedOpenForReading(
ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, err
}
// Verify we have a reference on this object; for performance reasons, we only
// do this in testing scenarios.
if p.shared.checkRefsOnOpen && meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
refName := p.sharedObjectRefName(meta)
if _, err := p.sharedStorage().Size(refName); err != nil {
if p.sharedStorage().IsNotExistError(err) {
if opts.MustExist {
p.st.Logger.Fatalf("marker object %q does not exist", refName)
// TODO(radu): maybe list references for the object.
}
return nil, errors.Errorf("marker object %q does not exist", refName)
}
return nil, errors.Wrapf(err, "checking marker object %q", refName)
}
}
objName := sharedObjectName(meta)
reader, size, err := p.sharedStorage().ReadObject(ctx, objName)
if err != nil {
if opts.MustExist && p.sharedStorage().IsNotExistError(err) {
p.st.Logger.Fatalf("object %q does not exist", objName)
// TODO(radu): maybe list references for the object.
}
return nil, err
}
return p.newSharedReadable(reader, size, meta.DiskFileNum), nil
}
func (p *provider) sharedSize(meta objstorage.ObjectMetadata) (int64, error) {
if err := p.sharedCheckInitialized(); err != nil {
return 0, err
}
objName := sharedObjectName(meta)
return p.sharedStorage().Size(objName)
}
// sharedUnref implements object "removal" with the shared backend. The ref
// marker object is removed and the backing object is removed only if there are
// no other ref markers.
func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
if meta.Shared.CleanupMethod == objstorage.SharedNoCleanup {
// Never delete objects in this mode.
return nil
}
if p.isProtected(meta.DiskFileNum) {
// TODO(radu): we need a mechanism to unref the object when it becomes
// unprotected.
return nil
}
refName := p.sharedObjectRefName(meta)
// Tolerate a not-exists error.
if err := p.sharedStorage().Delete(refName); err != nil && !p.sharedStorage().IsNotExistError(err) {
return err
}
otherRefs, err := p.sharedStorage().List(sharedObjectRefPrefix(meta), "" /* delimiter */)
if err != nil {
return err
}
if len(otherRefs) == 0 {
objName := sharedObjectName(meta)
if err := p.sharedStorage().Delete(objName); err != nil && !p.sharedStorage().IsNotExistError(err) {
return err
}
}
return nil
}

21
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go generated vendored
View File

@@ -10,9 +10,16 @@ import (
"github.com/cockroachdb/pebble/objstorage"
)
// NewRemoteWritable creates an objstorage.Writable out of an io.WriteCloser.
func NewRemoteWritable(obj io.WriteCloser) objstorage.Writable {
return &sharedWritable{storageWriter: obj}
}
// sharedWritable is a very simple implementation of Writable on top of the
// WriteCloser returned by shared.Storage.CreateObject.
// WriteCloser returned by remote.Storage.CreateObject.
type sharedWritable struct {
// Either both p and meta must be unset / zero values, or both must be set.
// The case where both are unset is true only in tests.
p *provider
meta objstorage.ObjectMetadata
storageWriter io.WriteCloser
@@ -36,9 +43,11 @@ func (w *sharedWritable) Finish() error {
}
// Create the marker object.
if err := w.p.sharedCreateRef(w.meta); err != nil {
w.Abort()
return err
if w.p != nil {
if err := w.p.sharedCreateRef(w.meta); err != nil {
w.Abort()
return err
}
}
return nil
}
@@ -49,6 +58,8 @@ func (w *sharedWritable) Abort() {
_ = w.storageWriter.Close()
w.storageWriter = nil
}
w.p.removeMetadata(w.meta.DiskFileNum)
if w.p != nil {
w.p.removeMetadata(w.meta.DiskFileNum)
}
// TODO(radu): delete the object if it was created.
}

656
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go generated vendored
View File

@@ -8,63 +8,164 @@ import (
"context"
"fmt"
"io"
"math/bits"
"sync"
"sync/atomic"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
"github.com/prometheus/client_golang/prometheus"
)
const (
shardingBlockSize = 1024 * 1024
// Exported to enable exporting from package pebble to enable
// exporting metrics with below buckets in CRDB.
var (
IOBuckets = prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50)
ChannelWriteBuckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50)
)
// Cache is a persistent cache backed by a local filesystem. It is intended
// to cache data that is in slower shared storage (e.g. S3), hence the
// package name 'sharedcache'.
type Cache struct {
shards []shard
logger base.Logger
shards []shard
writeWorkers writeWorkers
blockSize int
bm blockMath
shardingBlockSize int64
writeBackWaitGroup sync.WaitGroup
// TODO(josh): Have a dedicated metrics struct. Right now, this
// is just for testing.
misses atomic.Int32
logger base.Logger
metrics internalMetrics
}
// Metrics is a struct containing metrics exported by the secondary cache.
// TODO(josh): Reconsider the set of metrics exported by the secondary cache
// before we release the secondary cache to users. We choose to export many metrics
// right now, so we learn a lot from the benchmarking we are doing over the 23.2
// cycle.
type Metrics struct {
// The number of sstable bytes stored in the cache.
Size int64
// The count of cache blocks in the cache (not sstable blocks).
Count int64
// The number of calls to ReadAt.
TotalReads int64
// The number of calls to ReadAt that require reading data from 2+ shards.
MultiShardReads int64
// The number of calls to ReadAt that require reading data from 2+ cache blocks.
MultiBlockReads int64
// The number of calls to ReadAt where all data returned was read from the cache.
ReadsWithFullHit int64
// The number of calls to ReadAt where some data returned was read from the cache.
ReadsWithPartialHit int64
// The number of calls to ReadAt where no data returned was read from the cache.
ReadsWithNoHit int64
// The number of times a cache block was evicted from the cache.
Evictions int64
// The number of times writing a cache block to the cache failed.
WriteBackFailures int64
// The latency of calls to get some data from the cache.
GetLatency prometheus.Histogram
// The latency of reads of a single cache block from disk.
DiskReadLatency prometheus.Histogram
// The latency of writing data to write back to the cache to a channel.
// Generally should be low, but if the channel is full, could be high.
QueuePutLatency prometheus.Histogram
// The latency of calls to put some data read from block storage into the cache.
PutLatency prometheus.Histogram
// The latency of writes of a single cache block to disk.
DiskWriteLatency prometheus.Histogram
}
// See docs at Metrics.
type internalMetrics struct {
count atomic.Int64
totalReads atomic.Int64
multiShardReads atomic.Int64
multiBlockReads atomic.Int64
readsWithFullHit atomic.Int64
readsWithPartialHit atomic.Int64
readsWithNoHit atomic.Int64
evictions atomic.Int64
writeBackFailures atomic.Int64
getLatency prometheus.Histogram
diskReadLatency prometheus.Histogram
queuePutLatency prometheus.Histogram
putLatency prometheus.Histogram
diskWriteLatency prometheus.Histogram
}
const (
// writeWorkersPerShard is used to establish the number of worker goroutines
// that perform writes to the cache.
writeWorkersPerShard = 4
// writeTaskPerWorker is used to establish how many tasks can be queued up
// until we have to block.
writeTasksPerWorker = 4
)
// Open opens a cache. If there is no existing cache at fsDir, a new one
// is created.
func Open(
fs vfs.FS, logger base.Logger, fsDir string, blockSize int, sizeBytes int64, numShards int,
fs vfs.FS,
logger base.Logger,
fsDir string,
blockSize int,
// shardingBlockSize is the size of a shard block. The cache is split into contiguous
// shardingBlockSize units. The units are distributed across multiple independent shards
// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
// policies operate at the level of shard, not whole cache. This is done to reduce lock
// contention.
shardingBlockSize int64,
sizeBytes int64,
numShards int,
) (*Cache, error) {
min := shardingBlockSize * int64(numShards)
if sizeBytes < min {
return nil, errors.Errorf("cache size %d lower than min %d", sizeBytes, min)
if minSize := shardingBlockSize * int64(numShards); sizeBytes < minSize {
// Up the size so that we have one block per shard. In practice, this should
// only happen in tests.
sizeBytes = minSize
}
sc := &Cache{
logger: logger,
blockSize: blockSize,
c := &Cache{
logger: logger,
bm: makeBlockMath(blockSize),
shardingBlockSize: shardingBlockSize,
}
sc.shards = make([]shard, numShards)
c.shards = make([]shard, numShards)
blocksPerShard := sizeBytes / int64(numShards) / int64(blockSize)
for i := range sc.shards {
if err := sc.shards[i].init(fs, fsDir, i, blocksPerShard, blockSize); err != nil {
for i := range c.shards {
if err := c.shards[i].init(c, fs, fsDir, i, blocksPerShard, blockSize, shardingBlockSize); err != nil {
return nil, err
}
}
return sc, nil
c.writeWorkers.Start(c, numShards*writeWorkersPerShard)
c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
// Measures a channel write, so lower min.
c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: ChannelWriteBuckets})
return c, nil
}
// Close closes the cache. Methods such as ReadAt should not be called after Close is
// called.
func (c *Cache) Close() error {
c.writeBackWaitGroup.Wait()
c.writeWorkers.Stop()
var retErr error
for i := range c.shards {
@@ -76,6 +177,28 @@ func (c *Cache) Close() error {
return retErr
}
// Metrics return metrics for the cache. Callers should not mutate
// the returned histograms, which are pointer types.
func (c *Cache) Metrics() Metrics {
return Metrics{
Count: c.metrics.count.Load(),
Size: c.metrics.count.Load() * int64(c.bm.BlockSize()),
TotalReads: c.metrics.totalReads.Load(),
MultiShardReads: c.metrics.multiShardReads.Load(),
MultiBlockReads: c.metrics.multiBlockReads.Load(),
ReadsWithFullHit: c.metrics.readsWithFullHit.Load(),
ReadsWithPartialHit: c.metrics.readsWithPartialHit.Load(),
ReadsWithNoHit: c.metrics.readsWithNoHit.Load(),
Evictions: c.metrics.evictions.Load(),
WriteBackFailures: c.metrics.writeBackFailures.Load(),
GetLatency: c.metrics.getLatency,
DiskReadLatency: c.metrics.diskReadLatency,
QueuePutLatency: c.metrics.queuePutLatency,
PutLatency: c.metrics.putLatency,
DiskWriteLatency: c.metrics.diskWriteLatency,
}
}
// ReadFlags contains options for Cache.ReadAt.
type ReadFlags struct {
// ReadOnly instructs ReadAt to not write any new data into the cache; it is
@@ -90,27 +213,36 @@ func (c *Cache) ReadAt(
fileNum base.DiskFileNum,
p []byte,
ofs int64,
objReader shared.ObjectReader,
objReader remote.ObjectReader,
objSize int64,
flags ReadFlags,
) error {
c.metrics.totalReads.Add(1)
if ofs >= objSize {
if invariants.Enabled {
panic("invalid ReadAt offset")
panic(fmt.Sprintf("invalid ReadAt offset %v %v", ofs, objSize))
}
return io.EOF
}
// TODO(radu): for compaction reads, we may not want to read from the cache at
// all.
{
start := time.Now()
n, err := c.get(fileNum, p, ofs)
c.metrics.getLatency.Observe(float64(time.Since(start)))
if err != nil {
return err
}
if n == len(p) {
// Everything was in cache!
c.metrics.readsWithFullHit.Add(1)
return nil
}
if n == 0 {
c.metrics.readsWithNoHit.Add(1)
} else {
c.metrics.readsWithPartialHit.Add(1)
}
// Note this. The below code does not need the original ofs, as with the earlier
// reading from the cache done, the relevant offset is ofs + int64(n). Same with p.
@@ -118,29 +250,26 @@ func (c *Cache) ReadAt(
p = p[n:]
if invariants.Enabled {
if n != 0 && ofs%int64(c.blockSize) != 0 {
if n != 0 && c.bm.Remainder(ofs) != 0 {
panic(fmt.Sprintf("after non-zero read from cache, ofs is not block-aligned: %v %v", ofs, n))
}
}
}
c.misses.Add(1)
if flags.ReadOnly {
return objReader.ReadAt(ctx, p, ofs)
}
// We must do reads with offset & size that are multiples of the block size. Else
// later cache hits may return incorrect zeroed results from the cache.
firstBlockInd := ofs / int64(c.blockSize)
adjustedOfs := firstBlockInd * int64(c.blockSize)
firstBlockInd := c.bm.Block(ofs)
adjustedOfs := c.bm.BlockOffset(firstBlockInd)
// Take the length of what is left to read plus the length of the adjustment of
// the offset plus the size of a block minus one and divide by the size of a block
// to get the number of blocks to read from the object.
sizeOfOffAdjustment := int(ofs - adjustedOfs)
numBlocksToRead := ((len(p) + sizeOfOffAdjustment) + (c.blockSize - 1)) / c.blockSize
adjustedLen := numBlocksToRead * c.blockSize
adjustedLen := int(c.bm.RoundUp(int64(len(p) + sizeOfOffAdjustment)))
adjustedP := make([]byte, adjustedLen)
// Read the rest from the object. We may need to cap the length to avoid past EOF reads.
@@ -153,20 +282,10 @@ func (c *Cache) ReadAt(
}
copy(p, adjustedP[sizeOfOffAdjustment:])
// TODO(josh): For writing back to the cache, we may want a concurrency-limited approach
// that does batching, instead of what is below. I think it's reasonable though to
// run a production experiment with what is below, before making adjustments. Note that
// the writes being done are in multiples of the filesystem block size. The filesystem
// might do okay with them.
c.writeBackWaitGroup.Add(1)
go func() {
defer c.writeBackWaitGroup.Done()
if err := c.set(fileNum, adjustedP, adjustedOfs); err != nil {
// TODO(josh): Would like to log at error severity, but base.Logger doesn't
// have error severity.
c.logger.Infof("writing back to cache after miss failed: %v", err)
}
}()
start := time.Now()
c.writeWorkers.QueueWrite(fileNum, adjustedP, adjustedOfs)
c.metrics.queuePutLatency.Observe(float64(time.Since(start)))
return nil
}
@@ -181,10 +300,11 @@ func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ err
// The data extent might cross shard boundaries, hence the loop. In the hot
// path, max two iterations of this loop will be executed, since reads are sized
// in units of sstable block size.
var multiShard bool
for {
shard := c.getShard(fileNum, ofs+int64(n))
cappedLen := len(p[n:])
if toBoundary := int(shardingBlockSize - ((ofs + int64(n)) % shardingBlockSize)); cappedLen > toBoundary {
if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
cappedLen = toBoundary
}
numRead, err := shard.get(fileNum, p[n:n+cappedLen], ofs+int64(n))
@@ -201,6 +321,10 @@ func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ err
return n, nil
}
// Data extent crosses shard boundary, continue with next shard.
if !multiShard {
c.metrics.multiShardReads.Add(1)
multiShard = true
}
}
}
@@ -210,7 +334,7 @@ func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ err
// If all of p is not written to the shard, set returns a non-nil error.
func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
if invariants.Enabled {
if ofs%int64(c.blockSize) != 0 || len(p)%c.blockSize != 0 {
if c.bm.Remainder(ofs) != 0 || c.bm.Remainder(int64(len(p))) != 0 {
panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
}
}
@@ -222,7 +346,7 @@ func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
for {
shard := c.getShard(fileNum, ofs+int64(n))
cappedLen := len(p[n:])
if toBoundary := int(shardingBlockSize - ((ofs + int64(n)) % shardingBlockSize)); cappedLen > toBoundary {
if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
cappedLen = toBoundary
}
err := shard.set(fileNum, p[n:n+cappedLen], ofs+int64(n))
@@ -241,7 +365,7 @@ func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard {
const prime64 = 1099511628211
hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs)/shardingBlockSize
hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs/c.shardingBlockSize)
// TODO(josh): Instance change ops are often run in production. Such an operation
// updates len(c.shards); see openSharedCache. As a result, the behavior of this
// function changes, and the cache empties out at restart time. We may want a better
@@ -250,31 +374,46 @@ func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard {
}
type shard struct {
file vfs.File
sizeInBlocks int64
blockSize int
mu struct {
cache *Cache
file vfs.File
sizeInBlocks int64
bm blockMath
shardingBlockSize int64
mu struct {
sync.Mutex
// TODO(josh): None of these datastructures are space-efficient.
// Focusing on correctness to start.
where whereMap
locks []lockState
free []cacheBlockIndex
where whereMap
blocks []cacheBlockState
// Head of LRU list (doubly-linked circular).
lruHead cacheBlockIndex
// Head of free list (singly-linked chain).
freeHead cacheBlockIndex
}
}
type cacheBlockState struct {
lock lockState
logical logicalBlockID
// next is the next block in the LRU or free list (or invalidBlockIndex if it
// is the last block in the free list).
next cacheBlockIndex
// prev is the previous block in the LRU list. It is not used when the block
// is in the free list.
prev cacheBlockIndex
}
// Maps a logical block in an SST to an index of the cache block with the
// file contents (to the "cache block index").
type whereMap map[logicalBlockID]cacheBlockIndex
type logicalBlockID struct {
filenum base.DiskFileNum
// Offset into the file, in units of shard.BlockSize.
offsetInUnitsOfBlocks int64
filenum base.DiskFileNum
cacheBlockIdx cacheBlockIndex
}
type cacheBlockIndex int64
type lockState int64
const (
@@ -290,15 +429,23 @@ const (
)
func (s *shard) init(
fs vfs.FS, fsDir string, shardIdx int, sizeInBlocks int64, blockSize int,
cache *Cache,
fs vfs.FS,
fsDir string,
shardIdx int,
sizeInBlocks int64,
blockSize int,
shardingBlockSize int64,
) error {
*s = shard{
cache: cache,
sizeInBlocks: sizeInBlocks,
}
if blockSize < 1024 || shardingBlockSize%blockSize != 0 {
if blockSize < 1024 || shardingBlockSize%int64(blockSize) != 0 {
return errors.Newf("invalid block size %d (must divide %d)", blockSize, shardingBlockSize)
}
s.blockSize = blockSize
s.bm = makeBlockMath(blockSize)
s.shardingBlockSize = shardingBlockSize
file, err := fs.OpenReadWrite(fs.PathJoin(fsDir, fmt.Sprintf("SHARED-CACHE-%03d", shardIdx)))
if err != nil {
return err
@@ -314,9 +461,11 @@ func (s *shard) init(
// cache contents will be over-written, since all metadata is only stored in
// memory.
s.mu.where = make(whereMap)
for i := int64(0); i < sizeInBlocks; i++ {
s.mu.locks = append(s.mu.locks, unlocked)
s.mu.free = append(s.mu.free, cacheBlockIndex(i))
s.mu.blocks = make([]cacheBlockState, sizeInBlocks)
s.mu.lruHead = invalidBlockIndex
s.mu.freeHead = invalidBlockIndex
for i := range s.mu.blocks {
s.freePush(cacheBlockIndex(i))
}
return nil
@@ -329,6 +478,59 @@ func (s *shard) close() error {
return s.file.Close()
}
// freePush pushes a block to the front of the free list.
func (s *shard) freePush(index cacheBlockIndex) {
s.mu.blocks[index].next = s.mu.freeHead
s.mu.freeHead = index
}
// freePop removes the block from the front of the free list. Must not be called
// if the list is empty (i.e. freeHead = invalidBlockIndex).
func (s *shard) freePop() cacheBlockIndex {
index := s.mu.freeHead
s.mu.freeHead = s.mu.blocks[index].next
return index
}
// lruInsertFront inserts a block at the front of the LRU list.
func (s *shard) lruInsertFront(index cacheBlockIndex) {
b := &s.mu.blocks[index]
if s.mu.lruHead == invalidBlockIndex {
b.next = index
b.prev = index
} else {
b.next = s.mu.lruHead
h := &s.mu.blocks[s.mu.lruHead]
b.prev = h.prev
s.mu.blocks[h.prev].next = index
h.prev = index
}
s.mu.lruHead = index
}
func (s *shard) lruNext(index cacheBlockIndex) cacheBlockIndex {
return s.mu.blocks[index].next
}
func (s *shard) lruPrev(index cacheBlockIndex) cacheBlockIndex {
return s.mu.blocks[index].prev
}
// lruUnlink removes a block from the LRU list.
func (s *shard) lruUnlink(index cacheBlockIndex) {
b := &s.mu.blocks[index]
if b.next == index {
s.mu.lruHead = invalidBlockIndex
} else {
s.mu.blocks[b.prev].next = b.next
s.mu.blocks[b.next].prev = b.prev
if s.mu.lruHead == index {
s.mu.lruHead = b.next
}
}
b.next, b.prev = invalidBlockIndex, invalidBlockIndex
}
// get attempts to read the requested data from the shard. The data must not
// cross a shard boundary.
//
@@ -345,7 +547,7 @@ func (s *shard) close() error {
// be needed next.
func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
if invariants.Enabled {
if ofs/shardingBlockSize != (ofs+int64(len(p))-1)/shardingBlockSize {
if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
panic(fmt.Sprintf("get crosses shard boundary: %v %v", ofs, len(p)))
}
s.assertShardStateIsConsistent()
@@ -354,13 +556,14 @@ func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ err
// The data extent might cross cache block boundaries, hence the loop. In the hot
// path, max two iterations of this loop will be executed, since reads are sized
// in units of sstable block size.
var multiBlock bool
for {
k := logicalBlockID{
filenum: fileNum,
offsetInUnitsOfBlocks: (ofs + int64(n)) / int64(s.blockSize),
filenum: fileNum,
cacheBlockIdx: s.bm.Block(ofs + int64(n)),
}
s.mu.Lock()
cacheBlockInd, ok := s.mu.where[k]
cacheBlockIdx, ok := s.mu.where[k]
// TODO(josh): Multiple reads within the same few milliseconds (anything that is smaller
// than blob storage read latency) that miss on the same logical block ID will not necessarily
// be rare. We may want to do only one read, with the later readers blocking on the first read
@@ -370,39 +573,49 @@ func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ err
s.mu.Unlock()
return n, nil
}
if s.mu.locks[cacheBlockInd] == writeLockTaken {
if s.mu.blocks[cacheBlockIdx].lock == writeLockTaken {
// In practice, if we have two reads of the same SST block in close succession, we
// would expect the second to hit in the in-memory block cache. So it's not worth
// optimizing this case here.
s.mu.Unlock()
return n, nil
}
s.mu.locks[cacheBlockInd] += readLockTakenInc
s.mu.blocks[cacheBlockIdx].lock += readLockTakenInc
// Move to front of the LRU list.
s.lruUnlink(cacheBlockIdx)
s.lruInsertFront(cacheBlockIdx)
s.mu.Unlock()
readAt := int64(cacheBlockInd) * int64(s.blockSize)
readAt := s.bm.BlockOffset(cacheBlockIdx)
readSize := s.bm.BlockSize()
if n == 0 { // if first read
readAt += ofs % int64(s.blockSize)
}
readSize := s.blockSize
if n == 0 { // if first read
// Cast to int safe since ofs is modded by block size.
readSize -= int(ofs % int64(s.blockSize))
rem := s.bm.Remainder(ofs)
readAt += rem
readSize -= int(rem)
}
if len(p[n:]) <= readSize {
start := time.Now()
numRead, err := s.file.ReadAt(p[n:], readAt)
s.dropReadLock(cacheBlockInd)
s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
s.dropReadLock(cacheBlockIdx)
return n + numRead, err
}
start := time.Now()
numRead, err := s.file.ReadAt(p[n:n+readSize], readAt)
s.dropReadLock(cacheBlockInd)
s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
s.dropReadLock(cacheBlockIdx)
if err != nil {
return 0, err
}
// Note that numRead == readSize, since we checked for an error above.
n += numRead
if !multiBlock {
s.cache.metrics.multiBlockReads.Add(1)
multiBlock = true
}
}
}
@@ -413,10 +626,10 @@ func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ err
// If all of p is not written to the shard, set returns a non-nil error.
func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
if invariants.Enabled {
if ofs/shardingBlockSize != (ofs+int64(len(p))-1)/shardingBlockSize {
if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
panic(fmt.Sprintf("set crosses shard boundary: %v %v", ofs, len(p)))
}
if ofs%int64(s.blockSize) != 0 || len(p)%s.blockSize != 0 {
if s.bm.Remainder(ofs) != 0 || s.bm.Remainder(int64(len(p))) != 0 {
panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
}
s.assertShardStateIsConsistent()
@@ -438,68 +651,73 @@ func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
// If the logical block is already in the cache, we should skip doing a set.
k := logicalBlockID{
filenum: fileNum,
offsetInUnitsOfBlocks: (ofs + int64(n)) / int64(s.blockSize),
filenum: fileNum,
cacheBlockIdx: s.bm.Block(ofs + int64(n)),
}
s.mu.Lock()
if _, ok := s.mu.where[k]; ok {
s.mu.Unlock()
n += s.blockSize
n += s.bm.BlockSize()
continue
}
// Determine cache block index by looking at the free list and randomly evicting if
// it is empty.
var cacheBlockInd cacheBlockIndex
if len(s.mu.free) == 0 {
// TODO(josh): Right now, we do random eviction. Eventually, we will do something
// more sophisticated, e.g. leverage ClockPro.
foundBlockToEvict := false
var k logicalBlockID
for k1, v := range s.mu.where {
// This implies that hot blocks will not be evicted as often, meaning the
// eviction scheme is not quite random. I think that's fine for now, since we plan
// to leverage a more sophisticated eviction scheme eventually anyway.
lock := s.mu.locks[v]
if lock >= readLockTakenInc || lock == writeLockTaken {
continue
var cacheBlockIdx cacheBlockIndex
if s.mu.freeHead == invalidBlockIndex {
if invariants.Enabled && s.mu.lruHead == invalidBlockIndex {
panic("both LRU and free lists empty")
}
// Find the last element in the LRU list which is not locked.
for idx := s.lruPrev(s.mu.lruHead); ; idx = s.lruPrev(idx) {
if lock := s.mu.blocks[idx].lock; lock == unlocked {
cacheBlockIdx = idx
break
}
if idx == s.mu.lruHead {
// No unlocked block to evict.
//
// TODO(josh): We may want to block until a block frees up, instead of returning
// an error here. But I think we can do that later on, e.g. after running some production
// experiments.
s.mu.Unlock()
return errors.New("no block to evict so skipping write to cache")
}
foundBlockToEvict = true
cacheBlockInd = v
k = k1
break
}
// TODO(josh): We may want to block until a block frees up, instead of returning
// an error here. But I think we can do that later on, e.g. after running some production
// experiments.
if !foundBlockToEvict {
s.mu.Unlock()
return errors.New("no block to evict so skipping write to cache")
}
delete(s.mu.where, k)
s.cache.metrics.evictions.Add(1)
s.lruUnlink(cacheBlockIdx)
delete(s.mu.where, s.mu.blocks[cacheBlockIdx].logical)
} else {
cacheBlockInd = s.mu.free[len(s.mu.free)-1]
s.mu.free = s.mu.free[:len(s.mu.free)-1]
s.cache.metrics.count.Add(1)
cacheBlockIdx = s.freePop()
}
s.mu.where[k] = cacheBlockInd
s.mu.locks[cacheBlockInd] = writeLockTaken
s.lruInsertFront(cacheBlockIdx)
s.mu.where[k] = cacheBlockIdx
s.mu.blocks[cacheBlockIdx].logical = k
s.mu.blocks[cacheBlockIdx].lock = writeLockTaken
s.mu.Unlock()
writeAt := int64(cacheBlockInd) * int64(s.blockSize)
writeAt := s.bm.BlockOffset(cacheBlockIdx)
writeSize := s.blockSize
writeSize := s.bm.BlockSize()
if len(p[n:]) <= writeSize {
writeSize = len(p[n:])
}
start := time.Now()
_, err := s.file.WriteAt(p[n:n+writeSize], writeAt)
s.cache.metrics.diskWriteLatency.Observe(float64(time.Since(start)))
if err != nil {
s.freeBlock(k, cacheBlockInd)
// Free the block.
s.mu.Lock()
defer s.mu.Unlock()
delete(s.mu.where, k)
s.lruUnlink(cacheBlockIdx)
s.freePush(cacheBlockIdx)
return err
}
s.dropWriteLock(cacheBlockInd)
s.dropWriteLock(cacheBlockIdx)
n += writeSize
}
}
@@ -507,11 +725,9 @@ func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
// Doesn't inline currently. This might be okay, but something to keep in mind.
func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) {
s.mu.Lock()
s.mu.locks[cacheBlockInd] -= readLockTakenInc
if invariants.Enabled {
if s.mu.locks[cacheBlockInd] < 0 {
panic(fmt.Sprintf("unexpected lock state %v in dropReadLock: %v", s.mu.locks[cacheBlockInd], s.mu.locks))
}
s.mu.blocks[cacheBlockInd].lock -= readLockTakenInc
if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock < 0 {
panic(fmt.Sprintf("unexpected lock state %v in dropReadLock", s.mu.blocks[cacheBlockInd].lock))
}
s.mu.Unlock()
}
@@ -519,21 +735,10 @@ func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) {
// Doesn't inline currently. This might be okay, but something to keep in mind.
func (s *shard) dropWriteLock(cacheBlockInd cacheBlockIndex) {
s.mu.Lock()
if invariants.Enabled {
if s.mu.locks[cacheBlockInd] != writeLockTaken {
panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock: %v", s.mu.locks[cacheBlockInd], s.mu.locks))
}
if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock != writeLockTaken {
panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock", s.mu.blocks[cacheBlockInd].lock))
}
s.mu.locks[cacheBlockInd] = unlocked
s.mu.Unlock()
}
// Doesn't inline currently. This might be okay, but something to keep in mind.
func (s *shard) freeBlock(k logicalBlockID, cacheBlockInd cacheBlockIndex) {
s.mu.Lock()
delete(s.mu.where, k)
s.mu.locks[cacheBlockInd] = unlocked
s.mu.free = append(s.mu.free, cacheBlockInd)
s.mu.blocks[cacheBlockInd].lock = unlocked
s.mu.Unlock()
}
@@ -541,30 +746,155 @@ func (s *shard) assertShardStateIsConsistent() {
s.mu.Lock()
defer s.mu.Unlock()
cacheBlockInds := make(map[cacheBlockIndex]bool)
for _, v := range s.mu.where {
if cacheBlockInds[v] {
panic(fmt.Sprintf("repeated cache block index (where): %v %v %v %v", v, cacheBlockInds, s.mu.where, s.mu.free))
}
cacheBlockInds[v] = true
}
for _, v := range s.mu.free {
if cacheBlockInds[v] {
panic(fmt.Sprintf("repeated cache block index (free): %v %v %v %v", v, cacheBlockInds, s.mu.where, s.mu.free))
}
cacheBlockInds[v] = true
}
for i := int64(0); i < s.sizeInBlocks; i++ {
if !cacheBlockInds[cacheBlockIndex(i)] {
panic(fmt.Sprintf("missing cache block index: %v %v %v %v", i, cacheBlockInds, s.mu.where, s.mu.free))
lruLen := 0
if s.mu.lruHead != invalidBlockIndex {
for b := s.mu.lruHead; ; {
lruLen++
if idx, ok := s.mu.where[s.mu.blocks[b].logical]; !ok || idx != b {
panic("block in LRU list with no entry in where map")
}
b = s.lruNext(b)
if b == s.mu.lruHead {
break
}
}
}
if int64(len(s.mu.locks)) != s.sizeInBlocks {
panic(fmt.Sprintf("lock table isn't correct size: %v %v", len(s.mu.locks), s.sizeInBlocks))
if lruLen != len(s.mu.where) {
panic(fmt.Sprintf("lru list len is %d but where map has %d entries", lruLen, len(s.mu.where)))
}
for _, ls := range s.mu.locks {
if ls < writeLockTaken {
panic(fmt.Sprintf("lock state %v is not allowed: %v", ls, s.mu.locks))
freeLen := 0
for n := s.mu.freeHead; n != invalidBlockIndex; n = s.mu.blocks[n].next {
freeLen++
}
if lruLen+freeLen != int(s.sizeInBlocks) {
panic(fmt.Sprintf("%d lru blocks and %d free blocks don't add up to %d", lruLen, freeLen, s.sizeInBlocks))
}
for i := range s.mu.blocks {
if state := s.mu.blocks[i].lock; state < writeLockTaken {
panic(fmt.Sprintf("lock state %v is not allowed", state))
}
}
}
// cacheBlockIndex is the index of a blockSize-aligned cache block.
type cacheBlockIndex int64
// invalidBlockIndex is used for the head of a list when the list is empty.
const invalidBlockIndex cacheBlockIndex = -1
// blockMath is a helper type for performing conversions between offsets and
// block indexes.
type blockMath struct {
blockSizeBits int8
}
func makeBlockMath(blockSize int) blockMath {
bm := blockMath{
blockSizeBits: int8(bits.Len64(uint64(blockSize)) - 1),
}
if blockSize != (1 << bm.blockSizeBits) {
panic(fmt.Sprintf("blockSize %d is not a power of 2", blockSize))
}
return bm
}
func (bm blockMath) mask() int64 {
return (1 << bm.blockSizeBits) - 1
}
// BlockSize returns the block size.
func (bm blockMath) BlockSize() int {
return 1 << bm.blockSizeBits
}
// Block returns the block index containing the given offset.
func (bm blockMath) Block(offset int64) cacheBlockIndex {
return cacheBlockIndex(offset >> bm.blockSizeBits)
}
// Remainder returns the offset relative to the start of the cache block.
func (bm blockMath) Remainder(offset int64) int64 {
return offset & bm.mask()
}
// BlockOffset returns the object offset where the given block starts.
func (bm blockMath) BlockOffset(block cacheBlockIndex) int64 {
return int64(block) << bm.blockSizeBits
}
// RoundUp rounds up the given value to the closest multiple of block size.
func (bm blockMath) RoundUp(x int64) int64 {
return (x + bm.mask()) & ^(bm.mask())
}
type writeWorkers struct {
doneCh chan struct{}
doneWaitGroup sync.WaitGroup
numWorkers int
tasksCh chan writeTask
}
type writeTask struct {
fileNum base.DiskFileNum
p []byte
offset int64
}
// Start starts the worker goroutines.
func (w *writeWorkers) Start(c *Cache, numWorkers int) {
doneCh := make(chan struct{})
tasksCh := make(chan writeTask, numWorkers*writeTasksPerWorker)
w.numWorkers = numWorkers
w.doneCh = doneCh
w.tasksCh = tasksCh
w.doneWaitGroup.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
defer w.doneWaitGroup.Done()
for {
select {
case <-doneCh:
return
case task, ok := <-tasksCh:
if !ok {
// The tasks channel was closed; this is used in testing code to
// ensure all writes are completed.
return
}
// TODO(radu): set() can perform multiple writes; perhaps each one
// should be its own task.
start := time.Now()
err := c.set(task.fileNum, task.p, task.offset)
c.metrics.putLatency.Observe(float64(time.Since(start)))
if err != nil {
c.metrics.writeBackFailures.Add(1)
// TODO(radu): throttle logs.
c.logger.Infof("writing back to cache after miss failed: %v", err)
}
}
}
}()
}
}
// Stop waits for any in-progress writes to complete and stops the worker
// goroutines and waits for any in-pro. Any queued writes not yet started are
// discarded.
func (w *writeWorkers) Stop() {
close(w.doneCh)
w.doneCh = nil
w.tasksCh = nil
w.doneWaitGroup.Wait()
}
// QueueWrite adds a write task to the queue. Can block if the queue is full.
func (w *writeWorkers) QueueWrite(fileNum base.DiskFileNum, p []byte, offset int64) {
w.tasksCh <- writeTask{
fileNum: fileNum,
p: p,
offset: offset,
}
}

6
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go generated vendored
View File

@@ -31,7 +31,11 @@ func (p *provider) vfsOpenForReading(
}
return nil, err
}
return newFileReadable(file, p.st.FS, filename)
readaheadConfig := DefaultReadaheadConfig
if f := p.st.Local.ReadaheadConfigFn; f != nil {
readaheadConfig = f()
}
return newFileReadable(file, p.st.FS, readaheadConfig, filename)
}
func (p *provider) vfsCreate(

64
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go generated vendored
View File

@@ -27,22 +27,26 @@ type fileReadable struct {
// The following fields are used to possibly open the file again using the
// sequential reads option (see vfsReadHandle).
filename string
fs vfs.FS
filename string
fs vfs.FS
readaheadConfig ReadaheadConfig
}
var _ objstorage.Readable = (*fileReadable)(nil)
func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, error) {
func newFileReadable(
file vfs.File, fs vfs.FS, readaheadConfig ReadaheadConfig, filename string,
) (*fileReadable, error) {
info, err := file.Stat()
if err != nil {
return nil, err
}
r := &fileReadable{
file: file,
size: info.Size(),
filename: filename,
fs: fs,
file: file,
size: info.Size(),
filename: filename,
fs: fs,
readaheadConfig: readaheadConfig,
}
invariants.SetFinalizer(r, func(obj interface{}) {
if obj.(*fileReadable).file != nil {
@@ -76,14 +80,14 @@ func (r *fileReadable) Size() int64 {
// NewReadHandle is part of the objstorage.Readable interface.
func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
rh := readHandlePool.Get().(*vfsReadHandle)
rh.r = r
rh.rs = makeReadaheadState(fileMaxReadaheadSize)
rh.init(r)
return rh
}
type vfsReadHandle struct {
r *fileReadable
rs readaheadState
r *fileReadable
rs readaheadState
readaheadMode ReadaheadMode
// sequentialFile holds a file descriptor to the same underlying File,
// except with fadvise(FADV_SEQUENTIAL) called on it to take advantage of
@@ -108,6 +112,14 @@ var readHandlePool = sync.Pool{
},
}
func (rh *vfsReadHandle) init(r *fileReadable) {
*rh = vfsReadHandle{
r: r,
rs: makeReadaheadState(fileMaxReadaheadSize),
readaheadMode: r.readaheadConfig.Speculative,
}
}
// Close is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) Close() error {
var err error
@@ -121,14 +133,17 @@ func (rh *vfsReadHandle) Close() error {
// ReadAt is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error {
var n int
var err error
if rh.sequentialFile != nil {
// Use OS-level read-ahead.
n, err = rh.sequentialFile.ReadAt(p, offset)
} else {
n, err := rh.sequentialFile.ReadAt(p, offset)
if invariants.Enabled && err == nil && n != len(p) {
panic("short read")
}
return err
}
if rh.readaheadMode != NoReadahead {
if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
if readaheadSize >= fileMaxReadaheadSize {
if rh.readaheadMode == FadviseSequential && readaheadSize >= fileMaxReadaheadSize {
// We've reached the maximum readahead size. Beyond this point, rely on
// OS-level readahead.
rh.switchToOSReadahead()
@@ -136,8 +151,8 @@ func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error
_ = rh.r.file.Prefetch(offset, readaheadSize)
}
}
n, err = rh.r.file.ReadAt(p, offset)
}
n, err := rh.r.file.ReadAt(p, offset)
if invariants.Enabled && err == nil && n != len(p) {
panic("short read")
}
@@ -146,10 +161,16 @@ func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error
// SetupForCompaction is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) SetupForCompaction() {
rh.switchToOSReadahead()
rh.readaheadMode = rh.r.readaheadConfig.Informed
if rh.readaheadMode == FadviseSequential {
rh.switchToOSReadahead()
}
}
func (rh *vfsReadHandle) switchToOSReadahead() {
if invariants.Enabled && rh.readaheadMode != FadviseSequential {
panic("readheadMode not respected")
}
if rh.sequentialFile != nil {
return
}
@@ -164,8 +185,8 @@ func (rh *vfsReadHandle) switchToOSReadahead() {
// RecordCacheHit is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
if rh.sequentialFile != nil {
// Using OS-level readahead, so do nothing.
if rh.sequentialFile != nil || rh.readaheadMode == NoReadahead {
// Using OS-level or no readahead, so do nothing.
return
}
rh.rs.recordCacheHit(offset, size)
@@ -208,8 +229,7 @@ func UsePreallocatedReadHandle(
ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
) objstorage.ReadHandle {
if r, ok := readable.(*fileReadable); ok {
// See fileReadable.NewReadHandle.
rh.vfsReadHandle = vfsReadHandle{r: r}
rh.init(r)
return rh
}
return readable.NewReadHandle(ctx)

25
vendor/github.com/cockroachdb/pebble/objstorage/remote/factory.go generated vendored Normal file
View File

@@ -0,0 +1,25 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package remote
import "github.com/pkg/errors"
// MakeSimpleFactory returns a StorageFactory implementation that produces the given
// Storage objects.
func MakeSimpleFactory(m map[Locator]Storage) StorageFactory {
return simpleFactory(m)
}
type simpleFactory map[Locator]Storage
var _ StorageFactory = simpleFactory{}
// CreateStorage is part of the StorageFactory interface.
func (sf simpleFactory) CreateStorage(locator Locator) (Storage, error) {
if s, ok := sf[locator]; ok {
return s, nil
}
return nil, errors.Errorf("unknown locator '%s'", locator)
}

118
vendor/github.com/cockroachdb/pebble/objstorage/remote/localfs.go generated vendored Normal file
View File

@@ -0,0 +1,118 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package remote
import (
"context"
"io"
"os"
"path"
"github.com/cockroachdb/pebble/vfs"
)
// NewLocalFS returns a vfs-backed implementation of the remote.Storage
// interface (for testing). All objects will be stored at the directory
// dirname.
func NewLocalFS(dirname string, fs vfs.FS) Storage {
store := &localFSStore{
dirname: dirname,
vfs: fs,
}
return store
}
// localFSStore is a vfs-backed implementation of the remote.Storage
// interface (for testing).
type localFSStore struct {
dirname string
vfs vfs.FS
}
var _ Storage = (*localFSStore)(nil)
// Close is part of the remote.Storage interface.
func (s *localFSStore) Close() error {
*s = localFSStore{}
return nil
}
// ReadObject is part of the remote.Storage interface.
func (s *localFSStore) ReadObject(
ctx context.Context, objName string,
) (_ ObjectReader, objSize int64, _ error) {
f, err := s.vfs.Open(path.Join(s.dirname, objName))
if err != nil {
return nil, 0, err
}
stat, err := f.Stat()
if err != nil {
return nil, 0, err
}
return &localFSReader{f}, stat.Size(), nil
}
type localFSReader struct {
file vfs.File
}
var _ ObjectReader = (*localFSReader)(nil)
// ReadAt is part of the shared.ObjectReader interface.
func (r *localFSReader) ReadAt(_ context.Context, p []byte, offset int64) error {
n, err := r.file.ReadAt(p, offset)
// https://pkg.go.dev/io#ReaderAt
if err == io.EOF && n == len(p) {
return nil
}
return err
}
// Close is part of the shared.ObjectReader interface.
func (r *localFSReader) Close() error {
r.file.Close()
r.file = nil
return nil
}
// CreateObject is part of the remote.Storage interface.
func (s *localFSStore) CreateObject(objName string) (io.WriteCloser, error) {
file, err := s.vfs.Create(path.Join(s.dirname, objName))
return file, err
}
// List is part of the remote.Storage interface.
func (s *localFSStore) List(prefix, delimiter string) ([]string, error) {
// TODO(josh): For the intended use case of localfs.go of running 'pebble bench',
// List can always return <nil, nil>, since this indicates a file has only one ref,
// and since `pebble bench` implies running in a single-pebble-instance context.
// https://github.com/cockroachdb/pebble/blob/a9a079d4fb6bf4a9ebc52e4d83a76ad4cbf676cb/objstorage/objstorageprovider/shared.go#L292
return nil, nil
}
// Delete is part of the remote.Storage interface.
func (s *localFSStore) Delete(objName string) error {
return s.vfs.Remove(path.Join(s.dirname, objName))
}
// Size is part of the remote.Storage interface.
func (s *localFSStore) Size(objName string) (int64, error) {
f, err := s.vfs.Open(path.Join(s.dirname, objName))
if err != nil {
return 0, err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return 0, err
}
return stat.Size(), nil
}
// IsNotExistError is part of the remote.Storage interface.
func (s *localFSStore) IsNotExistError(err error) bool {
return err == os.ErrNotExist
}

4
vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go → vendor/github.com/cockroachdb/pebble/objstorage/remote/logging.go generated vendored
View File

@@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
package remote
import (
"context"
@@ -20,7 +20,7 @@ func WithLogging(wrapped Storage, logf func(fmt string, args ...interface{})) St
}
}
// loggingStore wraps a shared.Storage implementation and emits logs of the
// loggingStore wraps a remote.Storage implementation and emits logs of the
// operations.
type loggingStore struct {
logf func(fmt string, args ...interface{})

6
vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go → vendor/github.com/cockroachdb/pebble/objstorage/remote/mem.go generated vendored
View File

@@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
package remote
import (
"bytes"
@@ -13,7 +13,7 @@ import (
"sync"
)
// NewInMem returns an in-memory implementation of the shared.Storage
// NewInMem returns an in-memory implementation of the remote.Storage
// interface (for testing).
func NewInMem() Storage {
store := &inMemStore{}
@@ -21,7 +21,7 @@ func NewInMem() Storage {
return store
}
// inMemStore is an in-memory implementation of the shared.Storage interface
// inMemStore is an in-memory implementation of the remote.Storage interface
// (for testing).
type inMemStore struct {
mu struct {

51
vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go → vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go generated vendored
View File

@@ -2,13 +2,62 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
package remote
import (
"context"
"io"
)
// Locator is an opaque string identifying a remote.Storage implementation.
//
// The Locator must not contain secrets (like authentication keys). Locators are
// stored on disk in the shared object catalog and are passed around as part of
// RemoteObjectBacking; they can also appear in error messages.
type Locator string
// StorageFactory is used to return Storage implementations based on locators. A
// Pebble store that uses shared storage is configured with a StorageFactory.
type StorageFactory interface {
CreateStorage(locator Locator) (Storage, error)
}
// SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will
// have sstables shared across Pebble instances when doing skip-shared
// iteration (see db.ScanInternal) or shared file ingestion (see
// db.IngestAndExcise).
const SharedLevelsStart = 5
// CreateOnSharedStrategy specifies what table files should be created on shared
// storage. For use with CreateOnShared in options.
type CreateOnSharedStrategy int
const (
// CreateOnSharedNone denotes no files being created on shared storage.
CreateOnSharedNone CreateOnSharedStrategy = iota
// CreateOnSharedLower denotes the creation of files in lower levels of the
// LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on
// shared storage, and higher levels on local storage.
CreateOnSharedLower
// CreateOnSharedAll denotes the creation of all sstables on shared storage.
CreateOnSharedAll
)
// ShouldCreateShared returns whether new table files at the specified level
// should be created on shared storage.
func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool {
switch strategy {
case CreateOnSharedAll:
return true
case CreateOnSharedNone:
return false
case CreateOnSharedLower:
return level >= SharedLevelsStart
default:
panic("unexpected CreateOnSharedStrategy value")
}
}
// Storage is an interface for a blob storage driver. This is lower-level
// than an FS-like interface, however FS/File-like abstractions can be built on
// top of these methods.

156
vendor/github.com/cockroachdb/pebble/open.go generated vendored
View File

@@ -6,6 +6,7 @@ package pebble
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
@@ -20,13 +21,14 @@ import (
"github.com/cockroachdb/pebble/internal/arenaskl"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/constants"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/internal/manual"
"github.com/cockroachdb/pebble/internal/rate"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
"github.com/prometheus/client_golang/prometheus"
)
@@ -36,11 +38,23 @@ const (
// The max batch size is limited by the uint32 offsets stored in
// internal/batchskl.node, DeferredBatchOp, and flushableBatchEntry.
maxBatchSize = 4 << 30 // 4 GB
//
// We limit the size to MaxUint32 (just short of 4GB) so that the exclusive
// end of an allocation fits in uint32.
//
// On 32-bit systems, slices are naturally limited to MaxInt (just short of
// 2GB).
maxBatchSize = constants.MaxUint32OrInt
// The max memtable size is limited by the uint32 offsets stored in
// internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry.
maxMemTableSize = 4 << 30 // 4 GB
//
// We limit the size to MaxUint32 (just short of 4GB) so that the exclusive
// end of an allocation fits in uint32.
//
// On 32-bit systems, slices are naturally limited to MaxInt (just short of
// 2GB).
maxMemTableSize = constants.MaxUint32OrInt
)
// TableCacheSize can be used to determine the table
@@ -75,7 +89,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
// Open the database and WAL directories first.
walDirname, dataDir, walDir, err := prepareAndOpenDirs(dirname, opts)
if err != nil {
return nil, err
return nil, errors.Wrapf(err, "error opening database at %q", dirname)
}
defer func() {
if db == nil {
@@ -159,7 +173,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
merge: opts.Merger.Merge,
split: opts.Comparer.Split,
abbreviatedKey: opts.Comparer.AbbreviatedKey,
largeBatchThreshold: (opts.MemTableSize - int(memTableEmptySize)) / 2,
largeBatchThreshold: (opts.MemTableSize - uint64(memTableEmptySize)) / 2,
fileLock: fileLock,
dataDir: dataDir,
walDir: walDir,
@@ -169,7 +183,6 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
d.mu.versions = &versionSet{}
d.diskAvailBytes.Store(math.MaxUint64)
d.mu.versions.diskAvailBytes = d.getDiskAvailableBytesCached
defer func() {
// If an error or panic occurs during open, attempt to release the manually
@@ -194,6 +207,9 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
t.arenaBuf = nil
}
}
if d.cleanupManager != nil {
d.cleanupManager.Close()
}
if d.objProvider != nil {
d.objProvider.Close()
}
@@ -209,24 +225,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
apply: d.commitApply,
write: d.commitWrite,
})
d.deletionLimiter = rate.NewLimiter(
rate.Limit(d.opts.TargetByteDeletionRate),
d.opts.TargetByteDeletionRate)
d.mu.nextJobID = 1
d.mu.mem.nextSize = opts.MemTableSize
if d.mu.mem.nextSize > initialMemTableSize {
d.mu.mem.nextSize = initialMemTableSize
}
d.mu.cleaner.cond.L = &d.mu.Mutex
d.mu.compact.cond.L = &d.mu.Mutex
d.mu.compact.inProgress = make(map[*compaction]struct{})
d.mu.compact.noOngoingFlushStartTime = time.Now()
d.mu.snapshots.init()
// logSeqNum is the next sequence number that will be assigned. Start
// assigning sequence numbers from base.SeqNumStart to leave room for reserved
// sequence numbers (see comments around SeqNumStart).
// logSeqNum is the next sequence number that will be assigned.
// Start assigning sequence numbers from base.SeqNumStart to leave
// room for reserved sequence numbers (see comments around
// SeqNumStart).
d.mu.versions.logSeqNum.Store(base.SeqNumStart)
d.mu.formatVers.vers = formatVersion
d.mu.formatVers.vers.Store(uint64(formatVersion))
d.mu.formatVers.marker = formatVersionMarker
d.timeNow = time.Now
@@ -238,7 +251,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
jobID := d.mu.nextJobID
d.mu.nextJobID++
setCurrent := setCurrentFunc(d.mu.formatVers.vers, manifestMarker, opts.FS, dirname, d.dataDir)
setCurrent := setCurrentFunc(d.FormatMajorVersion(), manifestMarker, opts.FS, dirname, d.dataDir)
if !manifestExists {
// DB does not exist.
@@ -247,7 +260,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
// Create the DB.
if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
return nil, err
}
} else {
@@ -255,7 +268,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname)
}
// Load the version set.
if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
return nil, err
}
if opts.ErrorIfNotPristine {
@@ -297,13 +310,19 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
NoSyncOnClose: opts.NoSyncOnClose,
BytesPerSync: opts.BytesPerSync,
}
providerSettings.Shared.Storage = opts.Experimental.SharedStorage
providerSettings.Local.ReadaheadConfigFn = opts.Local.ReadaheadConfigFn
providerSettings.Remote.StorageFactory = opts.Experimental.RemoteStorage
providerSettings.Remote.CreateOnShared = opts.Experimental.CreateOnShared
providerSettings.Remote.CreateOnSharedLocator = opts.Experimental.CreateOnSharedLocator
providerSettings.Remote.CacheSizeBytes = opts.Experimental.SecondaryCacheSizeBytes
d.objProvider, err = objstorageprovider.Open(providerSettings)
if err != nil {
return nil, err
}
d.cleanupManager = openCleanupManager(opts, d.objProvider, d.onObsoleteTableDelete, d.getDeletionPacerInfo)
if manifestExists {
curVersion := d.mu.versions.currentVersion()
if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil {
@@ -363,6 +382,15 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
}
// Ratchet d.mu.versions.nextFileNum ahead of all known objects in the
// objProvider. This avoids FileNum collisions with obsolete sstables.
objects := d.objProvider.List()
for _, obj := range objects {
if d.mu.versions.nextFileNum <= obj.DiskFileNum.FileNum() {
d.mu.versions.nextFileNum = obj.DiskFileNum.FileNum() + 1
}
}
// Validate the most-recent OPTIONS file, if there is one.
var strictWALTail bool
if previousOptionsFilename != "" {
@@ -462,7 +490,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
//
// We ratchet the version this far into Open so that migrations have a read
// state available.
if !d.opts.ReadOnly && opts.FormatMajorVersion > d.mu.formatVers.vers {
if !d.opts.ReadOnly && opts.FormatMajorVersion > d.FormatMajorVersion() {
if err := d.ratchetFormatMajorVersionLocked(opts.FormatMajorVersion); err != nil {
return nil, err
}
@@ -505,7 +533,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
if !d.opts.ReadOnly {
d.scanObsoleteFiles(ls)
d.deleteObsoleteFiles(jobID, true /* waitForOngoing */)
d.deleteObsoleteFiles(jobID)
} else {
// All the log files are obsolete.
d.mu.versions.metrics.WAL.Files = int64(len(logFiles))
@@ -678,8 +706,15 @@ func (d *DB) replayWAL(
rr = record.NewReader(file, logNum)
offset int64 // byte offset in rr
lastFlushOffset int64
keysReplayed int64 // number of keys replayed
batchesReplayed int64 // number of batches replayed
)
// TODO(jackson): This function is interspersed with panics, in addition to
// corruption error propagation. Audit them to ensure we're truly only
// panicking where the error points to Pebble bug and not user or
// hardware-induced corruption.
if d.opts.ReadOnly {
// In read-only mode, we replay directly into the mutable memtable which will
// never be flushed.
@@ -734,6 +769,11 @@ func (d *DB) replayWAL(
ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...)
return nil
}
defer func() {
if err != nil {
err = errors.WithDetailf(err, "replaying log %s, offset %d", logNum, offset)
}
}()
for {
offset = rr.Offset()
@@ -766,14 +806,18 @@ func (d *DB) replayWAL(
// Specify Batch.db so that Batch.SetRepr will compute Batch.memTableSize
// which is used below.
b = Batch{db: d}
b = Batch{}
b.db = d
b.SetRepr(buf.Bytes())
seqNum := b.SeqNum()
maxSeqNum = seqNum + uint64(b.Count())
keysReplayed += int64(b.Count())
batchesReplayed++
{
br := b.Reader()
if kind, encodedFileNum, _, _ := br.Next(); kind == InternalKeyKindIngestSST {
if kind, encodedFileNum, _, ok, err := br.Next(); err != nil {
return nil, 0, err
} else if ok && kind == InternalKeyKindIngestSST {
fileNums := make([]base.DiskFileNum, 0, b.Count())
addFileNum := func(encodedFileNum []byte) {
fileNum, n := binary.Uvarint(encodedFileNum)
@@ -785,7 +829,10 @@ func (d *DB) replayWAL(
addFileNum(encodedFileNum)
for i := 1; i < int(b.Count()); i++ {
kind, encodedFileNum, _, ok := br.Next()
kind, encodedFileNum, _, ok, err := br.Next()
if err != nil {
return nil, 0, err
}
if kind != InternalKeyKindIngestSST {
panic("pebble: invalid batch key kind.")
}
@@ -795,21 +842,42 @@ func (d *DB) replayWAL(
addFileNum(encodedFileNum)
}
if _, _, _, ok := br.Next(); ok {
if _, _, _, ok, err := br.Next(); err != nil {
return nil, 0, err
} else if ok {
panic("pebble: invalid number of entries in batch.")
}
paths := make([]string, len(fileNums))
meta := make([]*fileMetadata, len(fileNums))
for i, n := range fileNums {
paths[i] = base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
}
var readable objstorage.Readable
objMeta, err := d.objProvider.Lookup(fileTypeTable, n)
if err != nil {
return nil, 0, errors.Wrap(err, "pebble: error when looking up ingested SSTs")
}
if objMeta.IsRemote() {
readable, err = d.objProvider.OpenForReading(context.TODO(), fileTypeTable, n, objstorage.OpenOptions{MustExist: true})
if err != nil {
return nil, 0, errors.Wrap(err, "pebble: error when opening flushable ingest files")
}
} else {
path := base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
f, err := d.opts.FS.Open(path)
if err != nil {
return nil, 0, err
}
var lr ingestLoadResult
lr, err = ingestLoad(d.opts, d.mu.formatVers.vers, paths, nil, d.cacheID, fileNums)
if err != nil {
return nil, 0, err
readable, err = sstable.NewSimpleReadable(f)
if err != nil {
return nil, 0, err
}
}
// NB: ingestLoad1 will close readable.
meta[i], err = ingestLoad1(d.opts, d.FormatMajorVersion(), readable, d.cacheID, n)
if err != nil {
return nil, 0, errors.Wrap(err, "pebble: error when loading flushable ingest files")
}
}
meta := lr.localMeta
if uint32(len(meta)) != b.Count() {
panic("pebble: couldn't load all files in WAL entry.")
@@ -869,11 +937,14 @@ func (d *DB) replayWAL(
// Make a copy of the data slice since it is currently owned by buf and will
// be reused in the next iteration.
b.data = append([]byte(nil), b.data...)
b.flushable = newFlushableBatch(&b, d.opts.Comparer)
b.flushable, err = newFlushableBatch(&b, d.opts.Comparer)
if err != nil {
return nil, 0, err
}
entry := d.newFlushableEntry(b.flushable, logNum, b.SeqNum())
// Disable memory accounting by adding a reader ref that will never be
// removed.
entry.readerRefs++
entry.readerRefs.Add(1)
if d.opts.ReadOnly {
d.mu.mem.queue = append(d.mu.mem.queue, entry)
// We added the flushable batch to the flushable to the queue.
@@ -911,7 +982,10 @@ func (d *DB) replayWAL(
}
buf.Reset()
}
d.opts.Logger.Infof("[JOB %d] WAL file %s with log number %s stopped reading at offset: %d; replayed %d keys in %d batches", jobID, filename, logNum.String(), offset, keysReplayed, batchesReplayed)
flushMem()
// mem is nil here.
if !d.opts.ReadOnly {
err = updateVE()
@@ -1063,6 +1137,12 @@ var ErrDBAlreadyExists = errors.New("pebble: database already exists")
// Note that errors can be wrapped with more details; use errors.Is().
var ErrDBNotPristine = errors.New("pebble: database already exists and is not pristine")
// IsCorruptionError returns true if the given error indicates database
// corruption.
func IsCorruptionError(err error) bool {
return errors.Is(err, base.ErrCorruption)
}
func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error {
var buf bytes.Buffer
var args []interface{}
@@ -1078,9 +1158,15 @@ func checkConsistency(v *manifest.Version, dirname string, objProvider objstorag
dedup[backingState.DiskFileNum] = struct{}{}
fileNum := backingState.DiskFileNum
fileSize := backingState.Size
// We allow foreign objects to have a mismatch between sizes. This is
// because we might skew the backing size stored by our objprovider
// to prevent us from over-prioritizing this file for compaction.
meta, err := objProvider.Lookup(base.FileTypeTable, fileNum)
var size int64
if err == nil {
if objProvider.IsSharedForeign(meta) {
continue
}
size, err = objProvider.Size(meta)
}
if err != nil {

166
vendor/github.com/cockroachdb/pebble/options.go generated vendored
View File

@@ -14,12 +14,15 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/fifo"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/rangekey"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
)
@@ -118,6 +121,16 @@ type IterOptions struct {
// false to skip scanning. This function must be thread-safe since the same
// function can be used by multiple iterators, if the iterator is cloned.
TableFilter func(userProps map[string]string) bool
// SkipPoint may be used to skip over point keys that don't match an
// arbitrary predicate during iteration. If set, the Iterator invokes
// SkipPoint for keys encountered. If SkipPoint returns true, the iterator
// will skip the key without yielding it to the iterator operation in
// progress.
//
// SkipPoint must be a pure function and always return the same result when
// provided the same arguments. The iterator may call SkipPoint multiple
// times for the same user key.
SkipPoint func(userKey []byte) bool
// PointKeyFilters can be used to avoid scanning tables and blocks in tables
// when iterating over point keys. This slice represents an intersection
// across all filters, i.e., all filters must indicate that the block is
@@ -231,13 +244,12 @@ func (o *IterOptions) getLogger() Logger {
}
// SpanIterOptions creates a SpanIterOptions from this IterOptions.
func (o *IterOptions) SpanIterOptions(level manifest.Level) keyspan.SpanIterOptions {
func (o *IterOptions) SpanIterOptions() keyspan.SpanIterOptions {
if o == nil {
return keyspan.SpanIterOptions{Level: level}
return keyspan.SpanIterOptions{}
}
return keyspan.SpanIterOptions{
RangeKeyFilters: o.RangeKeyFilters,
Level: level,
}
}
@@ -246,9 +258,21 @@ func (o *IterOptions) SpanIterOptions(level manifest.Level) keyspan.SpanIterOpti
type scanInternalOptions struct {
IterOptions
visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error
visitRangeDel func(start, end []byte, seqNum uint64) error
visitRangeKey func(start, end []byte, keys []rangekey.Key) error
visitSharedFile func(sst *SharedSSTMeta) error
// skipSharedLevels skips levels that are shareable (level >=
// sharedLevelStart).
skipSharedLevels bool
// includeObsoleteKeys specifies whether keys shadowed by newer internal keys
// are exposed. If false, only one internal key per user key is exposed.
includeObsoleteKeys bool
// rateLimitFunc is used to limit the amount of bytes read per second.
rateLimitFunc func(key *InternalKey, value LazyValue) error
}
// RangeKeyMasking configures automatic hiding of point keys by range keys. A
@@ -459,11 +483,25 @@ type Options struct {
// The default cache size is 8 MB.
Cache *cache.Cache
// LoadBlockSema, if set, is used to limit the number of blocks that can be
// loaded (i.e. read from the filesystem) in parallel. Each load acquires one
// unit from the semaphore for the duration of the read.
LoadBlockSema *fifo.Semaphore
// Cleaner cleans obsolete files.
//
// The default cleaner uses the DeleteCleaner.
Cleaner Cleaner
// Local contains option that pertain to files stored on the local filesystem.
Local struct {
// ReadaheadConfigFn is a function used to retrieve the current readahead
// mode. This function is consulted when a table enters the table cache.
ReadaheadConfigFn func() ReadaheadConfig
// TODO(radu): move BytesPerSync, LoadBlockSema, Cleaner here.
}
// Comparer defines a total ordering over the space of []byte keys: a 'less
// than' relationship. The same comparison algorithm must be used for reads
// and writes over the lifetime of the DB.
@@ -526,7 +564,12 @@ type Options struct {
// concurrent compaction is added. This works "on top" of
// L0CompactionConcurrency, so the higher of the count of compaction
// concurrency slots as determined by the two options is chosen.
CompactionDebtConcurrency int
CompactionDebtConcurrency uint64
// IngestSplit, if it returns true, allows for ingest-time splitting of
// existing sstables into two virtual sstables to allow ingestion sstables to
// slot into a lower level than they otherwise would have.
IngestSplit func() bool
// ReadCompactionRate controls the frequency of read triggered
// compactions by adjusting `AllowedSeeks` in manifest.FileMetadata:
@@ -591,10 +634,10 @@ type Options struct {
// desired size of each level of the LSM. Defaults to 10.
LevelMultiplier int
// MultiLevelCompactionHueristic determines whether to add an additional
// MultiLevelCompactionHeuristic determines whether to add an additional
// level to a conventional two level compaction. If nil, a multilevel
// compaction will never get triggered.
MultiLevelCompactionHueristic MultiLevelHeuristic
MultiLevelCompactionHeuristic MultiLevelHeuristic
// MaxWriterConcurrency is used to indicate the maximum number of
// compression workers the compression queue is allowed to use. If
@@ -653,15 +696,49 @@ type Options struct {
// major version is at least `FormatFlushableIngest`.
DisableIngestAsFlushable func() bool
// SharedStorage is a second FS-like storage medium that can be shared
// between multiple Pebble instances. It is used to store sstables only, and
// is managed by objstorage.Provider. Each sstable might only be written to
// by one Pebble instance, but other Pebble instances can possibly read the
// same files if they have the path to get to them. The pebble instance that
// wrote a file should not delete it if other Pebble instances are known to
// be reading this file. This FS is expected to have slower read/write
// performance than the default FS above.
SharedStorage shared.Storage
// RemoteStorage enables use of remote storage (e.g. S3) for storing
// sstables. Setting this option enables use of CreateOnShared option and
// allows ingestion of external files.
RemoteStorage remote.StorageFactory
// If CreateOnShared is non-zero, new sstables are created on remote storage
// (using CreateOnSharedLocator and with the appropriate
// CreateOnSharedStrategy). These sstables can be shared between different
// Pebble instances; the lifecycle of such objects is managed by the
// remote.Storage constructed by options.RemoteStorage.
//
// Can only be used when RemoteStorage is set (and recognizes
// CreateOnSharedLocator).
CreateOnShared remote.CreateOnSharedStrategy
CreateOnSharedLocator remote.Locator
// CacheSizeBytesBytes is the size of the on-disk block cache for objects
// on shared storage in bytes. If it is 0, no cache is used.
SecondaryCacheSizeBytes int64
// IneffectualPointDeleteCallback is called in compactions/flushes if any
// single delete is being elided without deleting a point set/merge.
IneffectualSingleDeleteCallback func(userKey []byte)
// SingleDeleteInvariantViolationCallback is called in compactions/flushes if any
// single delete has consumed a Set/Merge, and there is another immediately older
// Set/SetWithDelete/Merge. The user of Pebble has violated the invariant under
// which SingleDelete can be used correctly.
//
// Consider the sequence SingleDelete#3, Set#2, Set#1. There are three
// ways some of these keys can first meet in a compaction.
//
// - All 3 keys in the same compaction: this callback will detect the
// violation.
//
// - SingleDelete#3, Set#2 meet in a compaction first: Both keys will
// disappear. The violation will not be detected, and the DB will have
// Set#1 which is likely incorrect (from the user's perspective).
//
// - Set#2, Set#1 meet in a compaction first: The output will be Set#2,
// which will later be consumed by SingleDelete#3. The violation will
// not be detected and the DB will be correct.
SingleDeleteInvariantViolationCallback func(userKey []byte)
}
// Filters is a map from filter policy name to filter policy. It is used for
@@ -776,12 +853,18 @@ type Options struct {
// writing the contents of the old one in the
// background. MemTableStopWritesThreshold places a hard limit on the size of
// the queued MemTables.
MemTableSize int
//
// The default value is 4MB.
MemTableSize uint64
// Hard limit on the size of queued of MemTables. Writes are stopped when the
// sum of the queued memtable sizes exceeds
// MemTableStopWritesThreshold*MemTableSize. This value should be at least 2
// or writes will stop whenever a MemTable is being flushed.
// Hard limit on the number of queued of MemTables. Writes are stopped when
// the sum of the queued memtable sizes exceeds:
// MemTableStopWritesThreshold * MemTableSize.
//
// This value should be at least 2 or writes will stop whenever a MemTable is
// being flushed.
//
// The default value is 2.
MemTableStopWritesThreshold int
// Merger defines the associative merge operation to use for merging values
@@ -872,9 +955,10 @@ type Options struct {
// Deletion pacing is used to slow down deletions when compactions finish up
// or readers close and newly-obsolete files need cleaning up. Deleting lots
// of files at once can cause disk latency to go up on some SSDs, which this
// functionality guards against. This is only a best-effort target; pacing is
// disabled when there are too many obsolete files relative to live bytes, or
// there isn't enough disk space available.
// functionality guards against.
//
// This value is only a best-effort target; the effective rate can be
// higher if deletions are falling behind or disk space is running low.
//
// Setting this to 0 disables deletion pacing, which is also the default.
TargetByteDeletionRate int
@@ -912,6 +996,10 @@ type Options struct {
// A private option to disable stats collection.
disableTableStats bool
// testingAlwaysWaitForCleanup is set by some tests to force waiting for
// obsolete file deletion (to make events deterministic).
testingAlwaysWaitForCleanup bool
// fsCloser holds a closer that should be invoked after a DB using these
// Options is closed. This is used to automatically stop the
// long-running goroutine associated with the disk-health-checking FS.
@@ -924,6 +1012,9 @@ type Options struct {
}
}
// ReadaheadConfig controls the use of read-ahead.
type ReadaheadConfig = objstorageprovider.ReadaheadConfig
// DebugCheckLevels calls CheckLevels on the provided database.
// It may be set in the DebugCheck field of Options to check
// level invariants whenever a new version is installed.
@@ -1023,7 +1114,7 @@ func (o *Options) EnsureDefaults() *Options {
o.MaxOpenFiles = 1000
}
if o.MemTableSize <= 0 {
o.MemTableSize = 4 << 20
o.MemTableSize = 4 << 20 // 4 MB
}
if o.MemTableStopWritesThreshold <= 0 {
o.MemTableStopWritesThreshold = 2
@@ -1064,8 +1155,8 @@ func (o *Options) EnsureDefaults() *Options {
if o.Experimental.CPUWorkPermissionGranter == nil {
o.Experimental.CPUWorkPermissionGranter = defaultCPUWorkGranter{}
}
if o.Experimental.MultiLevelCompactionHueristic == nil {
o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{}
if o.Experimental.MultiLevelCompactionHeuristic == nil {
o.Experimental.MultiLevelCompactionHeuristic = WriteAmpHeuristic{}
}
o.initMaps()
@@ -1205,6 +1296,8 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, " wal_bytes_per_sync=%d\n", o.WALBytesPerSync)
fmt.Fprintf(&buf, " max_writer_concurrency=%d\n", o.Experimental.MaxWriterConcurrency)
fmt.Fprintf(&buf, " force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism)
fmt.Fprintf(&buf, " secondary_cache_size_bytes=%d\n", o.Experimental.SecondaryCacheSizeBytes)
fmt.Fprintf(&buf, " create_on_shared=%d\n", o.Experimental.CreateOnShared)
// Private options.
//
@@ -1260,7 +1353,11 @@ func parseOptions(s string, fn func(section, key, value string) error) error {
pos := strings.Index(line, "=")
if pos < 0 {
return errors.Errorf("pebble: invalid key=value syntax: %s", errors.Safe(line))
const maxLen = 50
if len(line) > maxLen {
line = line[:maxLen-3] + "..."
}
return base.CorruptionErrorf("invalid key=value syntax: %q", errors.Safe(line))
}
key := strings.TrimSpace(line[:pos])
@@ -1358,7 +1455,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
}
}
case "compaction_debt_concurrency":
o.Experimental.CompactionDebtConcurrency, err = strconv.Atoi(value)
o.Experimental.CompactionDebtConcurrency, err = strconv.ParseUint(value, 10, 64)
case "delete_range_flush_delay":
// NB: This is a deprecated serialization of the
// `flush_delay_delete_range`.
@@ -1423,7 +1520,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
case "max_open_files":
o.MaxOpenFiles, err = strconv.Atoi(value)
case "mem_table_size":
o.MemTableSize, err = strconv.Atoi(value)
o.MemTableSize, err = strconv.ParseUint(value, 10, 64)
case "mem_table_stop_writes_threshold":
o.MemTableStopWritesThreshold, err = strconv.Atoi(value)
case "min_compaction_rate":
@@ -1474,6 +1571,12 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
o.Experimental.MaxWriterConcurrency, err = strconv.Atoi(value)
case "force_writer_parallelism":
o.Experimental.ForceWriterParallelism, err = strconv.ParseBool(value)
case "secondary_cache_size_bytes":
o.Experimental.SecondaryCacheSizeBytes, err = strconv.ParseInt(value, 10, 64)
case "create_on_shared":
var createOnSharedInt int64
createOnSharedInt, err = strconv.ParseInt(value, 10, 64)
o.Experimental.CreateOnShared = remote.CreateOnSharedStrategy(createOnSharedInt)
default:
if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
return nil
@@ -1604,7 +1707,7 @@ func (o *Options) Validate() error {
}
if uint64(o.MemTableSize) >= maxMemTableSize {
fmt.Fprintf(&buf, "MemTableSize (%s) must be < %s\n",
humanize.Uint64(uint64(o.MemTableSize)), humanize.Uint64(maxMemTableSize))
humanize.Bytes.Uint64(uint64(o.MemTableSize)), humanize.Bytes.Uint64(maxMemTableSize))
}
if o.MemTableStopWritesThreshold < 2 {
fmt.Fprintf(&buf, "MemTableStopWritesThreshold (%d) must be >= 2\n",
@@ -1629,6 +1732,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions {
var readerOpts sstable.ReaderOptions
if o != nil {
readerOpts.Cache = o.Cache
readerOpts.LoadBlockSema = o.LoadBlockSema
readerOpts.Comparer = o.Comparer
readerOpts.Filters = o.Filters
if o.Merger != nil {

238
vendor/github.com/cockroachdb/pebble/pacer.go generated vendored
View File

@@ -5,29 +5,13 @@
package pebble
import (
"sync"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/rate"
)
var nilPacer = &noopPacer{}
type limiter interface {
DelayN(now time.Time, n int) time.Duration
AllowN(now time.Time, n int) bool
Burst() int
}
// pacer is the interface for flush and compaction rate limiters. The rate limiter
// is possible applied on each iteration step of a flush or compaction. This is to
// limit background IO usage so that it does not contend with foreground traffic.
type pacer interface {
maybeThrottle(bytesIterated uint64) error
}
// deletionPacerInfo contains any info from the db necessary to make deletion
// pacing decisions.
// pacing decisions (to limit background IO usage so that it does not contend
// with foreground traffic).
type deletionPacerInfo struct {
freeBytes uint64
obsoleteBytes uint64
@@ -40,79 +24,167 @@ type deletionPacerInfo struct {
// negatively impacted if too many blocks are deleted very quickly, so this
// mechanism helps mitigate that.
type deletionPacer struct {
limiter limiter
freeSpaceThreshold uint64
obsoleteBytesMaxRatio float64
// If there are less than freeSpaceThreshold bytes of free space on
// disk, increase the pace of deletions such that we delete enough bytes to
// get back to the threshold within the freeSpaceTimeframe.
freeSpaceThreshold uint64
freeSpaceTimeframe time.Duration
// If the ratio of obsolete bytes to live bytes is greater than
// obsoleteBytesMaxRatio, increase the pace of deletions such that we delete
// enough bytes to get back to the threshold within the obsoleteBytesTimeframe.
obsoleteBytesMaxRatio float64
obsoleteBytesTimeframe time.Duration
mu struct {
sync.Mutex
// history keeps rack of recent deletion history; it used to increase the
// deletion rate to match the pace of deletions.
history history
}
targetByteDeletionRate int64
getInfo func() deletionPacerInfo
}
const deletePacerHistory = 5 * time.Minute
// newDeletionPacer instantiates a new deletionPacer for use when deleting
// obsolete files. The limiter passed in must be a singleton shared across this
// pebble instance.
func newDeletionPacer(limiter limiter, getInfo func() deletionPacerInfo) *deletionPacer {
return &deletionPacer{
limiter: limiter,
// If there are less than freeSpaceThreshold bytes of free space on
// disk, do not pace deletions at all.
// obsolete files.
//
// targetByteDeletionRate is the rate (in bytes/sec) at which we want to
// normally limit deletes (when we are not falling behind or running out of
// space). A value of 0.0 disables pacing.
func newDeletionPacer(
now time.Time, targetByteDeletionRate int64, getInfo func() deletionPacerInfo,
) *deletionPacer {
d := &deletionPacer{
freeSpaceThreshold: 16 << 30, // 16 GB
// If the ratio of obsolete bytes to live bytes is greater than
// obsoleteBytesMaxRatio, do not pace deletions at all.
obsoleteBytesMaxRatio: 0.20,
freeSpaceTimeframe: 10 * time.Second,
getInfo: getInfo,
obsoleteBytesMaxRatio: 0.20,
obsoleteBytesTimeframe: 5 * time.Minute,
targetByteDeletionRate: targetByteDeletionRate,
getInfo: getInfo,
}
d.mu.history.Init(now, deletePacerHistory)
return d
}
// ReportDeletion is used to report a deletion to the pacer. The pacer uses it
// to keep track of the recent rate of deletions and potentially increase the
// deletion rate accordingly.
//
// ReportDeletion is thread-safe.
func (p *deletionPacer) ReportDeletion(now time.Time, bytesToDelete uint64) {
p.mu.Lock()
defer p.mu.Unlock()
p.mu.history.Add(now, int64(bytesToDelete))
}
// PacingDelay returns the recommended pacing wait time (in seconds) for
// deleting the given number of bytes.
//
// PacingDelay is thread-safe.
func (p *deletionPacer) PacingDelay(now time.Time, bytesToDelete uint64) (waitSeconds float64) {
if p.targetByteDeletionRate == 0 {
// Pacing disabled.
return 0.0
}
baseRate := float64(p.targetByteDeletionRate)
// If recent deletion rate is more than our target, use that so that we don't
// fall behind.
historicRate := func() float64 {
p.mu.Lock()
defer p.mu.Unlock()
return float64(p.mu.history.Sum(now)) / deletePacerHistory.Seconds()
}()
if historicRate > baseRate {
baseRate = historicRate
}
// Apply heuristics to increase the deletion rate.
var extraRate float64
info := p.getInfo()
if info.freeBytes <= p.freeSpaceThreshold {
// Increase the rate so that we can free up enough bytes within the timeframe.
extraRate = float64(p.freeSpaceThreshold-info.freeBytes) / p.freeSpaceTimeframe.Seconds()
}
if info.liveBytes == 0 {
// We don't know the obsolete bytes ratio. Disable pacing altogether.
return 0.0
}
obsoleteBytesRatio := float64(info.obsoleteBytes) / float64(info.liveBytes)
if obsoleteBytesRatio >= p.obsoleteBytesMaxRatio {
// Increase the rate so that we can free up enough bytes within the timeframe.
r := (obsoleteBytesRatio - p.obsoleteBytesMaxRatio) * float64(info.liveBytes) / p.obsoleteBytesTimeframe.Seconds()
if extraRate < r {
extraRate = r
}
}
return float64(bytesToDelete) / (baseRate + extraRate)
}
// history is a helper used to keep track of the recent history of a set of
// data points (in our case deleted bytes), at limited granularity.
// Specifically, we split the desired timeframe into 100 "epochs" and all times
// are effectively rounded down to the nearest epoch boundary.
type history struct {
epochDuration time.Duration
startTime time.Time
// currEpoch is the epoch of the most recent operation.
currEpoch int64
// val contains the recent epoch values.
// val[currEpoch % historyEpochs] is the current epoch.
// val[(currEpoch + 1) % historyEpochs] is the oldest epoch.
val [historyEpochs]int64
// sum is always equal to the sum of values in val.
sum int64
}
const historyEpochs = 100
// Init the history helper to keep track of data over the given number of
// seconds.
func (h *history) Init(now time.Time, timeframe time.Duration) {
*h = history{
epochDuration: timeframe / time.Duration(historyEpochs),
startTime: now,
currEpoch: 0,
sum: 0,
}
}
// limit applies rate limiting if the current free disk space is more than
// freeSpaceThreshold, and the ratio of obsolete to live bytes is less than
// obsoleteBytesMaxRatio.
func (p *deletionPacer) limit(amount uint64, info deletionPacerInfo) error {
obsoleteBytesRatio := float64(1.0)
if info.liveBytes > 0 {
obsoleteBytesRatio = float64(info.obsoleteBytes) / float64(info.liveBytes)
// Add adds a value for the current time.
func (h *history) Add(now time.Time, val int64) {
h.advance(now)
h.val[h.currEpoch%historyEpochs] += val
h.sum += val
}
// Sum returns the sum of recent values. The result is approximate in that the
// cut-off time is within 1% of the exact one.
func (h *history) Sum(now time.Time) int64 {
h.advance(now)
return h.sum
}
func (h *history) epoch(t time.Time) int64 {
return int64(t.Sub(h.startTime) / h.epochDuration)
}
// advance advances the time to the given time.
func (h *history) advance(now time.Time) {
epoch := h.epoch(now)
for h.currEpoch < epoch {
h.currEpoch++
// Forget the data for the oldest epoch.
h.sum -= h.val[h.currEpoch%historyEpochs]
h.val[h.currEpoch%historyEpochs] = 0
}
paceDeletions := info.freeBytes > p.freeSpaceThreshold &&
obsoleteBytesRatio < p.obsoleteBytesMaxRatio
if paceDeletions {
burst := p.limiter.Burst()
for amount > uint64(burst) {
d := p.limiter.DelayN(time.Now(), burst)
if d == rate.InfDuration {
return errors.Errorf("pacing failed")
}
time.Sleep(d)
amount -= uint64(burst)
}
d := p.limiter.DelayN(time.Now(), int(amount))
if d == rate.InfDuration {
return errors.Errorf("pacing failed")
}
time.Sleep(d)
} else {
burst := p.limiter.Burst()
for amount > uint64(burst) {
// AllowN will subtract burst if there are enough tokens available,
// else leave the tokens untouched. That is, we are making a
// best-effort to account for this activity in the limiter, but by
// ignoring the return value, we do the activity instantaneously
// anyway.
p.limiter.AllowN(time.Now(), burst)
amount -= uint64(burst)
}
p.limiter.AllowN(time.Now(), int(amount))
}
return nil
}
// maybeThrottle slows down a deletion of this file if it's faster than
// opts.TargetByteDeletionRate.
func (p *deletionPacer) maybeThrottle(bytesToDelete uint64) error {
return p.limit(bytesToDelete, p.getInfo())
}
type noopPacer struct{}
func (p *noopPacer) maybeThrottle(_ uint64) error {
return nil
}

49
vendor/github.com/cockroachdb/pebble/range_keys.go generated vendored
View File

@@ -17,7 +17,7 @@ import (
func (i *Iterator) constructRangeKeyIter() {
i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
&i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
&i.hasPrefix, &i.prefixOrFullSeekKey, true /* onlySets */, &i.rangeKey.rangeKeyBuffers.internal)
&i.hasPrefix, &i.prefixOrFullSeekKey, false /* internalKeys */, &i.rangeKey.rangeKeyBuffers.internal)
// If there's an indexed batch with range keys, include it.
if i.batch != nil {
@@ -36,19 +36,24 @@ func (i *Iterator) constructRangeKeyIter() {
}
// Next are the flushables: memtables and large batches.
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
if i.readState != nil {
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
}
}
}
current := i.readState.current
current := i.version
if current == nil {
current = i.readState.current
}
// Next are the file levels: L0 sub-levels followed by lower levels.
//
// Add file-specific iterators for L0 files containing range keys. This is less
@@ -63,7 +68,7 @@ func (i *Iterator) constructRangeKeyIter() {
// around Key Trailer order.
iter := current.RangeKeyLevels[0].Iter()
for f := iter.Last(); f != nil; f = iter.Prev() {
spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions(manifest.Level(0)))
spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions())
if err != nil {
i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
continue
@@ -77,7 +82,7 @@ func (i *Iterator) constructRangeKeyIter() {
continue
}
li := i.rangeKey.iterConfig.NewLevelIter()
spanIterOpts := i.opts.SpanIterOptions(manifest.Level(level))
spanIterOpts := i.opts.SpanIterOptions()
li.Init(spanIterOpts, i.cmp, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
manifest.Level(level), manifest.KeyTypeRange)
i.rangeKey.iterConfig.AddLevel(li)
@@ -365,24 +370,24 @@ func (m *rangeKeyMasking) Intersects(prop []byte) (bool, error) {
// KeyIsWithinLowerBound implements the limitedBlockPropertyFilter interface
// defined in the sstable package. It's used to restrict the masking block
// property filter to only applying within the bounds of the active range key.
func (m *rangeKeyMasking) KeyIsWithinLowerBound(ik *InternalKey) bool {
func (m *rangeKeyMasking) KeyIsWithinLowerBound(key []byte) bool {
// Invariant: m.maskSpan != nil
//
// The provided `ik` is an inclusive lower bound of the block we're
// The provided `key` is an inclusive lower bound of the block we're
// considering skipping.
return m.cmp(m.maskSpan.Start, ik.UserKey) <= 0
return m.cmp(m.maskSpan.Start, key) <= 0
}
// KeyIsWithinUpperBound implements the limitedBlockPropertyFilter interface
// defined in the sstable package. It's used to restrict the masking block
// property filter to only applying within the bounds of the active range key.
func (m *rangeKeyMasking) KeyIsWithinUpperBound(ik *InternalKey) bool {
func (m *rangeKeyMasking) KeyIsWithinUpperBound(key []byte) bool {
// Invariant: m.maskSpan != nil
//
// The provided `ik` is an *inclusive* upper bound of the block we're
// The provided `key` is an *inclusive* upper bound of the block we're
// considering skipping, so the range key's end must be strictly greater
// than the block bound for the block to be within bounds.
return m.cmp(m.maskSpan.End, ik.UserKey) > 0
return m.cmp(m.maskSpan.End, key) > 0
}
// lazyCombinedIter implements the internalIterator interface, wrapping a
@@ -511,7 +516,11 @@ func (i *lazyCombinedIter) initCombinedIteration(
// Initialize the Iterator's interleaving iterator.
i.parent.rangeKey.iiter.Init(
&i.parent.comparer, i.parent.pointIter, i.parent.rangeKey.rangeKeyIter,
&i.parent.rangeKeyMasking, i.parent.opts.LowerBound, i.parent.opts.UpperBound)
keyspan.InterleavingIterOpts{
Mask: &i.parent.rangeKeyMasking,
LowerBound: i.parent.opts.LowerBound,
UpperBound: i.parent.opts.UpperBound,
})
// Set the parent's primary iterator to point to the combined, interleaving
// iterator that's now initialized with our current state.

33
vendor/github.com/cockroachdb/pebble/rangekey/rangekey.go generated vendored Normal file
View File

@@ -0,0 +1,33 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package rangekey provides functionality for working with range keys.
package rangekey
import (
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/rangekey"
"github.com/cockroachdb/pebble/sstable"
)
// Fragmenter exports the keyspan.Fragmenter type.
type Fragmenter = keyspan.Fragmenter
// Key exports the keyspan.Key type.
type Key = keyspan.Key
// Span exports the keyspan.Span type.
type Span = keyspan.Span
// IsRangeKey returns if this InternalKey is a range key. Alias for
// rangekey.IsRangeKey.
func IsRangeKey(ik sstable.InternalKey) bool {
return rangekey.IsRangeKey(ik.Kind())
}
// Decode decodes an InternalKey into a keyspan.Span, if it is a range key. If
// keysDst is provided, keys will be appended to keysDst to reduce allocations.
func Decode(ik sstable.InternalKey, val []byte, keysDst []keyspan.Key) (Span, error) {
return rangekey.Decode(ik, val, keysDst)
}

20
vendor/github.com/cockroachdb/pebble/read_state.go generated vendored
View File

@@ -20,14 +20,14 @@ import "sync/atomic"
// in Go.
type readState struct {
db *DB
refcnt int32
refcnt atomic.Int32
current *version
memtables flushableList
}
// ref adds a reference to the readState.
func (s *readState) ref() {
atomic.AddInt32(&s.refcnt, 1)
s.refcnt.Add(1)
}
// unref removes a reference to the readState. If this was the last reference,
@@ -35,7 +35,7 @@ func (s *readState) ref() {
// is NOT held as version.unref() will acquire it. See unrefLocked() if DB.mu
// is held by the caller.
func (s *readState) unref() {
if atomic.AddInt32(&s.refcnt, -1) != 0 {
if s.refcnt.Add(-1) != 0 {
return
}
s.current.Unref()
@@ -50,10 +50,11 @@ func (s *readState) unref() {
// unrefLocked removes a reference to the readState. If this was the last
// reference, the reference the readState holds on the version is
// released. Requires DB.mu is held as version.unrefLocked() requires it. See
// unref() if DB.mu is NOT held by the caller.
// released.
//
// DB.mu must be held. See unref() if DB.mu is NOT held by the caller.
func (s *readState) unrefLocked() {
if atomic.AddInt32(&s.refcnt, -1) != 0 {
if s.refcnt.Add(-1) != 0 {
return
}
s.current.UnrefLocked()
@@ -61,9 +62,8 @@ func (s *readState) unrefLocked() {
mem.readerUnrefLocked(true)
}
// NB: Unlike readState.unref(), we don't attempt to cleanup newly obsolete
// tables as unrefLocked() is only called during DB shutdown to release the
// current readState.
// In this code path, the caller is responsible for scheduling obsolete table
// deletion as necessary.
}
// loadReadState returns the current readState. The returned readState must be
@@ -82,10 +82,10 @@ func (d *DB) loadReadState() *readState {
func (d *DB) updateReadStateLocked(checker func(*DB) error) {
s := &readState{
db: d,
refcnt: 1,
current: d.mu.versions.currentVersion(),
memtables: d.mu.mem.queue,
}
s.refcnt.Store(1)
s.current.Ref()
for _, mem := range s.memtables {
mem.readerRef()

149
vendor/github.com/cockroachdb/pebble/record/log_writer.go generated vendored
View File

@@ -20,10 +20,11 @@ import (
)
var walSyncLabels = pprof.Labels("pebble", "wal-sync")
var errClosedWriter = errors.New("pebble/record: closed LogWriter")
type block struct {
// buf[:written] has already been filled with fragments. Updated atomically.
written int32
written atomic.Int32
// buf[:flushed] has already been flushed to w.
flushed int32
buf [blockSize]byte
@@ -38,7 +39,7 @@ type syncer interface {
}
const (
syncConcurrencyBits = 9
syncConcurrencyBits = 12
// SyncConcurrency is the maximum number of concurrent sync operations that
// can be performed. Note that a sync operation is initiated either by a call
@@ -69,7 +70,7 @@ type syncQueue struct {
//
// The head index is stored in the most-significant bits so that we can
// atomically add to it and the overflow is harmless.
headTail uint64
headTail atomic.Uint64
// slots is a ring buffer of values stored in this queue. The size must be a
// power of 2. A slot is in use until the tail index has moved beyond it.
@@ -78,7 +79,7 @@ type syncQueue struct {
// blocked is an atomic boolean which indicates whether syncing is currently
// blocked or can proceed. It is used by the implementation of
// min-sync-interval to block syncing until the min interval has passed.
blocked uint32
blocked atomic.Bool
}
const dequeueBits = 32
@@ -91,7 +92,7 @@ func (q *syncQueue) unpack(ptrs uint64) (head, tail uint32) {
}
func (q *syncQueue) push(wg *sync.WaitGroup, err *error) {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
panic("pebble: queue is full")
@@ -103,15 +104,15 @@ func (q *syncQueue) push(wg *sync.WaitGroup, err *error) {
// Increment head. This passes ownership of slot to dequeue and acts as a
// store barrier for writing the slot.
atomic.AddUint64(&q.headTail, 1<<dequeueBits)
q.headTail.Add(1 << dequeueBits)
}
func (q *syncQueue) setBlocked() {
atomic.StoreUint32(&q.blocked, 1)
q.blocked.Store(true)
}
func (q *syncQueue) clearBlocked() {
atomic.StoreUint32(&q.blocked, 0)
q.blocked.Store(false)
}
func (q *syncQueue) empty() bool {
@@ -124,10 +125,10 @@ func (q *syncQueue) empty() bool {
// min-sync-interval. It additionally returns the real length of this queue,
// regardless of whether syncing is blocked.
func (q *syncQueue) load() (head, tail, realLength uint32) {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail = q.unpack(ptrs)
realLength = head - tail
if atomic.LoadUint32(&q.blocked) == 1 {
if q.blocked.Load() {
return 0, 0, realLength
}
return head, tail, realLength
@@ -152,7 +153,7 @@ func (q *syncQueue) pop(head, tail uint32, err error, queueSemChan chan struct{}
// We need to bump the tail count before signalling the wait group as
// signalling the wait group can trigger release a blocked goroutine which
// will try to enqueue before we've "freed" space in the queue.
atomic.AddUint64(&q.headTail, 1)
q.headTail.Add(1)
wg.Done()
// Is always non-nil in production.
if queueSemChan != nil {
@@ -265,10 +266,7 @@ type LogWriter struct {
block *block
free struct {
sync.Mutex
// Condition variable used to signal a block is freed.
cond sync.Cond
blocks []*block
allocated int
blocks []*block
}
flusher struct {
@@ -312,9 +310,17 @@ type LogWriterConfig struct {
QueueSemChan chan struct{}
}
// CapAllocatedBlocks is the maximum number of blocks allocated by the
// LogWriter.
const CapAllocatedBlocks = 16
// initialAllocatedBlocksCap is the initial capacity of the various slices
// intended to hold LogWriter blocks. The LogWriter may allocate more blocks
// than this threshold allows.
const initialAllocatedBlocksCap = 32
// blockPool pools *blocks to avoid allocations. Blocks are only added to the
// Pool when a LogWriter is closed. Before that, free blocks are maintained
// within a LogWriter's own internal free list `w.free.blocks`.
var blockPool = sync.Pool{
New: func() any { return &block{} },
}
// NewLogWriter returns a new LogWriter.
func NewLogWriter(w io.Writer, logNum base.FileNum, logWriterConfig LogWriterConfig) *LogWriter {
@@ -334,10 +340,8 @@ func NewLogWriter(w io.Writer, logNum base.FileNum, logWriterConfig LogWriterCon
},
queueSemChan: logWriterConfig.QueueSemChan,
}
r.free.cond.L = &r.free.Mutex
r.free.blocks = make([]*block, 0, CapAllocatedBlocks)
r.free.allocated = 1
r.block = &block{}
r.free.blocks = make([]*block, 0, initialAllocatedBlocksCap)
r.block = blockPool.Get().(*block)
r.flusher.ready.init(&r.flusher.Mutex, &r.flusher.syncQ)
r.flusher.closed = make(chan struct{})
r.flusher.pending = make([]*block, 0, cap(r.free.blocks))
@@ -401,18 +405,22 @@ func (w *LogWriter) flushLoop(context.Context) {
// before syncing work. The guarantee of this code is that when a sync is
// requested, any previously queued flush work will be synced. This
// motivates reading the syncing work (f.syncQ.load()) before picking up
// the flush work (atomic.LoadInt32(&w.block.written)).
// the flush work (w.block.written.Load()).
// The list of full blocks that need to be written. This is copied from
// f.pending on every loop iteration, though the number of elements is small
// (usually 1, max 16).
// f.pending on every loop iteration, though the number of elements is
// usually small (most frequently 1). In the case of the WAL LogWriter, the
// number of blocks is bounded by the size of the WAL's corresponding
// memtable (MemtableSize/BlockSize). With the default 64 MiB memtables,
// this works out to at most 2048 elements if the entirety of the memtable's
// contents are queued.
pending := make([]*block, 0, cap(f.pending))
for {
for {
// Grab the portion of the current block that requires flushing. Note that
// the current block can be added to the pending blocks list after we release
// the flusher lock, but it won't be part of pending.
written := atomic.LoadInt32(&w.block.written)
written := w.block.written.Load()
if len(f.pending) > 0 || written > w.block.flushed || !f.syncQ.empty() {
break
}
@@ -431,8 +439,7 @@ func (w *LogWriter) flushLoop(context.Context) {
// Found work to do, so no longer idle.
workStartTime := time.Now()
idleDuration := workStartTime.Sub(idleStartTime)
pending = pending[:len(f.pending)]
copy(pending, f.pending)
pending = append(pending[:0], f.pending...)
f.pending = f.pending[:0]
f.metrics.PendingBufferLen.AddSample(int64(len(pending)))
@@ -448,7 +455,7 @@ func (w *LogWriter) flushLoop(context.Context) {
// be ordered after we get the list of sync waiters from syncQ in order to
// prevent a race where a waiter adds itself to syncQ, but this thread
// picks up the entry in syncQ and not the buffered data.
written := atomic.LoadInt32(&w.block.written)
written := w.block.written.Load()
data := w.block.buf[w.block.flushed:written]
w.block.flushed = written
@@ -551,32 +558,22 @@ func (w *LogWriter) flushBlock(b *block) error {
if _, err := w.w.Write(b.buf[b.flushed:]); err != nil {
return err
}
b.written = 0
b.written.Store(0)
b.flushed = 0
w.free.Lock()
w.free.blocks = append(w.free.blocks, b)
w.free.cond.Signal()
w.free.Unlock()
return nil
}
// queueBlock queues the current block for writing to the underlying writer,
// allocates a new block and reserves space for the next header.
func (w *LogWriter) queueBlock() (waitDuration time.Duration) {
func (w *LogWriter) queueBlock() {
// Allocate a new block, blocking until one is available. We do this first
// because w.block is protected by w.flusher.Mutex.
w.free.Lock()
if len(w.free.blocks) == 0 {
if w.free.allocated < cap(w.free.blocks) {
w.free.allocated++
w.free.blocks = append(w.free.blocks, &block{})
} else {
now := time.Now()
for len(w.free.blocks) == 0 {
w.free.cond.Wait()
}
waitDuration = time.Since(now)
}
w.free.blocks = append(w.free.blocks, blockPool.Get().(*block))
}
nextBlock := w.free.blocks[len(w.free.blocks)-1]
w.free.blocks = w.free.blocks[:len(w.free.blocks)-1]
@@ -591,28 +588,6 @@ func (w *LogWriter) queueBlock() (waitDuration time.Duration) {
f.Unlock()
w.blockNum++
return waitDuration
}
// ReserveAllFreeBlocksForTesting is used to only for testing.
func (w *LogWriter) ReserveAllFreeBlocksForTesting() (releaseFunc func()) {
w.free.Lock()
defer w.free.Unlock()
free := w.free.blocks
w.free.blocks = nil
return func() {
w.free.Lock()
defer w.free.Unlock()
// It is possible that someone has pushed a free block and w.free.blocks
// is no longer nil. That is harmless. Also, the waiter loops on the
// condition len(w.free.blocks) == 0, so to actually unblock it we need to
// give it a free block.
if len(free) == 0 {
free = append(free, &block{})
}
w.free.blocks = free
w.free.cond.Broadcast()
}
}
// Close flushes and syncs any unwritten data and closes the writer.
@@ -647,6 +622,7 @@ func (w *LogWriter) Close() error {
if f.fsyncLatency != nil {
f.fsyncLatency.Observe(float64(syncLatency))
}
free := w.free.blocks
f.Unlock()
if w.c != nil {
@@ -656,7 +632,14 @@ func (w *LogWriter) Close() error {
return cerr
}
}
w.err = errors.New("pebble/record: closed LogWriter")
for _, b := range free {
b.flushed = 0
b.written.Store(0)
blockPool.Put(b)
}
w.err = errClosedWriter
return err
}
@@ -664,20 +647,20 @@ func (w *LogWriter) Close() error {
// of the record.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) WriteRecord(p []byte) (int64, error) {
logSize, _, err := w.SyncRecord(p, nil, nil)
logSize, err := w.SyncRecord(p, nil, nil)
return logSize, err
}
// SyncRecord writes a complete record. If wg!= nil the record will be
// SyncRecord writes a complete record. If wg != nil the record will be
// asynchronously persisted to the underlying writer and done will be called on
// the wait group upon completion. Returns the offset just past the end of the
// record.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) SyncRecord(
p []byte, wg *sync.WaitGroup, err *error,
) (logSize int64, waitDuration time.Duration, err2 error) {
) (logSize int64, err2 error) {
if w.err != nil {
return -1, 0, w.err
return -1, w.err
}
// The `i == 0` condition ensures we handle empty records. Such records can
@@ -685,9 +668,7 @@ func (w *LogWriter) SyncRecord(
// MANIFEST is currently written using Writer, it is good to support the same
// semantics with LogWriter.
for i := 0; i == 0 || len(p) > 0; i++ {
var wd time.Duration
p, wd = w.emitFragment(i, p)
waitDuration += wd
p = w.emitFragment(i, p)
}
if wg != nil {
@@ -701,35 +682,35 @@ func (w *LogWriter) SyncRecord(
f.ready.Signal()
}
offset := w.blockNum*blockSize + int64(w.block.written)
offset := w.blockNum*blockSize + int64(w.block.written.Load())
// Note that we don't return w.err here as a concurrent call to Close would
// race with our read. That's ok because the only error we could be seeing is
// one to syncing for which the caller can receive notification of by passing
// in a non-nil err argument.
return offset, waitDuration, nil
return offset, nil
}
// Size returns the current size of the file.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) Size() int64 {
return w.blockNum*blockSize + int64(w.block.written)
return w.blockNum*blockSize + int64(w.block.written.Load())
}
func (w *LogWriter) emitEOFTrailer() {
// Write a recyclable chunk header with a different log number. Readers
// will treat the header as EOF when the log number does not match.
b := w.block
i := b.written
i := b.written.Load()
binary.LittleEndian.PutUint32(b.buf[i+0:i+4], 0) // CRC
binary.LittleEndian.PutUint16(b.buf[i+4:i+6], 0) // Size
b.buf[i+6] = recyclableFullChunkType
binary.LittleEndian.PutUint32(b.buf[i+7:i+11], w.logNum+1) // Log number
atomic.StoreInt32(&b.written, i+int32(recyclableHeaderSize))
b.written.Store(i + int32(recyclableHeaderSize))
}
func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDuration time.Duration) {
func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte) {
b := w.block
i := b.written
i := b.written.Load()
first := n == 0
last := blockSize-i-recyclableHeaderSize >= int32(len(p))
@@ -753,17 +734,17 @@ func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDurati
j := i + int32(recyclableHeaderSize+r)
binary.LittleEndian.PutUint32(b.buf[i+0:i+4], crc.New(b.buf[i+6:j]).Value())
binary.LittleEndian.PutUint16(b.buf[i+4:i+6], uint16(r))
atomic.StoreInt32(&b.written, j)
b.written.Store(j)
if blockSize-b.written < recyclableHeaderSize {
if blockSize-b.written.Load() < recyclableHeaderSize {
// There is no room for another fragment in the block, so fill the
// remaining bytes with zeros and queue the block for flushing.
for i := b.written; i < blockSize; i++ {
for i := b.written.Load(); i < blockSize; i++ {
b.buf[i] = 0
}
waitDuration = w.queueBlock()
w.queueBlock()
}
return p[r:], waitDuration
return p[r:]
}
// Metrics must be called after Close. The callee will no longer modify the

611
vendor/github.com/cockroachdb/pebble/scan_internal.go generated vendored
View File

@@ -7,7 +7,6 @@ package pebble
import (
"context"
"fmt"
"sync"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
@@ -15,25 +14,26 @@ import (
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/objstorage/remote"
)
const (
// In skip-shared iteration mode, keys in levels sharedLevelsStart and greater
// (i.e. lower in the LSM) are skipped.
sharedLevelsStart = 5
sharedLevelsStart = remote.SharedLevelsStart
)
// ErrInvalidSkipSharedIteration is returned by ScanInternal if it was called
// with a shared file visitor function, and a file in a shareable level (i.e.
// level >= sharedLevelsStart) was found to not be in shared storage according
// to objstorage.Provider.
var ErrInvalidSkipSharedIteration = errors.New("pebble: cannot use skip-shared iteration due to non-shared files in lower levels")
// to objstorage.Provider, or not shareable for another reason such as for
// containing keys newer than the snapshot sequence number.
var ErrInvalidSkipSharedIteration = errors.New("pebble: cannot use skip-shared iteration due to non-shareable files in lower levels")
// SharedSSTMeta represents an sstable on shared storage that can be ingested
// by another pebble instance. This struct must contain all fields that are
// required for a Pebble instance to ingest a foreign sstable on shared storage,
// including constructing any relevant objstorage.Provider / sharedobjcat.Catalog
// including constructing any relevant objstorage.Provider / remoteobjcat.Catalog
// data structures, as well as creating virtual FileMetadatas.
//
// Note that the Pebble instance creating and returning a SharedSSTMeta might
@@ -43,7 +43,7 @@ var ErrInvalidSkipSharedIteration = errors.New("pebble: cannot use skip-shared i
type SharedSSTMeta struct {
// Backing is the shared object underlying this SST. Can be attached to an
// objstorage.Provider.
Backing objstorage.SharedObjectBackingHandle
Backing objstorage.RemoteObjectBackingHandle
// Smallest and Largest internal keys for the overall bounds. The kind and
// SeqNum of these will reflect what is physically present on the source Pebble
@@ -90,57 +90,25 @@ func (s *SharedSSTMeta) cloneFromFileMeta(f *fileMetadata) {
}
}
type sharedByLevel []SharedSSTMeta
func (s sharedByLevel) Len() int { return len(s) }
func (s sharedByLevel) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s sharedByLevel) Less(i, j int) bool { return s[i].Level < s[j].Level }
type pcIterPos int
const (
pcIterPosCur pcIterPos = iota
pcIterPosNext
pcIterPosPrev
)
// pointCollapsingSSTIterator implements sstable.Iterator while composing
// pointCollapsingIterator.
type pointCollapsingSSTIterator struct {
pointCollapsingIterator
childIter sstable.Iterator
}
var pcSSTIterPool = sync.Pool{
New: func() interface{} {
return &pointCollapsingSSTIterator{}
},
}
// MaybeFilteredKeys implements the sstable.Iterator interface.
func (p *pointCollapsingSSTIterator) MaybeFilteredKeys() bool {
return p.childIter.MaybeFilteredKeys()
}
// SetCloseHook implements the sstable.Iterator interface.
func (p *pointCollapsingSSTIterator) SetCloseHook(fn func(i sstable.Iterator) error) {
p.childIter.SetCloseHook(fn)
}
// Close implements the sstable.Iterator interface.
func (p *pointCollapsingSSTIterator) Close() error {
err := p.pointCollapsingIterator.Close()
p.pointCollapsingIterator = pointCollapsingIterator{}
p.childIter = nil
pcSSTIterPool.Put(p)
return err
}
// pointCollapsingIterator is an internalIterator that collapses point keys and
// returns at most one point internal key for each user key. Merges are merged,
// sets are emitted as-is, and SingleDeletes are collapsed with the next point
// key, however we don't guarantee that we generate and return a SetWithDel if
// a set shadows a del. Point keys deleted by rangedels are also considered shadowed and not
// exposed.
// returns at most one point internal key for each user key. Merges and
// SingleDels are not supported and result in a panic if encountered. Point keys
// deleted by rangedels are considered shadowed and not exposed.
//
// TODO(bilal): Implement the unimplemented internalIterator methods below.
// Currently this iterator only supports the forward iteration case necessary
// for scanInternal, however foreign sstable iterators will also need to use this
// or a simplified version of this.
// Only used in ScanInternal to return at most one internal key per user key.
type pointCollapsingIterator struct {
iter keyspan.InterleavingIter
pos pcIterPos
@@ -161,28 +129,19 @@ type pointCollapsingIterator struct {
// - If pos == pcIterPosCur, iterKey is pointing to an `iter`-owned current
// key, and savedKey is either undefined or pointing to a version of the
// current key owned by this iterator (i.e. backed by savedKeyBuf).
// - if pos == pcIterPosPrev, iterKey is pointing to the key before
// p.savedKey. p.savedKey is treated as the current key and is owned by
// this iterator, while p.iterKey is the previous key that is the current
// position of the child iterator.
savedKey InternalKey
savedKeyBuf []byte
// elideRangeDeletes ignores range deletes returned by the interleaving
// iterator if true.
elideRangeDeletes bool
// Value at the current iterator position, at iterKey.
iterValue base.LazyValue
// Saved value backed by valueBuf, if set. Used in reverse iteration, and
// for merges.
savedValue base.LazyValue
// Used for Merge keys only.
valueMerger ValueMerger
valueBuf []byte
// If fixedSeqNum is non-zero, all emitted points are verified to have this
// fixed sequence number.
fixedSeqNum uint64
}
func (p *pointCollapsingIterator) Span() *keyspan.Span {
return p.iter.Span()
}
// SeekPrefixGE implements the InternalIterator interface.
func (p *pointCollapsingIterator) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
@@ -213,20 +172,12 @@ func (p *pointCollapsingIterator) SeekGE(
func (p *pointCollapsingIterator) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*base.InternalKey, base.LazyValue) {
p.resetKey()
p.iterKey, p.iterValue = p.iter.SeekLT(key, flags)
p.pos = pcIterPosCur
if p.iterKey == nil {
return nil, base.LazyValue{}
}
return p.findPrevEntry()
panic("unimplemented")
}
func (p *pointCollapsingIterator) resetKey() {
p.savedKey.UserKey = p.savedKeyBuf[:0]
p.savedKey.Trailer = 0
p.valueMerger = nil
p.valueBuf = p.valueBuf[:0]
p.iterKey = nil
p.pos = pcIterPosCur
}
@@ -244,22 +195,6 @@ func (p *pointCollapsingIterator) verifySeqNum(key *base.InternalKey) *base.Inte
return key
}
// finishAndReturnMerge finishes off the valueMerger and returns the saved key.
func (p *pointCollapsingIterator) finishAndReturnMerge() (*base.InternalKey, base.LazyValue) {
value, closer, err := p.valueMerger.Finish(true /* includesBase */)
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
p.valueBuf = append(p.valueBuf[:0], value...)
if closer != nil {
_ = closer.Close()
}
p.valueMerger = nil
val := base.MakeInPlaceValue(p.valueBuf)
return p.verifySeqNum(&p.savedKey), val
}
// findNextEntry is called to return the next key. p.iter must be positioned at the
// start of the first user key we are interested in.
func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyValue) {
@@ -270,22 +205,13 @@ func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyV
// NB: p.savedKey is either the current key (iff p.iterKey == firstKey),
// or the previous key.
if !firstIteration && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
if p.valueMerger != nil {
if p.savedKey.Kind() != InternalKeyKindMerge {
panic(fmt.Sprintf("expected key %s to have MERGE kind", p.iterKey))
}
p.pos = pcIterPosNext
return p.finishAndReturnMerge()
}
p.saveKey()
continue
}
firstIteration = false
if s := p.iter.Span(); s != nil && s.CoversAt(p.seqNum, p.iterKey.SeqNum()) {
// All future keys for this user key must be deleted.
if p.valueMerger != nil {
return p.finishAndReturnMerge()
} else if p.savedKey.Kind() == InternalKeyKindSingleDelete {
if p.savedKey.Kind() == InternalKeyKindSingleDelete {
panic("cannot process singledel key in point collapsing iterator")
}
// Fast forward to the next user key.
@@ -321,58 +247,9 @@ func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyV
// Panic, as this iterator is not expected to observe single deletes.
panic("cannot process singledel key in point collapsing iterator")
case InternalKeyKindMerge:
if p.valueMerger == nil {
// Set up merger. This is the first Merge key encountered.
value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
if !callerOwned {
p.valueBuf = append(p.valueBuf[:0], value...)
} else {
p.valueBuf = value
}
p.valueMerger, err = p.merge(p.iterKey.UserKey, p.valueBuf)
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
p.saveKey()
p.iterKey, p.iterValue = p.iter.Next()
continue
}
switch p.iterKey.Kind() {
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// Merge into key.
value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
if !callerOwned {
p.valueBuf = append(p.valueBuf[:0], value...)
} else {
p.valueBuf = value
}
if err := p.valueMerger.MergeOlder(value); err != nil {
p.err = err
return nil, base.LazyValue{}
}
}
if p.iterKey.Kind() != InternalKeyKindMerge {
p.savedKey.SetKind(p.iterKey.Kind())
p.pos = pcIterPosCur
return p.finishAndReturnMerge()
}
p.iterKey, p.iterValue = p.iter.Next()
// Panic, as this iterator is not expected to observe merges.
panic("cannot process merge key in point collapsing iterator")
case InternalKeyKindRangeDelete:
if p.elideRangeDeletes {
// Skip this range delete, and process any point after it.
p.iterKey, p.iterValue = p.iter.Next()
p.saveKey()
continue
}
// These are interleaved by the interleaving iterator ahead of all points.
// We should pass them as-is, but also account for any points ahead of
// them.
@@ -382,121 +259,10 @@ func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyV
panic(fmt.Sprintf("unexpected kind: %d", p.iterKey.Kind()))
}
}
if p.valueMerger != nil {
p.pos = pcIterPosNext
return p.finishAndReturnMerge()
}
p.resetKey()
return nil, base.LazyValue{}
}
// findPrevEntry finds the relevant point key to return for the previous user key
// (i.e. in reverse iteration). Requires that the iterator is already positioned
// at the first-in-reverse (i.e. rightmost / largest) internal key encountered
// for that user key.
func (p *pointCollapsingIterator) findPrevEntry() (*base.InternalKey, base.LazyValue) {
if p.iterKey == nil {
p.pos = pcIterPosCur
return nil, base.LazyValue{}
}
p.saveKey()
firstIteration := true
for p.iterKey != nil {
if !firstIteration && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
p.pos = pcIterPosPrev
return p.verifySeqNum(&p.savedKey), p.savedValue
}
firstIteration = false
if s := p.iter.Span(); s != nil && s.CoversAt(p.seqNum, p.iterKey.SeqNum()) {
// Skip this key.
p.iterKey, p.iterValue = p.iter.Prev()
p.saveKey()
continue
}
switch p.iterKey.Kind() {
case InternalKeyKindSet, InternalKeyKindDelete, InternalKeyKindSetWithDelete:
// Instead of calling saveKey(), we take advantage of the invariant that
// p.savedKey.UserKey == p.iterKey.UserKey (otherwise we'd have gone into
// the !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) case at
// the top of this for loop). That allows us to just save the trailer
// and move on.
if invariants.Enabled && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
panic("unexpected inequality between p.iterKey and p.savedKey")
}
p.savedKey.Trailer = p.iterKey.Trailer
// Copy value into p.savedValue.
value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
if !callerOwned {
p.valueBuf = append(p.valueBuf[:0], value...)
} else {
p.valueBuf = value
}
p.valueMerger = nil
p.savedValue = base.MakeInPlaceValue(p.valueBuf)
p.iterKey, p.iterValue = p.iter.Prev()
continue
case InternalKeyKindSingleDelete:
// Panic, as this iterator is not expected to observe single deletes.
panic("cannot process singledel key in point collapsing iterator")
case InternalKeyKindMerge:
panic("cannot process merge key in point collapsing iterator in reverse iteration")
case InternalKeyKindRangeDelete:
if p.elideRangeDeletes {
// Skip this range delete, and process any point before it.
p.iterKey, p.iterValue = p.iter.Prev()
continue
}
// These are interleaved by the interleaving iterator behind all points.
if p.savedKey.Kind() != InternalKeyKindRangeDelete {
// If the previous key was not a rangedel, we need to return it. Pretend that we're at the
// previous user key (i.e. with p.pos = pcIterPosPrev) even if we're not, so on the next
// Prev() we encounter and return this rangedel. For now return the point ahead of
// this range del (if any).
p.pos = pcIterPosPrev
return p.verifySeqNum(&p.savedKey), p.savedValue
}
// We take advantage of the fact that a Prev() *on* a RangeDel iterKey
// always takes us to a different user key, so on the next iteration
// we will fall into the !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey)
// case.
//
// Instead of calling saveKey(), we take advantage of the invariant that
// p.savedKey.UserKey == p.iterKey.UserKey (otherwise we'd have gone into
// the !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) case at
// the top of this for loop). That allows us to just save the trailer
// and move on.
if invariants.Enabled && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) {
panic("unexpected inequality between p.iterKey and p.savedKey")
}
p.savedKey.Trailer = p.iterKey.Trailer
// Copy value into p.savedValue.
value, callerOwned, err := p.iterValue.Value(p.valueBuf[:0])
if err != nil {
p.err = err
return nil, base.LazyValue{}
}
if !callerOwned {
p.valueBuf = append(p.valueBuf[:0], value...)
} else {
p.valueBuf = value
}
p.valueMerger = nil
p.savedValue = base.MakeInPlaceValue(p.valueBuf)
p.iterKey, p.iterValue = p.iter.Prev()
continue
default:
panic(fmt.Sprintf("unexpected kind: %d", p.iterKey.Kind()))
}
}
p.pos = pcIterPosPrev
return p.verifySeqNum(&p.savedKey), p.savedValue
}
// First implements the InternalIterator interface.
func (p *pointCollapsingIterator) First() (*base.InternalKey, base.LazyValue) {
p.resetKey()
@@ -510,13 +276,7 @@ func (p *pointCollapsingIterator) First() (*base.InternalKey, base.LazyValue) {
// Last implements the InternalIterator interface.
func (p *pointCollapsingIterator) Last() (*base.InternalKey, base.LazyValue) {
p.resetKey()
p.iterKey, p.iterValue = p.iter.Last()
p.pos = pcIterPosCur
if p.iterKey == nil {
return nil, base.LazyValue{}
}
return p.findPrevEntry()
panic("unimplemented")
}
func (p *pointCollapsingIterator) saveKey() {
@@ -531,33 +291,9 @@ func (p *pointCollapsingIterator) saveKey() {
// Next implements the InternalIterator interface.
func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) {
switch p.pos {
case pcIterPosPrev:
p.saveKey()
if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
p.iterKey, p.iterValue = p.iter.Next()
p.pos = pcIterPosCur
} else {
// Fast forward to the next user key.
key, val := p.iter.Next()
// p.iterKey.SeqNum() >= key.SeqNum() is an optimization that allows us to
// use p.iterKey.SeqNum() < key.SeqNum() as a sign that the user key has
// changed, without needing to do the full key comparison.
for key != nil && p.savedKey.SeqNum() >= key.SeqNum() &&
p.comparer.Equal(p.savedKey.UserKey, key.UserKey) {
key, val = p.iter.Next()
}
if key == nil {
// There are no keys to return.
p.resetKey()
return nil, base.LazyValue{}
}
p.iterKey, p.iterValue = key, val
p.pos = pcIterPosCur
}
fallthrough
case pcIterPosCur:
p.saveKey()
if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete {
// Step over the interleaved range delete and process the very next
// internal key, even if it's at the same user key. This is because a
// point for that user key has not been returned yet.
@@ -591,56 +327,12 @@ func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) {
// NextPrefix implements the InternalIterator interface.
func (p *pointCollapsingIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
// TODO(bilal): Implement this. It'll be similar to SeekGE, except we'll call
// the child iterator's NextPrefix, and have some special logic in case pos
// is pcIterPosNext.
panic("unimplemented")
}
// Prev implements the InternalIterator interface.
func (p *pointCollapsingIterator) Prev() (*base.InternalKey, base.LazyValue) {
switch p.pos {
case pcIterPosNext:
// Rewind backwards to the previous iter key.
p.saveKey()
key, val := p.iter.Prev()
for key != nil && p.savedKey.SeqNum() <= key.SeqNum() &&
p.comparer.Equal(p.savedKey.UserKey, key.UserKey) {
if key.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
// We need to pause at this range delete and return it as-is, as "cur"
// is referencing the point key after it, not the range delete.
break
}
key, val = p.iter.Prev()
}
p.iterKey = key
p.iterValue = val
p.pos = pcIterPosCur
fallthrough
case pcIterPosCur:
p.saveKey()
key, val := p.iter.Prev()
for key != nil && p.savedKey.SeqNum() <= key.SeqNum() &&
p.comparer.Equal(p.savedKey.UserKey, key.UserKey) {
if key.Kind() == InternalKeyKindRangeDelete && !p.elideRangeDeletes {
// We need to pause at this range delete and return it as-is, as "cur"
// is referencing the point key after it, not the range delete.
break
}
key, val = p.iter.Prev()
}
p.iterKey = key
p.iterValue = val
p.pos = pcIterPosCur
case pcIterPosPrev:
// Do nothing.
p.pos = pcIterPosCur
}
if p.iterKey == nil {
p.resetKey()
return nil, base.LazyValue{}
}
return p.findPrevEntry()
panic("unimplemented")
}
// Error implements the InternalIterator interface.
@@ -669,6 +361,32 @@ func (p *pointCollapsingIterator) String() string {
var _ internalIterator = &pointCollapsingIterator{}
// IteratorLevelKind is used to denote whether the current ScanInternal iterator
// is unknown, belongs to a flushable, or belongs to an LSM level type.
type IteratorLevelKind int8
const (
// IteratorLevelUnknown indicates an unknown LSM level.
IteratorLevelUnknown IteratorLevelKind = iota
// IteratorLevelLSM indicates an LSM level.
IteratorLevelLSM
// IteratorLevelFlushable indicates a flushable (i.e. memtable).
IteratorLevelFlushable
)
// IteratorLevel is used with scanInternalIterator to surface additional iterator-specific info where possible.
// Note: this is struct is only provided for point keys.
type IteratorLevel struct {
Kind IteratorLevelKind
// FlushableIndex indicates the position within the flushable queue of this level.
// Only valid if kind == IteratorLevelFlushable.
FlushableIndex int
// The level within the LSM. Only valid if Kind == IteratorLevelLSM.
Level int
// Sublevel is only valid if Kind == IteratorLevelLSM and Level == 0.
Sublevel int
}
// scanInternalIterator is an iterator that returns all internal keys, including
// tombstones. For instance, an InternalKeyKindDelete would be returned as an
// InternalKeyKindDelete instead of the iterator skipping over to the next key.
@@ -682,19 +400,23 @@ var _ internalIterator = &pointCollapsingIterator{}
// *must* return the range delete as well as the range key unset/delete that did
// the shadowing.
type scanInternalIterator struct {
db *DB
opts scanInternalOptions
comparer *base.Comparer
merge Merge
iter internalIterator
readState *readState
version *version
rangeKey *iteratorRangeKeyState
pointKeyIter pointCollapsingIterator
pointKeyIter internalIterator
iterKey *InternalKey
iterValue LazyValue
alloc *iterAlloc
newIters tableNewIters
newIterRangeKey keyspan.TableNewSpanIter
seqNum uint64
iterLevels []IteratorLevel
mergingIter *mergingIter
// boundsBuf holds two buffers used to store the lower and upper bounds.
// Whenever the InternalIterator's bounds change, the new bounds are copied
@@ -723,7 +445,7 @@ func (d *DB) truncateSharedFile(
sst = &SharedSSTMeta{}
sst.cloneFromFileMeta(file)
sst.Level = uint8(level)
sst.Backing, err = d.objProvider.SharedObjectBacking(&objMeta)
sst.Backing, err = d.objProvider.RemoteObjectBacking(&objMeta)
if err != nil {
return nil, false, err
}
@@ -748,18 +470,18 @@ func (d *DB) truncateSharedFile(
if rangeDelIter != nil {
rangeDelIter = keyspan.Truncate(
cmp, rangeDelIter, lower, upper, nil, nil,
false, /* panicOnPartialOverlap */
false, /* panicOnUpperTruncate */
)
defer rangeDelIter.Close()
}
rangeKeyIter, err := d.tableNewRangeKeyIter(file, keyspan.SpanIterOptions{Level: manifest.Level(level)})
rangeKeyIter, err := d.tableNewRangeKeyIter(file, keyspan.SpanIterOptions{})
if err != nil {
return nil, false, err
}
if rangeKeyIter != nil {
rangeKeyIter = keyspan.Truncate(
cmp, rangeKeyIter, lower, upper, nil, nil,
false, /* panicOnPartialOverlap */
false, /* panicOnUpperTruncate */
)
defer rangeKeyIter.Close()
}
@@ -769,6 +491,7 @@ func (d *DB) truncateSharedFile(
sst.SmallestPointKey.UserKey = sst.SmallestPointKey.UserKey[:0]
sst.SmallestPointKey.Trailer = 0
key, _ := iter.SeekGE(lower, base.SeekGEFlagsNone)
foundPointKey := key != nil
if key != nil {
sst.SmallestPointKey.CopyFrom(*key)
}
@@ -776,14 +499,24 @@ func (d *DB) truncateSharedFile(
span := rangeDelIter.SeekGE(lower)
if span != nil && (len(sst.SmallestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.SmallestKey(), sst.SmallestPointKey) < 0) {
sst.SmallestPointKey.CopyFrom(span.SmallestKey())
foundPointKey = true
}
}
if !foundPointKey {
// There are no point keys in the span we're interested in.
sst.SmallestPointKey = InternalKey{}
sst.LargestPointKey = InternalKey{}
}
sst.SmallestRangeKey.UserKey = sst.SmallestRangeKey.UserKey[:0]
sst.SmallestRangeKey.Trailer = 0
if rangeKeyIter != nil {
span := rangeKeyIter.SeekGE(lower)
if span != nil {
sst.SmallestRangeKey.CopyFrom(span.SmallestKey())
} else {
// There are no range keys in the span we're interested in.
sst.SmallestRangeKey = InternalKey{}
sst.LargestRangeKey = InternalKey{}
}
}
}
@@ -793,6 +526,7 @@ func (d *DB) truncateSharedFile(
sst.LargestPointKey.UserKey = sst.LargestPointKey.UserKey[:0]
sst.LargestPointKey.Trailer = 0
key, _ := iter.SeekLT(upper, base.SeekLTFlagsNone)
foundPointKey := key != nil
if key != nil {
sst.LargestPointKey.CopyFrom(*key)
}
@@ -800,14 +534,24 @@ func (d *DB) truncateSharedFile(
span := rangeDelIter.SeekLT(upper)
if span != nil && (len(sst.LargestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.LargestKey(), sst.LargestPointKey) > 0) {
sst.LargestPointKey.CopyFrom(span.LargestKey())
foundPointKey = true
}
}
if !foundPointKey {
// There are no point keys in the span we're interested in.
sst.SmallestPointKey = InternalKey{}
sst.LargestPointKey = InternalKey{}
}
sst.LargestRangeKey.UserKey = sst.LargestRangeKey.UserKey[:0]
sst.LargestRangeKey.Trailer = 0
if rangeKeyIter != nil {
span := rangeKeyIter.SeekLT(upper)
if span != nil {
sst.LargestRangeKey.CopyFrom(span.LargestKey())
} else {
// There are no range keys in the span we're interested in.
sst.SmallestRangeKey = InternalKey{}
sst.LargestRangeKey = InternalKey{}
}
}
}
@@ -843,19 +587,19 @@ func (d *DB) truncateSharedFile(
if err != nil {
return nil, false, err
}
// On occasion, estimateSize gives us a low estimate, i.e. a 0 file size. This
// can cause panics in places where we divide by file sizes. Correct for it
// here.
if sst.Size == 0 {
sst.Size = 1
}
return sst, false, nil
}
func scanInternalImpl(
ctx context.Context,
lower, upper []byte,
iter *scanInternalIterator,
visitPointKey func(key *InternalKey, value LazyValue) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []keyspan.Key) error,
visitSharedFile func(sst *SharedSSTMeta) error,
ctx context.Context, lower, upper []byte, iter *scanInternalIterator, opts *scanInternalOptions,
) error {
if visitSharedFile != nil && (lower == nil || upper == nil) {
if opts.visitSharedFile != nil && (lower == nil || upper == nil) {
panic("lower and upper bounds must be specified in skip-shared iteration mode")
}
// Before starting iteration, check if any files in levels sharedLevelsStart
@@ -864,14 +608,18 @@ func scanInternalImpl(
// of keys. For files that are shared, call visitSharedFile with a truncated
// version of that file.
cmp := iter.comparer.Compare
db := iter.readState.db
provider := db.objProvider
if visitSharedFile != nil {
provider := iter.db.ObjProvider()
seqNum := iter.seqNum
current := iter.version
if current == nil {
current = iter.readState.current
}
if opts.visitSharedFile != nil {
if provider == nil {
panic("expected non-nil Provider in skip-shared iteration mode")
}
for level := sharedLevelsStart; level < numLevels; level++ {
files := iter.readState.current.Levels[level].Iter()
files := current.Levels[level].Iter()
for f := files.SeekGE(cmp, lower); f != nil && cmp(f.Smallest.UserKey, upper) < 0; f = files.Next() {
var objMeta objstorage.ObjectMetadata
var err error
@@ -880,18 +628,21 @@ func scanInternalImpl(
return err
}
if !objMeta.IsShared() {
return errors.Wrapf(ErrInvalidSkipSharedIteration, "when processing file %s", objMeta.DiskFileNum)
return errors.Wrapf(ErrInvalidSkipSharedIteration, "file %s is not shared", objMeta.DiskFileNum)
}
if !base.Visible(f.LargestSeqNum, seqNum, base.InternalKeySeqNumMax) {
return errors.Wrapf(ErrInvalidSkipSharedIteration, "file %s contains keys newer than snapshot", objMeta.DiskFileNum)
}
var sst *SharedSSTMeta
var skip bool
sst, skip, err = iter.readState.db.truncateSharedFile(ctx, lower, upper, level, f, objMeta)
sst, skip, err = iter.db.truncateSharedFile(ctx, lower, upper, level, f, objMeta)
if err != nil {
return err
}
if skip {
continue
}
if err = visitSharedFile(sst); err != nil {
if err = opts.visitSharedFile(sst); err != nil {
return err
}
}
@@ -901,21 +652,50 @@ func scanInternalImpl(
for valid := iter.seekGE(lower); valid && iter.error() == nil; valid = iter.next() {
key := iter.unsafeKey()
if opts.rateLimitFunc != nil {
if err := opts.rateLimitFunc(key, iter.lazyValue()); err != nil {
return err
}
}
switch key.Kind() {
case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
span := iter.unsafeSpan()
if err := visitRangeKey(span.Start, span.End, span.Keys); err != nil {
return err
if opts.visitRangeKey != nil {
span := iter.unsafeSpan()
// NB: The caller isn't interested in the sequence numbers of these
// range keys. Rather, the caller wants them to be in trailer order
// _after_ zeroing of sequence numbers. Copy span.Keys, sort it, and then
// call visitRangeKey.
keysCopy := make([]keyspan.Key, len(span.Keys))
for i := range span.Keys {
keysCopy[i] = span.Keys[i]
keysCopy[i].Trailer = base.MakeTrailer(0, span.Keys[i].Kind())
}
keyspan.SortKeysByTrailer(&keysCopy)
if err := opts.visitRangeKey(span.Start, span.End, keysCopy); err != nil {
return err
}
}
case InternalKeyKindRangeDelete:
rangeDel := iter.unsafeRangeDel()
if err := visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil {
return err
if opts.visitRangeDel != nil {
rangeDel := iter.unsafeRangeDel()
if err := opts.visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil {
return err
}
}
default:
val := iter.lazyValue()
if err := visitPointKey(key, val); err != nil {
return err
if opts.visitPointKey != nil {
var info IteratorLevel
if len(iter.mergingIter.heap.items) > 0 {
mergingIterIdx := iter.mergingIter.heap.items[0].index
info = iter.iterLevels[mergingIterIdx]
} else {
info = IteratorLevel{Kind: IteratorLevelUnknown}
}
val := iter.lazyValue()
if err := opts.visitPointKey(key, val, info); err != nil {
return err
}
}
}
}
@@ -935,9 +715,13 @@ func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *
numMergingLevels := len(memtables)
numLevelIters := 0
current := i.readState.current
current := i.version
if current == nil {
current = i.readState.current
}
numMergingLevels += len(current.L0SublevelFiles)
numLevelIters += len(current.L0SublevelFiles)
for level := 1; level < len(current.Levels); level++ {
if current.Levels[level].Empty() {
continue
@@ -960,19 +744,26 @@ func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *
rangeDelIters := make([]keyspan.FragmentIterator, 0, numMergingLevels)
rangeDelLevels := make([]keyspan.LevelIter, 0, numLevelIters)
i.iterLevels = make([]IteratorLevel, numMergingLevels)
mlevelsIndex := 0
// Next are the memtables.
for j := len(memtables) - 1; j >= 0; j-- {
mem := memtables[j]
mlevels = append(mlevels, mergingIterLevel{
iter: mem.newIter(&i.opts.IterOptions),
})
i.iterLevels[mlevelsIndex] = IteratorLevel{
Kind: IteratorLevelFlushable,
FlushableIndex: j,
}
mlevelsIndex++
if rdi := mem.newRangeDelIter(&i.opts.IterOptions); rdi != nil {
rangeDelIters = append(rangeDelIters, rdi)
}
}
// Next are the file levels: L0 sub-levels followed by lower levels.
mlevelsIndex := len(mlevels)
levelsIndex := len(levels)
mlevels = mlevels[:numMergingLevels]
levels = levels[:numLevelIters]
@@ -983,7 +774,7 @@ func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *
rli := &rangeDelLevels[levelsIndex]
li.init(
context.Background(), i.opts.IterOptions, i.comparer.Compare, i.comparer.Split, i.newIters, files, level,
context.Background(), i.opts.IterOptions, i.comparer, i.newIters, files, level,
internalIterOpts{})
li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext)
mlevels[mlevelsIndex].iter = li
@@ -996,12 +787,14 @@ func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *
mlevelsIndex++
}
// Add level iterators for the L0 sublevels, iterating from newest to
// oldest.
for i := len(current.L0SublevelFiles) - 1; i >= 0; i-- {
addLevelIterForFiles(current.L0SublevelFiles[i].Iter(), manifest.L0Sublevel(i))
for j := len(current.L0SublevelFiles) - 1; j >= 0; j-- {
i.iterLevels[mlevelsIndex] = IteratorLevel{
Kind: IteratorLevelLSM,
Level: 0,
Sublevel: j,
}
addLevelIterForFiles(current.L0SublevelFiles[j].Iter(), manifest.L0Sublevel(j))
}
// Add level iterators for the non-empty non-L0 levels.
for level := 1; level < numLevels; level++ {
if current.Levels[level].Empty() {
@@ -1010,18 +803,35 @@ func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *
if i.opts.skipSharedLevels && level >= sharedLevelsStart {
continue
}
i.iterLevels[mlevelsIndex] = IteratorLevel{Kind: IteratorLevelLSM, Level: level}
addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level))
}
buf.merging.init(&i.opts.IterOptions, &InternalIteratorStats{}, i.comparer.Compare, i.comparer.Split, mlevels...)
buf.merging.snapshot = i.seqNum
rangeDelMiter.Init(i.comparer.Compare, keyspan.VisibleTransform(i.seqNum), new(keyspan.MergingBuffers), rangeDelIters...)
i.pointKeyIter = pointCollapsingIterator{
comparer: i.comparer,
merge: i.merge,
seqNum: i.seqNum,
if i.opts.includeObsoleteKeys {
iiter := &keyspan.InterleavingIter{}
iiter.Init(i.comparer, &buf.merging, &rangeDelMiter,
keyspan.InterleavingIterOpts{
LowerBound: i.opts.LowerBound,
UpperBound: i.opts.UpperBound,
})
i.pointKeyIter = iiter
} else {
pcIter := &pointCollapsingIterator{
comparer: i.comparer,
merge: i.merge,
seqNum: i.seqNum,
}
pcIter.iter.Init(i.comparer, &buf.merging, &rangeDelMiter, keyspan.InterleavingIterOpts{
LowerBound: i.opts.LowerBound,
UpperBound: i.opts.UpperBound,
})
i.pointKeyIter = pcIter
}
i.pointKeyIter.iter.Init(i.comparer, &buf.merging, &rangeDelMiter, nil /* mask */, i.opts.LowerBound, i.opts.UpperBound)
i.iter = &i.pointKeyIter
i.iter = i.pointKeyIter
}
// constructRangeKeyIter constructs the range-key iterator stack, populating
@@ -1033,23 +843,28 @@ func (i *scanInternalIterator) constructRangeKeyIter() {
// RangeKeyUnsets and RangeKeyDels.
i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
nil /* hasPrefix */, nil /* prefix */, false, /* onlySets */
nil /* hasPrefix */, nil /* prefix */, true, /* internalKeys */
&i.rangeKey.rangeKeyBuffers.internal)
// Next are the flushables: memtables and large batches.
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts.IterOptions); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
if i.readState != nil {
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts.IterOptions); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
}
}
}
current := i.readState.current
current := i.version
if current == nil {
current = i.readState.current
}
// Next are the file levels: L0 sub-levels followed by lower levels.
//
// Add file-specific iterators for L0 files containing range keys. This is less
@@ -1064,7 +879,7 @@ func (i *scanInternalIterator) constructRangeKeyIter() {
// around Key Trailer order.
iter := current.RangeKeyLevels[0].Iter()
for f := iter.Last(); f != nil; f = iter.Prev() {
spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions(manifest.Level(0)))
spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions())
if err != nil {
i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
continue
@@ -1081,7 +896,7 @@ func (i *scanInternalIterator) constructRangeKeyIter() {
continue
}
li := i.rangeKey.iterConfig.NewLevelIter()
spanIterOpts := i.opts.SpanIterOptions(manifest.Level(level))
spanIterOpts := i.opts.SpanIterOptions()
li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
manifest.Level(level), manifest.KeyTypeRange)
i.rangeKey.iterConfig.AddLevel(li)
@@ -1111,7 +926,10 @@ func (i *scanInternalIterator) lazyValue() LazyValue {
// unsafeRangeDel returns a range key span. Behaviour undefined if UnsafeKey returns
// a non-rangedel kind.
func (i *scanInternalIterator) unsafeRangeDel() *keyspan.Span {
return i.pointKeyIter.iter.Span()
type spanInternalIterator interface {
Span() *keyspan.Span
}
return i.pointKeyIter.(spanInternalIterator).Span()
}
// unsafeSpan returns a range key span. Behaviour undefined if UnsafeKey returns
@@ -1137,7 +955,12 @@ func (i *scanInternalIterator) close() error {
if err := i.iter.Close(); err != nil {
return err
}
i.readState.unref()
if i.readState != nil {
i.readState.unref()
}
if i.version != nil {
i.version.Unref()
}
if i.rangeKey != nil {
i.rangeKey.PrepareForReuse()
*i.rangeKey = iteratorRangeKeyState{

410
vendor/github.com/cockroachdb/pebble/snapshot.go generated vendored
View File

@@ -8,16 +8,29 @@ import (
"context"
"io"
"math"
"sync"
"sync/atomic"
"time"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/rangekey"
)
// ErrSnapshotExcised is returned from WaitForFileOnlySnapshot if an excise
// overlapping with one of the EventuallyFileOnlySnapshot's KeyRanges gets
// applied before the transition of that EFOS to a file-only snapshot.
var ErrSnapshotExcised = errors.New("pebble: snapshot excised before conversion to file-only snapshot")
// Snapshot provides a read-only point-in-time view of the DB state.
type Snapshot struct {
// The db the snapshot was created from.
db *DB
seqNum uint64
// Set if part of an EventuallyFileOnlySnapshot.
efos *EventuallyFileOnlySnapshot
// The list the snapshot is linked into.
list *snapshotList
@@ -44,17 +57,17 @@ func (s *Snapshot) Get(key []byte) ([]byte, io.Closer, error) {
// NewIter returns an iterator that is unpositioned (Iterator.Valid() will
// return false). The iterator can be positioned via a call to SeekGE,
// SeekLT, First or Last.
func (s *Snapshot) NewIter(o *IterOptions) *Iterator {
func (s *Snapshot) NewIter(o *IterOptions) (*Iterator, error) {
return s.NewIterWithContext(context.Background(), o)
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
// tracing.
func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator {
func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) (*Iterator, error) {
if s.db == nil {
panic(ErrClosed)
}
return s.db.newIter(ctx, nil /* batch */, s, o)
return s.db.newIter(ctx, nil /* batch */, snapshotIterOpts{seqNum: s.seqNum}, o), nil
}
// ScanInternal scans all internal keys within the specified bounds, truncating
@@ -66,36 +79,36 @@ func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iter
func (s *Snapshot) ScanInternal(
ctx context.Context,
lower, upper []byte,
visitPointKey func(key *InternalKey, value LazyValue) error,
visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []keyspan.Key) error,
visitRangeKey func(start, end []byte, keys []rangekey.Key) error,
visitSharedFile func(sst *SharedSSTMeta) error,
) error {
if s.db == nil {
panic(ErrClosed)
}
iter := s.db.newInternalIter(s, &scanInternalOptions{
scanInternalOpts := &scanInternalOptions{
visitPointKey: visitPointKey,
visitRangeDel: visitRangeDel,
visitRangeKey: visitRangeKey,
visitSharedFile: visitSharedFile,
skipSharedLevels: visitSharedFile != nil,
IterOptions: IterOptions{
KeyTypes: IterKeyTypePointsAndRanges,
LowerBound: lower,
UpperBound: upper,
},
skipSharedLevels: visitSharedFile != nil,
})
}
iter := s.db.newInternalIter(snapshotIterOpts{seqNum: s.seqNum}, scanInternalOpts)
defer iter.close()
return scanInternalImpl(ctx, lower, upper, iter, visitPointKey, visitRangeDel, visitRangeKey, visitSharedFile)
return scanInternalImpl(ctx, lower, upper, iter, scanInternalOpts)
}
// Close closes the snapshot, releasing its resources. Close must be called.
// Failure to do so will result in a tiny memory leak and a large leak of
// resources on disk due to the entries the snapshot is preventing from being
// deleted.
func (s *Snapshot) Close() error {
if s.db == nil {
panic(ErrClosed)
}
s.db.mu.Lock()
// closeLocked is similar to Close(), except it requires that db.mu be held
// by the caller.
func (s *Snapshot) closeLocked() error {
s.db.mu.snapshots.remove(s)
// If s was the previous earliest snapshot, we might be able to reclaim
@@ -103,11 +116,26 @@ func (s *Snapshot) Close() error {
if e := s.db.mu.snapshots.earliest(); e > s.seqNum {
s.db.maybeScheduleCompactionPicker(pickElisionOnly)
}
s.db.mu.Unlock()
s.db = nil
return nil
}
// Close closes the snapshot, releasing its resources. Close must be called.
// Failure to do so will result in a tiny memory leak and a large leak of
// resources on disk due to the entries the snapshot is preventing from being
// deleted.
//
// d.mu must NOT be held by the caller.
func (s *Snapshot) Close() error {
db := s.db
if db == nil {
panic(ErrClosed)
}
db.mu.Lock()
defer db.mu.Unlock()
return s.closeLocked()
}
type snapshotList struct {
root Snapshot
}
@@ -175,3 +203,345 @@ func (l *snapshotList) remove(s *Snapshot) {
s.prev = nil // avoid memory leaks
s.list = nil // avoid memory leaks
}
// EventuallyFileOnlySnapshot (aka EFOS) provides a read-only point-in-time view
// of the database state, similar to Snapshot. An EventuallyFileOnlySnapshot
// induces less write amplification than Snapshot, at the cost of increased space
// amplification. While a Snapshot may increase write amplification across all
// flushes and compactions for the duration of its lifetime, an
// EventuallyFileOnlySnapshot only incurs that cost for flushes/compactions if
// memtables at the time of EFOS instantiation contained keys that the EFOS is
// interested in (i.e. its protectedRanges). In that case, the EFOS prevents
// elision of keys visible to it, similar to a Snapshot, until those memtables
// are flushed, and once that happens, the "EventuallyFileOnlySnapshot"
// transitions to a file-only snapshot state in which it pins zombies sstables
// like an open Iterator would, without pinning any memtables. Callers that can
// tolerate the increased space amplification of pinning zombie sstables until
// the snapshot is closed may prefer EventuallyFileOnlySnapshots for their
// reduced write amplification. Callers that desire the benefits of the file-only
// state that requires no pinning of memtables should call
// `WaitForFileOnlySnapshot()` (and possibly re-mint an EFOS if it returns
// ErrSnapshotExcised) before relying on the EFOS to keep producing iterators
// with zero write-amp and zero pinning of memtables in memory.
//
// EventuallyFileOnlySnapshots interact with the IngestAndExcise operation in
// subtle ways. No new iterators can be created once
// EventuallyFileOnlySnapshot.excised is set to true.
type EventuallyFileOnlySnapshot struct {
mu struct {
// NB: If both this mutex and db.mu are being grabbed, db.mu should be
// grabbed _before_ grabbing this one.
sync.Mutex
// Either the snap field is set below, or the version is set at any given
// point of time. If a snapshot is referenced, this is not a file-only
// snapshot yet, and if a version is set (and ref'd) this is a file-only
// snapshot.
// The wrapped regular snapshot, if not a file-only snapshot yet.
snap *Snapshot
// The wrapped version reference, if a file-only snapshot.
vers *version
}
// Key ranges to watch for an excise on.
protectedRanges []KeyRange
// excised, if true, signals that the above ranges were excised during the
// lifetime of this snapshot.
excised atomic.Bool
// The db the snapshot was created from.
db *DB
seqNum uint64
closed chan struct{}
}
func (d *DB) makeEventuallyFileOnlySnapshot(
keyRanges []KeyRange, internalKeyRanges []internalKeyRange,
) *EventuallyFileOnlySnapshot {
isFileOnly := true
d.mu.Lock()
defer d.mu.Unlock()
seqNum := d.mu.versions.visibleSeqNum.Load()
// Check if any of the keyRanges overlap with a memtable.
for i := range d.mu.mem.queue {
mem := d.mu.mem.queue[i]
if ingestMemtableOverlaps(d.cmp, mem, internalKeyRanges) {
isFileOnly = false
break
}
}
es := &EventuallyFileOnlySnapshot{
db: d,
seqNum: seqNum,
protectedRanges: keyRanges,
closed: make(chan struct{}),
}
if isFileOnly {
es.mu.vers = d.mu.versions.currentVersion()
es.mu.vers.Ref()
} else {
s := &Snapshot{
db: d,
seqNum: seqNum,
}
s.efos = es
es.mu.snap = s
d.mu.snapshots.pushBack(s)
}
return es
}
// Transitions this EventuallyFileOnlySnapshot to a file-only snapshot. Requires
// earliestUnflushedSeqNum and vers to correspond to the same Version from the
// current or a past acquisition of db.mu. vers must have been Ref()'d before
// that mutex was released, if it was released.
//
// NB: The caller is expected to check for es.excised before making this
// call.
//
// d.mu must be held when calling this method.
func (es *EventuallyFileOnlySnapshot) transitionToFileOnlySnapshot(vers *version) error {
es.mu.Lock()
select {
case <-es.closed:
vers.UnrefLocked()
es.mu.Unlock()
return ErrClosed
default:
}
if es.mu.snap == nil {
es.mu.Unlock()
panic("pebble: tried to transition an eventually-file-only-snapshot twice")
}
// The caller has already called Ref() on vers.
es.mu.vers = vers
// NB: The callers should have already done a check of es.excised.
oldSnap := es.mu.snap
es.mu.snap = nil
es.mu.Unlock()
return oldSnap.closeLocked()
}
// hasTransitioned returns true if this EFOS has transitioned to a file-only
// snapshot.
func (es *EventuallyFileOnlySnapshot) hasTransitioned() bool {
es.mu.Lock()
defer es.mu.Unlock()
return es.mu.vers != nil
}
// waitForFlush waits for a flush on any memtables that need to be flushed
// before this EFOS can transition to a file-only snapshot. If this EFOS is
// waiting on a flush of the mutable memtable, it forces a rotation within
// `dur` duration. For immutable memtables, it schedules a flush and waits for
// it to finish.
func (es *EventuallyFileOnlySnapshot) waitForFlush(ctx context.Context, dur time.Duration) error {
es.db.mu.Lock()
defer es.db.mu.Unlock()
earliestUnflushedSeqNum := es.db.getEarliestUnflushedSeqNumLocked()
for earliestUnflushedSeqNum < es.seqNum {
select {
case <-es.closed:
return ErrClosed
case <-ctx.Done():
return ctx.Err()
default:
}
// Check if the current mutable memtable contains keys less than seqNum.
// If so, rotate it.
if es.db.mu.mem.mutable.logSeqNum < es.seqNum && dur.Nanoseconds() > 0 {
es.db.maybeScheduleDelayedFlush(es.db.mu.mem.mutable, dur)
} else {
// Find the last memtable that contains seqNums less than es.seqNum,
// and force a flush on it.
var mem *flushableEntry
for i := range es.db.mu.mem.queue {
if es.db.mu.mem.queue[i].logSeqNum < es.seqNum {
mem = es.db.mu.mem.queue[i]
}
}
mem.flushForced = true
es.db.maybeScheduleFlush()
}
es.db.mu.compact.cond.Wait()
earliestUnflushedSeqNum = es.db.getEarliestUnflushedSeqNumLocked()
}
if es.excised.Load() {
return ErrSnapshotExcised
}
return nil
}
// WaitForFileOnlySnapshot blocks the calling goroutine until this snapshot
// has been converted into a file-only snapshot (i.e. all memtables containing
// keys < seqNum are flushed). A duration can be passed in, and if nonzero,
// a delayed flush will be scheduled at that duration if necessary.
//
// Idempotent; can be called multiple times with no side effects.
func (es *EventuallyFileOnlySnapshot) WaitForFileOnlySnapshot(
ctx context.Context, dur time.Duration,
) error {
if es.hasTransitioned() {
return nil
}
if err := es.waitForFlush(ctx, dur); err != nil {
return err
}
if invariants.Enabled {
// Since we aren't returning an error, we _must_ have transitioned to a
// file-only snapshot by now.
if !es.hasTransitioned() {
panic("expected EFOS to have transitioned to file-only snapshot after flush")
}
}
return nil
}
// Close closes the file-only snapshot and releases all referenced resources.
// Not idempotent.
func (es *EventuallyFileOnlySnapshot) Close() error {
close(es.closed)
es.db.mu.Lock()
defer es.db.mu.Unlock()
es.mu.Lock()
defer es.mu.Unlock()
if es.mu.snap != nil {
if err := es.mu.snap.closeLocked(); err != nil {
return err
}
}
if es.mu.vers != nil {
es.mu.vers.UnrefLocked()
}
return nil
}
// Get implements the Reader interface.
func (es *EventuallyFileOnlySnapshot) Get(key []byte) (value []byte, closer io.Closer, err error) {
// TODO(jackson): Use getInternal.
iter, err := es.NewIter(nil)
if err != nil {
return nil, nil, err
}
var valid bool
if es.db.opts.Comparer.Split != nil {
valid = iter.SeekPrefixGE(key)
} else {
valid = iter.SeekGE(key)
}
if !valid {
if err = firstError(iter.Error(), iter.Close()); err != nil {
return nil, nil, err
}
return nil, nil, ErrNotFound
}
if !es.db.equal(iter.Key(), key) {
return nil, nil, firstError(iter.Close(), ErrNotFound)
}
return iter.Value(), iter, nil
}
// NewIter returns an iterator that is unpositioned (Iterator.Valid() will
// return false). The iterator can be positioned via a call to SeekGE,
// SeekLT, First or Last.
func (es *EventuallyFileOnlySnapshot) NewIter(o *IterOptions) (*Iterator, error) {
return es.NewIterWithContext(context.Background(), o)
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
// tracing.
func (es *EventuallyFileOnlySnapshot) NewIterWithContext(
ctx context.Context, o *IterOptions,
) (*Iterator, error) {
select {
case <-es.closed:
panic(ErrClosed)
default:
}
es.mu.Lock()
defer es.mu.Unlock()
if es.mu.vers != nil {
sOpts := snapshotIterOpts{seqNum: es.seqNum, vers: es.mu.vers}
return es.db.newIter(ctx, nil /* batch */, sOpts, o), nil
}
if es.excised.Load() {
return nil, ErrSnapshotExcised
}
sOpts := snapshotIterOpts{seqNum: es.seqNum}
iter := es.db.newIter(ctx, nil /* batch */, sOpts, o)
// If excised is true, then keys relevant to the snapshot might not be
// present in the readState being used by the iterator. Error out.
if es.excised.Load() {
iter.Close()
return nil, ErrSnapshotExcised
}
return iter, nil
}
// ScanInternal scans all internal keys within the specified bounds, truncating
// any rangedels and rangekeys to those bounds. For use when an external user
// needs to be aware of all internal keys that make up a key range.
//
// See comment on db.ScanInternal for the behaviour that can be expected of
// point keys deleted by range dels and keys masked by range keys.
func (es *EventuallyFileOnlySnapshot) ScanInternal(
ctx context.Context,
lower, upper []byte,
visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []rangekey.Key) error,
visitSharedFile func(sst *SharedSSTMeta) error,
) error {
if es.db == nil {
panic(ErrClosed)
}
if es.excised.Load() {
return ErrSnapshotExcised
}
var sOpts snapshotIterOpts
es.mu.Lock()
if es.mu.vers != nil {
sOpts = snapshotIterOpts{
seqNum: es.seqNum,
vers: es.mu.vers,
}
} else {
sOpts = snapshotIterOpts{
seqNum: es.seqNum,
}
}
es.mu.Unlock()
opts := &scanInternalOptions{
IterOptions: IterOptions{
KeyTypes: IterKeyTypePointsAndRanges,
LowerBound: lower,
UpperBound: upper,
},
visitPointKey: visitPointKey,
visitRangeDel: visitRangeDel,
visitRangeKey: visitRangeKey,
visitSharedFile: visitSharedFile,
skipSharedLevels: visitSharedFile != nil,
}
iter := es.db.newInternalIter(sOpts, opts)
defer iter.close()
// If excised is true, then keys relevant to the snapshot might not be
// present in the readState being used by the iterator. Error out.
if es.excised.Load() {
return ErrSnapshotExcised
}
return scanInternalImpl(ctx, lower, upper, iter, opts)
}

36
vendor/github.com/cockroachdb/pebble/sstable/block.go generated vendored
View File

@@ -10,7 +10,6 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manual"
@@ -400,10 +399,9 @@ type blockIter struct {
// For a block encoded with a restart interval of 1, cached and cachedBuf
// will not be used as there are no prefix compressed entries between the
// restart points.
cached []blockEntry
cachedBuf []byte
cacheHandle cache.Handle
// The first user key in the block. This is used by the caller to set bounds
cached []blockEntry
cachedBuf []byte
handle bufferHandle
// for block iteration for already loaded blocks.
firstUserKey []byte
lazyValueHandling struct {
@@ -458,10 +456,10 @@ func (i *blockIter) init(
// ingested.
// - Foreign sstable iteration: globalSeqNum is always set.
func (i *blockIter) initHandle(
cmp Compare, block cache.Handle, globalSeqNum uint64, hideObsoletePoints bool,
cmp Compare, block bufferHandle, globalSeqNum uint64, hideObsoletePoints bool,
) error {
i.cacheHandle.Release()
i.cacheHandle = block
i.handle.Release()
i.handle = block
return i.init(cmp, block.Get(), globalSeqNum, hideObsoletePoints)
}
@@ -690,8 +688,11 @@ func (i *blockIter) getFirstUserKey() []byte {
// SeekGE implements internalIterator.SeekGE, as documented in the pebble
// package.
func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) {
i.clearCache()
if invariants.Enabled && i.isDataInvalidated() {
panic(errors.AssertionFailedf("invalidated blockIter used"))
}
i.clearCache()
// Find the index of the smallest restart point whose key is > the key
// sought; index will be numRestarts if there is no such restart point.
i.offset = 0
@@ -820,8 +821,11 @@ func (i *blockIter) SeekPrefixGE(
// SeekLT implements internalIterator.SeekLT, as documented in the pebble
// package.
func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, base.LazyValue) {
i.clearCache()
if invariants.Enabled && i.isDataInvalidated() {
panic(errors.AssertionFailedf("invalidated blockIter used"))
}
i.clearCache()
// Find the index of the smallest restart point whose key is >= the key
// sought; index will be numRestarts if there is no such restart point.
i.offset = 0
@@ -988,6 +992,10 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
// First implements internalIterator.First, as documented in the pebble
// package.
func (i *blockIter) First() (*InternalKey, base.LazyValue) {
if invariants.Enabled && i.isDataInvalidated() {
panic(errors.AssertionFailedf("invalidated blockIter used"))
}
i.offset = 0
if !i.valid() {
return nil, base.LazyValue{}
@@ -1017,6 +1025,10 @@ func decodeRestart(b []byte) int32 {
// Last implements internalIterator.Last, as documented in the pebble package.
func (i *blockIter) Last() (*InternalKey, base.LazyValue) {
if invariants.Enabled && i.isDataInvalidated() {
panic(errors.AssertionFailedf("invalidated blockIter used"))
}
// Seek forward from the last restart point.
i.offset = decodeRestart(i.data[i.restarts+4*(i.numRestarts-1):])
if !i.valid() {
@@ -1515,8 +1527,8 @@ func (i *blockIter) Error() error {
// Close implements internalIterator.Close, as documented in the pebble
// package.
func (i *blockIter) Close() error {
i.cacheHandle.Release()
i.cacheHandle = cache.Handle{}
i.handle.Release()
i.handle = bufferHandle{}
i.val = nil
i.lazyValue = base.LazyValue{}
i.lazyValueHandling.vbr = nil

28
vendor/github.com/cockroachdb/pebble/sstable/block_property.go generated vendored
View File

@@ -9,6 +9,7 @@ import (
"fmt"
"math"
"sync"
"unsafe"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/rangekey"
@@ -189,13 +190,13 @@ type BoundLimitedBlockPropertyFilter interface {
// indicates that the filter may be used to filter blocks that exclusively
// contain keys ≥ `key`, so long as the blocks' keys also satisfy the upper
// bound.
KeyIsWithinLowerBound(key *InternalKey) bool
KeyIsWithinLowerBound(key []byte) bool
// KeyIsWithinUpperBound tests whether the provided internal key falls
// within the current upper bound of the filter. A true return value
// indicates that the filter may be used to filter blocks that exclusively
// contain keys ≤ `key`, so long as the blocks' keys also satisfy the lower
// bound.
KeyIsWithinUpperBound(key *InternalKey) bool
KeyIsWithinUpperBound(key []byte) bool
}
// BlockIntervalCollector is a helper implementation of BlockPropertyCollector
@@ -643,15 +644,21 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
// considered intersecting.
continue
}
byteProps := []byte(props)
if len(byteProps) < 1 {
if len(props) < 1 {
return false, base.CorruptionErrorf(
"block properties for %s is corrupted", f.filters[i].Name())
}
shortID := shortID(byteProps[0])
intersects, err := f.filters[i].Intersects(byteProps[1:])
if err != nil || !intersects {
return false, err
shortID := shortID(props[0])
{
// Use an unsafe conversion to avoid allocating. Intersects() is not
// supposed to modify the given slice.
// Note that unsafe.StringData only works if the string is not empty
// (which we already checked).
byteProps := unsafe.Slice(unsafe.StringData(props), len(props))
intersects, err := f.filters[i].Intersects(byteProps[1:])
if err != nil || !intersects {
return false, err
}
}
// Intersects the sstable, so need to use this filter when
// deciding whether to read blocks.
@@ -684,12 +691,11 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
// be unused within this file.
return true, nil
}
byteProps := []byte(props)
if len(byteProps) < 1 {
if len(props) < 1 {
return false, base.CorruptionErrorf(
"block properties for %s is corrupted", f.boundLimitedFilter.Name())
}
f.boundLimitedShortID = int(byteProps[0])
f.boundLimitedShortID = int(props[0])
// We don't check for table-level intersection for the bound-limited filter.
// The bound-limited filter is treated as vacuously intersecting.

148
vendor/github.com/cockroachdb/pebble/sstable/buffer_pool.go generated vendored Normal file
View File

@@ -0,0 +1,148 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sstable
import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/cache"
)
// A bufferHandle is a handle to manually-managed memory. The handle may point
// to a block in the block cache (h.Get() != nil), or a buffer that exists
// outside the block cache allocated from a BufferPool (b.Valid()).
type bufferHandle struct {
h cache.Handle
b Buf
}
// Get retrieves the underlying buffer referenced by the handle.
func (bh bufferHandle) Get() []byte {
if v := bh.h.Get(); v != nil {
return v
} else if bh.b.p != nil {
return bh.b.p.pool[bh.b.i].b
}
return nil
}
// Release releases the buffer, either back to the block cache or BufferPool.
func (bh bufferHandle) Release() {
bh.h.Release()
bh.b.Release()
}
// A BufferPool holds a pool of buffers for holding sstable blocks. An initial
// size of the pool is provided on Init, but a BufferPool will grow to meet the
// largest working set size. It'll never shrink. When a buffer is released, the
// BufferPool recycles the buffer for future allocations.
//
// A BufferPool should only be used for short-lived allocations with
// well-understood working set sizes to avoid excessive memory consumption.
//
// BufferPool is not thread-safe.
type BufferPool struct {
// pool contains all the buffers held by the pool, including buffers that
// are in-use. For every i < len(pool): pool[i].v is non-nil.
pool []allocedBuffer
}
type allocedBuffer struct {
v *cache.Value
// b holds the current byte slice. It's backed by v, but may be a subslice
// of v's memory while the buffer is in-use [ len(b) ≤ len(v.Buf()) ].
//
// If the buffer is not currently in-use, b is nil. When being recycled, the
// BufferPool.Alloc will reset b to be a subslice of v.Buf().
b []byte
}
// Init initializes the pool with an initial working set buffer size of
// `initialSize`.
func (p *BufferPool) Init(initialSize int) {
*p = BufferPool{
pool: make([]allocedBuffer, 0, initialSize),
}
}
// initPreallocated is like Init but for internal sstable package use in
// instances where a pre-allocated slice of []allocedBuffer already exists. It's
// used to avoid an extra allocation initializing BufferPool.pool.
func (p *BufferPool) initPreallocated(pool []allocedBuffer) {
*p = BufferPool{
pool: pool[:0],
}
}
// Release releases all buffers held by the pool and resets the pool to an
// uninitialized state.
func (p *BufferPool) Release() {
for i := range p.pool {
if p.pool[i].b != nil {
panic(errors.AssertionFailedf("Release called on a BufferPool with in-use buffers"))
}
cache.Free(p.pool[i].v)
}
*p = BufferPool{}
}
// Alloc allocates a new buffer of size n. If the pool already holds a buffer at
// least as large as n, the pooled buffer is used instead.
//
// Alloc is O(MAX(N,M)) where N is the largest number of concurrently in-use
// buffers allocated and M is the initialSize passed to Init.
func (p *BufferPool) Alloc(n int) Buf {
unusableBufferIdx := -1
for i := 0; i < len(p.pool); i++ {
if p.pool[i].b == nil {
if len(p.pool[i].v.Buf()) >= n {
p.pool[i].b = p.pool[i].v.Buf()[:n]
return Buf{p: p, i: i}
}
unusableBufferIdx = i
}
}
// If we would need to grow the size of the pool to allocate another buffer,
// but there was a slot available occupied by a buffer that's just too
// small, replace the too-small buffer.
if len(p.pool) == cap(p.pool) && unusableBufferIdx >= 0 {
i := unusableBufferIdx
cache.Free(p.pool[i].v)
p.pool[i].v = cache.Alloc(n)
p.pool[i].b = p.pool[i].v.Buf()
return Buf{p: p, i: i}
}
// Allocate a new buffer.
v := cache.Alloc(n)
p.pool = append(p.pool, allocedBuffer{v: v, b: v.Buf()[:n]})
return Buf{p: p, i: len(p.pool) - 1}
}
// A Buf holds a reference to a manually-managed, pooled byte buffer.
type Buf struct {
p *BufferPool
// i holds the index into p.pool where the buffer may be found. This scheme
// avoids needing to allocate the handle to the buffer on the heap at the
// cost of copying two words instead of one.
i int
}
// Valid returns true if the buf holds a valid buffer.
func (b Buf) Valid() bool {
return b.p != nil
}
// Release releases the buffer back to the pool.
func (b *Buf) Release() {
if b.p == nil {
return
}
// Clear the allocedBuffer's byte slice. This signals the allocated buffer
// is no longer in use and a future call to BufferPool.Alloc may reuse this
// buffer.
b.p.pool[b.i].b = nil
b.p = nil
}

11
vendor/github.com/cockroachdb/pebble/sstable/compression.go generated vendored
View File

@@ -52,8 +52,10 @@ func decompressInto(blockType blockType, compressed []byte, buf []byte) ([]byte,
return result, nil
}
// decompressBlock decompresses an SST block, with space allocated from a cache.
func decompressBlock(cache *cache.Cache, blockType blockType, b []byte) (*cache.Value, error) {
// decompressBlock decompresses an SST block, with manually-allocated space.
// NB: If decompressBlock returns (nil, nil), no decompression was necessary and
// the caller may use `b` directly.
func decompressBlock(blockType blockType, b []byte) (*cache.Value, error) {
if blockType == noCompressionBlockType {
return nil, nil
}
@@ -62,14 +64,13 @@ func decompressBlock(cache *cache.Cache, blockType blockType, b []byte) (*cache.
if err != nil {
return nil, err
}
if prefixLen != 0 {
b = b[prefixLen:]
}
b = b[prefixLen:]
// Allocate sufficient space from the cache.
decoded := cache.Alloc(decodedLen)
decodedBuf := decoded.Buf()
if _, err := decompressInto(blockType, b, decodedBuf); err != nil {
cache.Free(decoded)
return nil, err
}
return decoded, nil
}

6
vendor/github.com/cockroachdb/pebble/sstable/format.go generated vendored
View File

@@ -25,8 +25,9 @@ const (
TableFormatPebblev2 // Range keys.
TableFormatPebblev3 // Value blocks.
TableFormatPebblev4 // DELSIZED tombstones.
NumTableFormats
TableFormatMax = TableFormatPebblev4
TableFormatMax = NumTableFormats - 1
)
// TableFormatPebblev4, in addition to DELSIZED, introduces the use of
@@ -91,7 +92,8 @@ const (
// obsolete, i.e., obsolete within the sstable, and we can guarantee that at
// most one point will be exposed per user key. MERGE keys create more
// complexity: MERGE followed by MERGE causes multiple keys to not be
// obsolete. Same applies for MERGE followed by SET/SETWITHDEL. Note that:
// obsolete. Same applies for MERGE followed by SET/SETWITHDEL/DEL*. Note
// that:
//
// - For regular sst iteration, the obsolete marking is a performance
// optimization, and multiple keys for the same userkey can be handled by

306
vendor/github.com/cockroachdb/pebble/sstable/layout.go generated vendored Normal file
View File

@@ -0,0 +1,306 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sstable
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
"sort"
"unsafe"
"github.com/cockroachdb/pebble/internal/base"
)
// Layout describes the block organization of an sstable.
type Layout struct {
// NOTE: changes to fields in this struct should also be reflected in
// ValidateBlockChecksums, which validates a static list of BlockHandles
// referenced in this struct.
Data []BlockHandleWithProperties
Index []BlockHandle
TopIndex BlockHandle
Filter BlockHandle
RangeDel BlockHandle
RangeKey BlockHandle
ValueBlock []BlockHandle
ValueIndex BlockHandle
Properties BlockHandle
MetaIndex BlockHandle
Footer BlockHandle
Format TableFormat
}
// Describe returns a description of the layout. If the verbose parameter is
// true, details of the structure of each block are returned as well.
func (l *Layout) Describe(
w io.Writer, verbose bool, r *Reader, fmtRecord func(key *base.InternalKey, value []byte),
) {
ctx := context.TODO()
type block struct {
BlockHandle
name string
}
var blocks []block
for i := range l.Data {
blocks = append(blocks, block{l.Data[i].BlockHandle, "data"})
}
for i := range l.Index {
blocks = append(blocks, block{l.Index[i], "index"})
}
if l.TopIndex.Length != 0 {
blocks = append(blocks, block{l.TopIndex, "top-index"})
}
if l.Filter.Length != 0 {
blocks = append(blocks, block{l.Filter, "filter"})
}
if l.RangeDel.Length != 0 {
blocks = append(blocks, block{l.RangeDel, "range-del"})
}
if l.RangeKey.Length != 0 {
blocks = append(blocks, block{l.RangeKey, "range-key"})
}
for i := range l.ValueBlock {
blocks = append(blocks, block{l.ValueBlock[i], "value-block"})
}
if l.ValueIndex.Length != 0 {
blocks = append(blocks, block{l.ValueIndex, "value-index"})
}
if l.Properties.Length != 0 {
blocks = append(blocks, block{l.Properties, "properties"})
}
if l.MetaIndex.Length != 0 {
blocks = append(blocks, block{l.MetaIndex, "meta-index"})
}
if l.Footer.Length != 0 {
if l.Footer.Length == levelDBFooterLen {
blocks = append(blocks, block{l.Footer, "leveldb-footer"})
} else {
blocks = append(blocks, block{l.Footer, "footer"})
}
}
sort.Slice(blocks, func(i, j int) bool {
return blocks[i].Offset < blocks[j].Offset
})
for i := range blocks {
b := &blocks[i]
fmt.Fprintf(w, "%10d %s (%d)\n", b.Offset, b.name, b.Length)
if !verbose {
continue
}
if b.name == "filter" {
continue
}
if b.name == "footer" || b.name == "leveldb-footer" {
trailer, offset := make([]byte, b.Length), b.Offset
_ = r.readable.ReadAt(ctx, trailer, int64(offset))
if b.name == "footer" {
checksumType := ChecksumType(trailer[0])
fmt.Fprintf(w, "%10d checksum type: %s\n", offset, checksumType)
trailer, offset = trailer[1:], offset+1
}
metaHandle, n := binary.Uvarint(trailer)
metaLen, m := binary.Uvarint(trailer[n:])
fmt.Fprintf(w, "%10d meta: offset=%d, length=%d\n", offset, metaHandle, metaLen)
trailer, offset = trailer[n+m:], offset+uint64(n+m)
indexHandle, n := binary.Uvarint(trailer)
indexLen, m := binary.Uvarint(trailer[n:])
fmt.Fprintf(w, "%10d index: offset=%d, length=%d\n", offset, indexHandle, indexLen)
trailer, offset = trailer[n+m:], offset+uint64(n+m)
fmt.Fprintf(w, "%10d [padding]\n", offset)
trailing := 12
if b.name == "leveldb-footer" {
trailing = 8
}
offset += uint64(len(trailer) - trailing)
trailer = trailer[len(trailer)-trailing:]
if b.name == "footer" {
version := trailer[:4]
fmt.Fprintf(w, "%10d version: %d\n", offset, binary.LittleEndian.Uint32(version))
trailer, offset = trailer[4:], offset+4
}
magicNumber := trailer
fmt.Fprintf(w, "%10d magic number: 0x%x\n", offset, magicNumber)
continue
}
h, err := r.readBlock(
context.Background(), b.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */, nil /* buffer pool */)
if err != nil {
fmt.Fprintf(w, " [err: %s]\n", err)
continue
}
getRestart := func(data []byte, restarts, i int32) int32 {
return decodeRestart(data[restarts+4*i:])
}
formatIsRestart := func(data []byte, restarts, numRestarts, offset int32) {
i := sort.Search(int(numRestarts), func(i int) bool {
return getRestart(data, restarts, int32(i)) >= offset
})
if i < int(numRestarts) && getRestart(data, restarts, int32(i)) == offset {
fmt.Fprintf(w, " [restart]\n")
} else {
fmt.Fprintf(w, "\n")
}
}
formatRestarts := func(data []byte, restarts, numRestarts int32) {
for i := int32(0); i < numRestarts; i++ {
offset := getRestart(data, restarts, i)
fmt.Fprintf(w, "%10d [restart %d]\n",
b.Offset+uint64(restarts+4*i), b.Offset+uint64(offset))
}
}
formatTrailer := func() {
trailer := make([]byte, blockTrailerLen)
offset := int64(b.Offset + b.Length)
_ = r.readable.ReadAt(ctx, trailer, offset)
bt := blockType(trailer[0])
checksum := binary.LittleEndian.Uint32(trailer[1:])
fmt.Fprintf(w, "%10d [trailer compression=%s checksum=0x%04x]\n", offset, bt, checksum)
}
var lastKey InternalKey
switch b.name {
case "data", "range-del", "range-key":
iter, _ := newBlockIter(r.Compare, h.Get())
for key, value := iter.First(); key != nil; key, value = iter.Next() {
ptr := unsafe.Pointer(uintptr(iter.ptr) + uintptr(iter.offset))
shared, ptr := decodeVarint(ptr)
unshared, ptr := decodeVarint(ptr)
value2, _ := decodeVarint(ptr)
total := iter.nextOffset - iter.offset
// The format of the numbers in the record line is:
//
// (<total> = <length> [<shared>] + <unshared> + <value>)
//
// <total> is the total number of bytes for the record.
// <length> is the size of the 3 varint encoded integers for <shared>,
// <unshared>, and <value>.
// <shared> is the number of key bytes shared with the previous key.
// <unshared> is the number of unshared key bytes.
// <value> is the number of value bytes.
fmt.Fprintf(w, "%10d record (%d = %d [%d] + %d + %d)",
b.Offset+uint64(iter.offset), total,
total-int32(unshared+value2), shared, unshared, value2)
formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
if fmtRecord != nil {
fmt.Fprintf(w, " ")
if l.Format < TableFormatPebblev3 {
fmtRecord(key, value.InPlaceValue())
} else {
// InPlaceValue() will succeed even for data blocks where the
// actual value is in a different location, since this value was
// fetched from a blockIter which does not know about value
// blocks.
v := value.InPlaceValue()
if base.TrailerKind(key.Trailer) != InternalKeyKindSet {
fmtRecord(key, v)
} else if !isValueHandle(valuePrefix(v[0])) {
fmtRecord(key, v[1:])
} else {
vh := decodeValueHandle(v[1:])
fmtRecord(key, []byte(fmt.Sprintf("value handle %+v", vh)))
}
}
}
if base.InternalCompare(r.Compare, lastKey, *key) >= 0 {
fmt.Fprintf(w, " WARNING: OUT OF ORDER KEYS!\n")
}
lastKey.Trailer = key.Trailer
lastKey.UserKey = append(lastKey.UserKey[:0], key.UserKey...)
}
formatRestarts(iter.data, iter.restarts, iter.numRestarts)
formatTrailer()
case "index", "top-index":
iter, _ := newBlockIter(r.Compare, h.Get())
for key, value := iter.First(); key != nil; key, value = iter.Next() {
bh, err := decodeBlockHandleWithProperties(value.InPlaceValue())
if err != nil {
fmt.Fprintf(w, "%10d [err: %s]\n", b.Offset+uint64(iter.offset), err)
continue
}
fmt.Fprintf(w, "%10d block:%d/%d",
b.Offset+uint64(iter.offset), bh.Offset, bh.Length)
formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
}
formatRestarts(iter.data, iter.restarts, iter.numRestarts)
formatTrailer()
case "properties":
iter, _ := newRawBlockIter(r.Compare, h.Get())
for valid := iter.First(); valid; valid = iter.Next() {
fmt.Fprintf(w, "%10d %s (%d)",
b.Offset+uint64(iter.offset), iter.Key().UserKey, iter.nextOffset-iter.offset)
formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
}
formatRestarts(iter.data, iter.restarts, iter.numRestarts)
formatTrailer()
case "meta-index":
iter, _ := newRawBlockIter(r.Compare, h.Get())
for valid := iter.First(); valid; valid = iter.Next() {
value := iter.Value()
var bh BlockHandle
var n int
var vbih valueBlocksIndexHandle
isValueBlocksIndexHandle := false
if bytes.Equal(iter.Key().UserKey, []byte(metaValueIndexName)) {
vbih, n, err = decodeValueBlocksIndexHandle(value)
bh = vbih.h
isValueBlocksIndexHandle = true
} else {
bh, n = decodeBlockHandle(value)
}
if n == 0 || n != len(value) {
fmt.Fprintf(w, "%10d [err: %s]\n", b.Offset+uint64(iter.offset), err)
continue
}
var vbihStr string
if isValueBlocksIndexHandle {
vbihStr = fmt.Sprintf(" value-blocks-index-lengths: %d(num), %d(offset), %d(length)",
vbih.blockNumByteLength, vbih.blockOffsetByteLength, vbih.blockLengthByteLength)
}
fmt.Fprintf(w, "%10d %s block:%d/%d%s",
b.Offset+uint64(iter.offset), iter.Key().UserKey,
bh.Offset, bh.Length, vbihStr)
formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
}
formatRestarts(iter.data, iter.restarts, iter.numRestarts)
formatTrailer()
case "value-block":
// We don't peer into the value-block since it can't be interpreted
// without the valueHandles.
case "value-index":
// We have already read the value-index to construct the list of
// value-blocks, so no need to do it again.
}
h.Release()
}
last := blocks[len(blocks)-1]
fmt.Fprintf(w, "%10d EOF\n", last.Offset+last.Length)
}

6
vendor/github.com/cockroachdb/pebble/sstable/options.go generated vendored
View File

@@ -5,6 +5,7 @@
package sstable
import (
"github.com/cockroachdb/fifo"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
)
@@ -111,6 +112,11 @@ type ReaderOptions struct {
// The default cache size is a zero-size cache.
Cache *cache.Cache
// LoadBlockSema, if set, is used to limit the number of blocks that can be
// loaded (i.e. read from the filesystem) in parallel. Each load acquires one
// unit from the semaphore for the duration of the read.
LoadBlockSema *fifo.Semaphore
// User properties specified in this map will not be added to sst.Properties.UserProperties.
DeniedUserProperties map[string]struct{}

139
vendor/github.com/cockroachdb/pebble/sstable/properties.go generated vendored
View File

@@ -25,10 +25,20 @@ var propBoolFalse = []byte{'0'}
var propOffsetTagMap = make(map[uintptr]string)
func init() {
t := reflect.TypeOf(Properties{})
func generateTagMaps(t reflect.Type, indexPrefix []int) {
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
if f.Type.Kind() == reflect.Struct {
if tag := f.Tag.Get("prop"); i == 0 && tag == "pebble.embbeded_common_properties" {
// CommonProperties struct embedded in Properties. Note that since
// CommonProperties is placed at the top of properties we can use
// the offsets of the fields within CommonProperties to determine
// the offsets of those fields within Properties.
generateTagMaps(f.Type, []int{i})
continue
}
panic("pebble: unknown struct type in Properties")
}
if tag := f.Tag.Get("prop"); tag != "" {
switch f.Type.Kind() {
case reflect.Bool:
@@ -38,16 +48,84 @@ func init() {
default:
panic(fmt.Sprintf("unsupported property field type: %s %s", f.Name, f.Type))
}
if len(indexPrefix) > 0 {
// Prepend the index prefix so that we can use FieldByIndex on the top-level struct.
f.Index = append(indexPrefix[:len(indexPrefix):len(indexPrefix)], f.Index...)
}
propTagMap[tag] = f
propOffsetTagMap[f.Offset] = tag
}
}
}
func init() {
generateTagMaps(reflect.TypeOf(Properties{}), nil)
}
// CommonProperties holds properties for either a virtual or a physical sstable. This
// can be used by code which doesn't care to make the distinction between physical
// and virtual sstables properties.
//
// For virtual sstables, fields are constructed through extrapolation upon virtual
// reader construction. See MakeVirtualReader for implementation details.
//
// NB: The values of these properties can affect correctness. For example,
// if NumRangeKeySets == 0, but the sstable actually contains range keys, then
// the iterators will behave incorrectly.
type CommonProperties struct {
// The number of entries in this table.
NumEntries uint64 `prop:"rocksdb.num.entries"`
// Total raw key size.
RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
// Total raw value size.
RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
// Total raw key size of point deletion tombstones. This value is comparable
// to RawKeySize.
RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
// Sum of the raw value sizes carried by point deletion tombstones
// containing size estimates. See the DeleteSized key kind. This value is
// comparable to Raw{Key,Value}Size.
RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
// The number of point deletion entries ("tombstones") in this table that
// carry a size hint indicating the size of the value the tombstone deletes.
NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
// The number of deletion entries in this table, including both point and
// range deletions.
NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
// The number of range deletions in this table.
NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"`
// The number of RANGEKEYDELs in this table.
NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"`
// The number of RANGEKEYSETs in this table.
NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
// Total size of value blocks and value index block. Only serialized if > 0.
ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
}
// String is only used for testing purposes.
func (c *CommonProperties) String() string {
var buf bytes.Buffer
v := reflect.ValueOf(*c)
loaded := make(map[uintptr]struct{})
writeProperties(loaded, v, &buf)
return buf.String()
}
// NumPointDeletions is the number of point deletions in the sstable. For virtual
// sstables, this is an estimate.
func (c *CommonProperties) NumPointDeletions() uint64 {
return c.NumDeletions - c.NumRangeDeletions
}
// Properties holds the sstable property values. The properties are
// automatically populated during sstable creation and load from the properties
// meta block when an sstable is opened.
type Properties struct {
// CommonProperties needs to be at the top of the Properties struct so that the
// offsets of the fields in CommonProperties match the offsets of the embedded
// fields of CommonProperties in Properties.
CommonProperties `prop:"pebble.embbeded_common_properties"`
// The name of the comparer used in this table.
ComparerName string `prop:"rocksdb.comparator"`
// The compression algorithm used to compress blocks.
@@ -81,22 +159,8 @@ type Properties struct {
MergerName string `prop:"rocksdb.merge.operator"`
// The number of blocks in this table.
NumDataBlocks uint64 `prop:"rocksdb.num.data.blocks"`
// The number of deletion entries in this table, including both point and
// range deletions.
NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
// The number of point deletion entries ("tombstones") in this table that
// carry a size hint indicating the size of the value the tombstone deletes.
NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
// The number of entries in this table.
NumEntries uint64 `prop:"rocksdb.num.entries"`
// The number of merge operands in the table.
NumMergeOperands uint64 `prop:"rocksdb.merge.operands"`
// The number of range deletions in this table.
NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"`
// The number of RANGEKEYDELs in this table.
NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"`
// The number of RANGEKEYSETs in this table.
NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
// The number of RANGEKEYUNSETs in this table.
NumRangeKeyUnsets uint64 `prop:"pebble.num.range-key-unsets"`
// The number of value blocks in this table. Only serialized if > 0.
@@ -111,21 +175,10 @@ type Properties struct {
// A comma separated list of names of the property collectors used in this
// table.
PropertyCollectorNames string `prop:"rocksdb.property.collectors"`
// Total raw key size.
RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
// Total raw key size of point deletion tombstones. This value is comparable
// to RawKeySize.
RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
// Sum of the raw value sizes carried by point deletion tombstones
// containing size estimates. See the DeleteSized key kind. This value is
// comparable to Raw{Key,Value}Size.
RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
// Total raw rangekey key size.
RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"`
// Total raw rangekey value size.
RawRangeKeyValueSize uint64 `prop:"pebble.raw.range-key.value.size"`
// Total raw value size.
RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
// The total number of keys in this table that were pinned by open snapshots.
SnapshotPinnedKeys uint64 `prop:"pebble.num.snapshot-pinned-keys"`
// The cumulative bytes of keys in this table that were pinned by
@@ -138,8 +191,6 @@ type Properties struct {
TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"`
// User collected properties.
UserProperties map[string]string
// Total size of value blocks and value index block. Only serialized if > 0.
ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
// If filtering is enabled, was the filter created on the whole key.
WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"`
@@ -160,12 +211,15 @@ func (p *Properties) NumRangeKeys() uint64 {
return p.NumRangeKeyDels + p.NumRangeKeySets + p.NumRangeKeyUnsets
}
func (p *Properties) String() string {
var buf bytes.Buffer
v := reflect.ValueOf(*p)
func writeProperties(loaded map[uintptr]struct{}, v reflect.Value, buf *bytes.Buffer) {
vt := v.Type()
for i := 0; i < v.NumField(); i++ {
ft := vt.Field(i)
if ft.Type.Kind() == reflect.Struct {
// Embedded struct within the properties.
writeProperties(loaded, v.Field(i), buf)
continue
}
tag := ft.Tag.Get("prop")
if tag == "" {
continue
@@ -175,25 +229,33 @@ func (p *Properties) String() string {
// TODO(peter): Use f.IsZero() when we can rely on go1.13.
if zero := reflect.Zero(f.Type()); zero.Interface() == f.Interface() {
// Skip printing of zero values which were not loaded from disk.
if _, ok := p.Loaded[ft.Offset]; !ok {
if _, ok := loaded[ft.Offset]; !ok {
continue
}
}
fmt.Fprintf(&buf, "%s: ", tag)
fmt.Fprintf(buf, "%s: ", tag)
switch ft.Type.Kind() {
case reflect.Bool:
fmt.Fprintf(&buf, "%t\n", f.Bool())
fmt.Fprintf(buf, "%t\n", f.Bool())
case reflect.Uint32:
fmt.Fprintf(&buf, "%d\n", f.Uint())
fmt.Fprintf(buf, "%d\n", f.Uint())
case reflect.Uint64:
fmt.Fprintf(&buf, "%d\n", f.Uint())
fmt.Fprintf(buf, "%d\n", f.Uint())
case reflect.String:
fmt.Fprintf(&buf, "%s\n", f.String())
fmt.Fprintf(buf, "%s\n", f.String())
default:
panic("not reached")
}
}
}
func (p *Properties) String() string {
var buf bytes.Buffer
v := reflect.ValueOf(*p)
writeProperties(p.Loaded, v, &buf)
// Write the UserProperties.
keys := make([]string, 0, len(p.UserProperties))
for key := range p.UserProperties {
keys = append(keys, key)
@@ -214,6 +276,7 @@ func (p *Properties) load(
}
p.Loaded = make(map[uintptr]struct{})
v := reflect.ValueOf(p).Elem()
for valid := i.First(); valid; valid = i.Next() {
if f, ok := propTagMap[string(i.Key().UserKey)]; ok {
p.Loaded[f.Offset] = struct{}{}

3366
vendor/github.com/cockroachdb/pebble/sstable/reader.go generated vendored
View File

File diff suppressed because it is too large Load Diff

291
vendor/github.com/cockroachdb/pebble/sstable/reader_iter.go generated vendored Normal file
View File

@@ -0,0 +1,291 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sstable
import (
"fmt"
"os"
"sync"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Iterator iterates over an entire table of data.
type Iterator interface {
base.InternalIterator
// NextPrefix implements (base.InternalIterator).NextPrefix.
NextPrefix(succKey []byte) (*InternalKey, base.LazyValue)
// MaybeFilteredKeys may be called when an iterator is exhausted to indicate
// whether or not the last positioning method may have skipped any keys due
// to block-property filters. This is used by the Pebble levelIter to
// control when an iterator steps to the next sstable.
//
// MaybeFilteredKeys may always return false positives, that is it may
// return true when no keys were filtered. It should only be called when the
// iterator is exhausted. It must never return false negatives when the
// iterator is exhausted.
MaybeFilteredKeys() bool
SetCloseHook(fn func(i Iterator) error)
}
// Iterator positioning optimizations and singleLevelIterator and
// twoLevelIterator:
//
// An iterator is absolute positioned using one of the Seek or First or Last
// calls. After absolute positioning, there can be relative positioning done
// by stepping using Prev or Next.
//
// We implement optimizations below where an absolute positioning call can in
// some cases use the current position to do less work. To understand these,
// we first define some terms. An iterator is bounds-exhausted if the bounds
// (upper of lower) have been reached. An iterator is data-exhausted if it has
// the reached the end of the data (forward or reverse) in the sstable. A
// singleLevelIterator only knows a local-data-exhausted property since when
// it is used as part of a twoLevelIterator, the twoLevelIterator can step to
// the next lower-level index block.
//
// The bounds-exhausted property is tracked by
// singleLevelIterator.exhaustedBounds being +1 (upper bound reached) or -1
// (lower bound reached). The same field is reused by twoLevelIterator. Either
// may notice the exhaustion of the bound and set it. Note that if
// singleLevelIterator sets this property, it is not a local property (since
// the bound has been reached regardless of whether this is in the context of
// the twoLevelIterator or not).
//
// The data-exhausted property is tracked in a more subtle manner. We define
// two predicates:
// - partial-local-data-exhausted (PLDE):
// i.data.isDataInvalidated() || !i.data.valid()
// - partial-global-data-exhausted (PGDE):
// i.index.isDataInvalidated() || !i.index.valid() || i.data.isDataInvalidated() ||
// !i.data.valid()
//
// PLDE is defined for a singleLevelIterator. PGDE is defined for a
// twoLevelIterator. Oddly, in our code below the singleLevelIterator does not
// know when it is part of a twoLevelIterator so it does not know when its
// property is local or global.
//
// Now to define data-exhausted:
// - Prerequisite: we must know that the iterator has been positioned and
// i.err is nil.
// - bounds-exhausted must not be true:
// If bounds-exhausted is true, we have incomplete knowledge of
// data-exhausted since PLDE or PGDE could be true because we could have
// chosen not to load index block or data block and figured out that the
// bound is exhausted (due to block property filters filtering out index and
// data blocks and going past the bound on the top level index block). Note
// that if we tried to separate out the BPF case from others we could
// develop more knowledge here.
// - PGDE is true for twoLevelIterator. PLDE is true if it is a standalone
// singleLevelIterator. !PLDE or !PGDE of course imply that data-exhausted
// is not true.
//
// An implication of the above is that if we are going to somehow utilize
// knowledge of data-exhausted in an optimization, we must not forget the
// existing value of bounds-exhausted since by forgetting the latter we can
// erroneously think that data-exhausted is true. Bug #2036 was due to this
// forgetting.
//
// Now to the two categories of optimizations we currently have:
// - Monotonic bounds optimization that reuse prior iterator position when
// doing seek: These only work with !data-exhausted. We could choose to make
// these work with data-exhausted but have not bothered because in the
// context of a DB if data-exhausted were true, the DB would move to the
// next file in the level. Note that this behavior of moving to the next
// file is not necessarily true for L0 files, so there could be some benefit
// in the future in this optimization. See the WARNING-data-exhausted
// comments if trying to optimize this in the future.
// - TrySeekUsingNext optimizations: these work regardless of exhaustion
// state.
//
// Implementation detail: In the code PLDE only checks that
// i.data.isDataInvalidated(). This narrower check is safe, since this is a
// subset of the set expressed by the OR expression. Also, it is not a
// de-optimization since whenever we exhaust the iterator we explicitly call
// i.data.invalidate(). PGDE checks i.index.isDataInvalidated() &&
// i.data.isDataInvalidated(). Again, this narrower check is safe, and not a
// de-optimization since whenever we exhaust the iterator we explicitly call
// i.index.invalidate() and i.data.invalidate(). The && is questionable -- for
// now this is a bit of defensive code. We should seriously consider removing
// it, since defensive code suggests we are not confident about our invariants
// (and if we are not confident, we need more invariant assertions, not
// defensive code).
//
// TODO(sumeer): remove the aforementioned defensive code.
var singleLevelIterPool = sync.Pool{
New: func() interface{} {
i := &singleLevelIterator{}
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(i, checkSingleLevelIterator)
return i
},
}
var twoLevelIterPool = sync.Pool{
New: func() interface{} {
i := &twoLevelIterator{}
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(i, checkTwoLevelIterator)
return i
},
}
// TODO(jackson): rangedel fragmentBlockIters can't be pooled because of some
// code paths that double Close the iters. Fix the double close and pool the
// *fragmentBlockIter type directly.
var rangeKeyFragmentBlockIterPool = sync.Pool{
New: func() interface{} {
i := &rangeKeyFragmentBlockIter{}
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(i, checkRangeKeyFragmentBlockIterator)
return i
},
}
func checkSingleLevelIterator(obj interface{}) {
i := obj.(*singleLevelIterator)
if p := i.data.handle.Get(); p != nil {
fmt.Fprintf(os.Stderr, "singleLevelIterator.data.handle is not nil: %p\n", p)
os.Exit(1)
}
if p := i.index.handle.Get(); p != nil {
fmt.Fprintf(os.Stderr, "singleLevelIterator.index.handle is not nil: %p\n", p)
os.Exit(1)
}
}
func checkTwoLevelIterator(obj interface{}) {
i := obj.(*twoLevelIterator)
if p := i.data.handle.Get(); p != nil {
fmt.Fprintf(os.Stderr, "singleLevelIterator.data.handle is not nil: %p\n", p)
os.Exit(1)
}
if p := i.index.handle.Get(); p != nil {
fmt.Fprintf(os.Stderr, "singleLevelIterator.index.handle is not nil: %p\n", p)
os.Exit(1)
}
}
func checkRangeKeyFragmentBlockIterator(obj interface{}) {
i := obj.(*rangeKeyFragmentBlockIter)
if p := i.blockIter.handle.Get(); p != nil {
fmt.Fprintf(os.Stderr, "fragmentBlockIter.blockIter.handle is not nil: %p\n", p)
os.Exit(1)
}
}
// compactionIterator is similar to Iterator but it increments the number of
// bytes that have been iterated through.
type compactionIterator struct {
*singleLevelIterator
bytesIterated *uint64
prevOffset uint64
}
// compactionIterator implements the base.InternalIterator interface.
var _ base.InternalIterator = (*compactionIterator)(nil)
func (i *compactionIterator) String() string {
if i.vState != nil {
return i.vState.fileNum.String()
}
return i.reader.fileNum.String()
}
func (i *compactionIterator) SeekGE(
key []byte, flags base.SeekGEFlags,
) (*InternalKey, base.LazyValue) {
panic("pebble: SeekGE unimplemented")
}
func (i *compactionIterator) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekPrefixGE unimplemented")
}
func (i *compactionIterator) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*InternalKey, base.LazyValue) {
panic("pebble: SeekLT unimplemented")
}
func (i *compactionIterator) First() (*InternalKey, base.LazyValue) {
i.err = nil // clear cached iteration error
return i.skipForward(i.singleLevelIterator.First())
}
func (i *compactionIterator) Last() (*InternalKey, base.LazyValue) {
panic("pebble: Last unimplemented")
}
// Note: compactionIterator.Next mirrors the implementation of Iterator.Next
// due to performance. Keep the two in sync.
func (i *compactionIterator) Next() (*InternalKey, base.LazyValue) {
if i.err != nil {
return nil, base.LazyValue{}
}
return i.skipForward(i.data.Next())
}
func (i *compactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
panic("pebble: NextPrefix unimplemented")
}
func (i *compactionIterator) Prev() (*InternalKey, base.LazyValue) {
panic("pebble: Prev unimplemented")
}
func (i *compactionIterator) skipForward(
key *InternalKey, val base.LazyValue,
) (*InternalKey, base.LazyValue) {
if key == nil {
for {
if key, _ := i.index.Next(); key == nil {
break
}
result := i.loadBlock(+1)
if result != loadBlockOK {
if i.err != nil {
break
}
switch result {
case loadBlockFailed:
// We checked that i.index was at a valid entry, so
// loadBlockFailed could not have happened due to to i.index
// being exhausted, and must be due to an error.
panic("loadBlock should not have failed with no error")
case loadBlockIrrelevant:
panic("compactionIter should not be using block intervals for skipping")
default:
panic(fmt.Sprintf("unexpected case %d", result))
}
}
// result == loadBlockOK
if key, val = i.data.First(); key != nil {
break
}
}
}
curOffset := i.recordOffset()
*i.bytesIterated += uint64(curOffset - i.prevOffset)
i.prevOffset = curOffset
if i.vState != nil && key != nil {
cmp := i.cmp(key.UserKey, i.vState.upper.UserKey)
if cmp > 0 || (i.vState.upper.IsExclusiveSentinel() && cmp == 0) {
return nil, base.LazyValue{}
}
}
return key, val
}

1391
vendor/github.com/cockroachdb/pebble/sstable/reader_iter_single_lvl.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1085
vendor/github.com/cockroachdb/pebble/sstable/reader_iter_two_lvl.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

206
vendor/github.com/cockroachdb/pebble/sstable/reader_virtual.go generated vendored Normal file
View File

@@ -0,0 +1,206 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sstable
import (
"context"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
)
// VirtualReader wraps Reader. Its purpose is to restrict functionality of the
// Reader which should be inaccessible to virtual sstables, and enforce bounds
// invariants associated with virtual sstables. All reads on virtual sstables
// should go through a VirtualReader.
//
// INVARIANT: Any iterators created through a virtual reader will guarantee that
// they don't expose keys outside the virtual sstable bounds.
type VirtualReader struct {
vState virtualState
reader *Reader
Properties CommonProperties
}
// Lightweight virtual sstable state which can be passed to sstable iterators.
type virtualState struct {
lower InternalKey
upper InternalKey
fileNum base.FileNum
Compare Compare
isForeign bool
}
func ceilDiv(a, b uint64) uint64 {
return (a + b - 1) / b
}
// MakeVirtualReader is used to contruct a reader which can read from virtual
// sstables.
func MakeVirtualReader(
reader *Reader, meta manifest.VirtualFileMeta, isForeign bool,
) VirtualReader {
if reader.fileNum != meta.FileBacking.DiskFileNum {
panic("pebble: invalid call to MakeVirtualReader")
}
vState := virtualState{
lower: meta.Smallest,
upper: meta.Largest,
fileNum: meta.FileNum,
Compare: reader.Compare,
isForeign: isForeign,
}
v := VirtualReader{
vState: vState,
reader: reader,
}
v.Properties.RawKeySize = ceilDiv(reader.Properties.RawKeySize*meta.Size, meta.FileBacking.Size)
v.Properties.RawValueSize = ceilDiv(reader.Properties.RawValueSize*meta.Size, meta.FileBacking.Size)
v.Properties.NumEntries = ceilDiv(reader.Properties.NumEntries*meta.Size, meta.FileBacking.Size)
v.Properties.NumDeletions = ceilDiv(reader.Properties.NumDeletions*meta.Size, meta.FileBacking.Size)
v.Properties.NumRangeDeletions = ceilDiv(reader.Properties.NumRangeDeletions*meta.Size, meta.FileBacking.Size)
v.Properties.NumRangeKeyDels = ceilDiv(reader.Properties.NumRangeKeyDels*meta.Size, meta.FileBacking.Size)
// Note that we rely on NumRangeKeySets for correctness. If the sstable may
// contain range keys, then NumRangeKeySets must be > 0. ceilDiv works because
// meta.Size will not be 0 for virtual sstables.
v.Properties.NumRangeKeySets = ceilDiv(reader.Properties.NumRangeKeySets*meta.Size, meta.FileBacking.Size)
v.Properties.ValueBlocksSize = ceilDiv(reader.Properties.ValueBlocksSize*meta.Size, meta.FileBacking.Size)
v.Properties.NumSizedDeletions = ceilDiv(reader.Properties.NumSizedDeletions*meta.Size, meta.FileBacking.Size)
v.Properties.RawPointTombstoneKeySize = ceilDiv(reader.Properties.RawPointTombstoneKeySize*meta.Size, meta.FileBacking.Size)
v.Properties.RawPointTombstoneValueSize = ceilDiv(reader.Properties.RawPointTombstoneValueSize*meta.Size, meta.FileBacking.Size)
return v
}
// NewCompactionIter is the compaction iterator function for virtual readers.
func (v *VirtualReader) NewCompactionIter(
bytesIterated *uint64, rp ReaderProvider, bufferPool *BufferPool,
) (Iterator, error) {
return v.reader.newCompactionIter(bytesIterated, rp, &v.vState, bufferPool)
}
// NewIterWithBlockPropertyFiltersAndContextEtc wraps
// Reader.NewIterWithBlockPropertyFiltersAndContext. We assume that the passed
// in [lower, upper) bounds will have at least some overlap with the virtual
// sstable bounds. No overlap is not currently supported in the iterator.
func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContextEtc(
ctx context.Context,
lower, upper []byte,
filterer *BlockPropertiesFilterer,
hideObsoletePoints, useFilterBlock bool,
stats *base.InternalIteratorStats,
rp ReaderProvider,
) (Iterator, error) {
return v.reader.newIterWithBlockPropertyFiltersAndContext(
ctx, lower, upper, filterer, hideObsoletePoints, useFilterBlock, stats, rp, &v.vState,
)
}
// ValidateBlockChecksumsOnBacking will call ValidateBlockChecksumsOnBacking on the underlying reader.
// Note that block checksum validation is NOT restricted to virtual sstable bounds.
func (v *VirtualReader) ValidateBlockChecksumsOnBacking() error {
return v.reader.ValidateBlockChecksums()
}
// NewRawRangeDelIter wraps Reader.NewRawRangeDelIter.
func (v *VirtualReader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) {
iter, err := v.reader.NewRawRangeDelIter()
if err != nil {
return nil, err
}
if iter == nil {
return nil, nil
}
// Truncation of spans isn't allowed at a user key that also contains points
// in the same virtual sstable, as it would lead to covered points getting
// uncovered. Set panicOnUpperTruncate to true if the file's upper bound
// is not an exclusive sentinel.
//
// As an example, if an sstable contains a rangedel a-c and point keys at
// a.SET.2 and b.SET.3, the file bounds [a#2,SET-b#RANGEDELSENTINEL] are
// allowed (as they exclude b.SET.3), or [a#2,SET-c#RANGEDELSENTINEL] (as it
// includes both point keys), but not [a#2,SET-b#3,SET] (as it would truncate
// the rangedel at b and lead to the point being uncovered).
return keyspan.Truncate(
v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey,
&v.vState.lower, &v.vState.upper, !v.vState.upper.IsExclusiveSentinel(), /* panicOnUpperTruncate */
), nil
}
// NewRawRangeKeyIter wraps Reader.NewRawRangeKeyIter.
func (v *VirtualReader) NewRawRangeKeyIter() (keyspan.FragmentIterator, error) {
iter, err := v.reader.NewRawRangeKeyIter()
if err != nil {
return nil, err
}
if iter == nil {
return nil, nil
}
// Truncation of spans isn't allowed at a user key that also contains points
// in the same virtual sstable, as it would lead to covered points getting
// uncovered. Set panicOnUpperTruncate to true if the file's upper bound
// is not an exclusive sentinel.
//
// As an example, if an sstable contains a range key a-c and point keys at
// a.SET.2 and b.SET.3, the file bounds [a#2,SET-b#RANGEKEYSENTINEL] are
// allowed (as they exclude b.SET.3), or [a#2,SET-c#RANGEKEYSENTINEL] (as it
// includes both point keys), but not [a#2,SET-b#3,SET] (as it would truncate
// the range key at b and lead to the point being uncovered).
return keyspan.Truncate(
v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey,
&v.vState.lower, &v.vState.upper, !v.vState.upper.IsExclusiveSentinel(), /* panicOnUpperTruncate */
), nil
}
// Constrain bounds will narrow the start, end bounds if they do not fit within
// the virtual sstable. The function will return if the new end key is
// inclusive.
func (v *virtualState) constrainBounds(
start, end []byte, endInclusive bool,
) (lastKeyInclusive bool, first []byte, last []byte) {
first = start
if start == nil || v.Compare(start, v.lower.UserKey) < 0 {
first = v.lower.UserKey
}
// Note that we assume that start, end has some overlap with the virtual
// sstable bounds.
last = v.upper.UserKey
lastKeyInclusive = !v.upper.IsExclusiveSentinel()
if end != nil {
cmp := v.Compare(end, v.upper.UserKey)
switch {
case cmp == 0:
lastKeyInclusive = !v.upper.IsExclusiveSentinel() && endInclusive
last = v.upper.UserKey
case cmp > 0:
lastKeyInclusive = !v.upper.IsExclusiveSentinel()
last = v.upper.UserKey
default:
lastKeyInclusive = endInclusive
last = end
}
}
// TODO(bananabrick): What if someone passes in bounds completely outside of
// virtual sstable bounds?
return lastKeyInclusive, first, last
}
// EstimateDiskUsage just calls VirtualReader.reader.EstimateDiskUsage after
// enforcing the virtual sstable bounds.
func (v *VirtualReader) EstimateDiskUsage(start, end []byte) (uint64, error) {
_, f, l := v.vState.constrainBounds(start, end, true /* endInclusive */)
return v.reader.EstimateDiskUsage(f, l)
}
// CommonProperties implements the CommonReader interface.
func (v *VirtualReader) CommonProperties() *CommonProperties {
return &v.Properties
}

20
vendor/github.com/cockroachdb/pebble/sstable/value_block.go generated vendored
View File

@@ -13,7 +13,6 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"golang.org/x/exp/rand"
@@ -728,7 +727,7 @@ func (ukb *UserKeyPrefixBound) IsEmpty() bool {
type blockProviderWhenOpen interface {
readBlockForVBR(
ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats,
) (cache.Handle, error)
) (bufferHandle, error)
}
type blockProviderWhenClosed struct {
@@ -749,9 +748,12 @@ func (bpwc *blockProviderWhenClosed) close() {
func (bpwc blockProviderWhenClosed) readBlockForVBR(
ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats,
) (cache.Handle, error) {
) (bufferHandle, error) {
ctx = objiotracing.WithBlockType(ctx, objiotracing.ValueBlock)
return bpwc.r.readBlock(ctx, h, nil, nil, stats)
// TODO(jackson,sumeer): Consider whether to use a buffer pool in this case.
// The bpwc is not allowed to outlive the iterator tree, so it cannot
// outlive the buffer pool.
return bpwc.r.readBlock(ctx, h, nil, nil, stats, nil /* buffer pool */)
}
// ReaderProvider supports the implementation of blockProviderWhenClosed.
@@ -790,16 +792,16 @@ type valueBlockReader struct {
// The value blocks index is lazily retrieved the first time the reader
// needs to read a value that resides in a value block.
vbiBlock []byte
vbiCache cache.Handle
vbiCache bufferHandle
// When sequentially iterating through all key-value pairs, the cost of
// repeatedly getting a block that is already in the cache and releasing the
// cache.Handle can be ~40% of the cpu overhead. So the reader remembers the
// bufferHandle can be ~40% of the cpu overhead. So the reader remembers the
// last value block it retrieved, in case there is locality of access, and
// this value block can be used for the next value retrieval.
valueBlockNum uint32
valueBlock []byte
valueBlockPtr unsafe.Pointer
valueCache cache.Handle
valueCache bufferHandle
lazyFetcher base.LazyFetcher
closed bool
bufToMangle []byte
@@ -833,12 +835,12 @@ func (r *valueBlockReader) close() {
// we were to reopen this valueBlockReader and retrieve the same
// Handle.value from the cache, we don't want to accidentally unref it when
// attempting to unref the old handle.
r.vbiCache = cache.Handle{}
r.vbiCache = bufferHandle{}
r.valueBlock = nil
r.valueBlockPtr = nil
r.valueCache.Release()
// See comment above.
r.valueCache = cache.Handle{}
r.valueCache = bufferHandle{}
r.closed = true
// rp, vbih, stats remain valid, so that LazyFetcher.ValueFetcher can be
// implemented.

42
vendor/github.com/cockroachdb/pebble/sstable/writer.go generated vendored
View File

@@ -852,32 +852,35 @@ func (w *Writer) makeAddPointDecisionV3(
} else {
cmpUser = w.compare(prevPointUserKey, key.UserKey)
}
keyKindIsSetOrMerge := keyKind == InternalKeyKindSet || keyKind == InternalKeyKindSetWithDelete ||
keyKind == InternalKeyKindMerge
keyKindIsPointDelete := keyKind == InternalKeyKindDelete ||
keyKind == InternalKeyKindSingleDelete || keyKind == InternalKeyKindDeleteSized
if !keyKindIsSetOrMerge && !keyKindIsPointDelete {
// Ensure that no one adds a point key kind without considering the obsolete
// handling for that kind.
switch keyKind {
case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge,
InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
default:
panic(errors.AssertionFailedf("unexpected key kind %s", keyKind.String()))
}
// If same user key, then the current key is obsolete if any of the
// following is true:
// C1 The prev key was obsolete.
// C2 The prev key was not a MERGE
// C3 The current key is not a SET or SETWITHDELETE or MERGE, then it is
// obsolete. This rule excludes SET, SETWITHDELETE, MERGE, since they
// will be merged with the previous key.
// C2 The prev key was not a MERGE. When the previous key is a MERGE we must
// preserve SET* and MERGE since their values will be merged into the
// previous key. We also must preserve DEL* since there may be an older
// SET*/MERGE in a lower level that must not be merged with the MERGE --
// if we omit the DEL* that lower SET*/MERGE will become visible.
//
// Regardless of whether it is the same user key or not
// C4 The current key is some kind of point delete, and we are writing to
// the lowest level, then it is also obsolete.
// C3 The current key is some kind of point delete, and we are writing to
// the lowest level, then it is also obsolete. The correctness of this
// relies on the same user key not spanning multiple sstables in a level.
//
// C1 ensures that for a user key there is at most one transition from
// !obsolete to obsolete. Consider a user key k, for which the first n keys
// are not obsolete. We consider the various value of n:
//
// n = 0: This happens due to forceObsolete being set by the caller, or due
// to C4. forceObsolete must only be set due a RANGEDEL, and that RANGEDEL
// must also delete all the lower seqnums for the same user key. C4 triggers
// to C3. forceObsolete must only be set due a RANGEDEL, and that RANGEDEL
// must also delete all the lower seqnums for the same user key. C3 triggers
// due to a point delete and that deletes all the lower seqnums for the same
// user key.
//
@@ -885,12 +888,13 @@ func (w *Writer) makeAddPointDecisionV3(
// MERGE, or the current key is some kind of point delete.
//
// n > 1: This is due to a sequence of MERGE keys, potentially followed by a
// single SET or SETWITHDELETE.
isObsolete = (cmpUser == 0 &&
(prevKeyKind != InternalKeyKindMerge || prevPointKeyInfo.isObsolete || !keyKindIsSetOrMerge)) ||
(w.writingToLowestLevel &&
(keyKind == InternalKeyKindDelete || keyKind == InternalKeyKindSingleDelete ||
keyKind == InternalKeyKindDeleteSized))
// single non-MERGE key.
isObsoleteC1AndC2 := cmpUser == 0 &&
(prevPointKeyInfo.isObsolete || prevKeyKind != InternalKeyKindMerge)
isObsoleteC3 := w.writingToLowestLevel &&
(keyKind == InternalKeyKindDelete || keyKind == InternalKeyKindSingleDelete ||
keyKind == InternalKeyKindDeleteSized)
isObsolete = isObsoleteC1AndC2 || isObsoleteC3
// TODO(sumeer): storing isObsolete SET and SETWITHDEL in value blocks is
// possible, but requires some care in documenting and checking invariants.
// There is code that assumes nothing in value blocks because of single MVCC

211
vendor/github.com/cockroachdb/pebble/table_cache.go generated vendored
View File

@@ -9,7 +9,6 @@ import (
"context"
"fmt"
"io"
"math"
"runtime/debug"
"runtime/pprof"
"sync"
@@ -22,7 +21,6 @@ import (
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/internal/private"
"github.com/cockroachdb/pebble/internal/rangekey"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"github.com/cockroachdb/pebble/sstable"
@@ -196,6 +194,40 @@ func (c *tableCacheContainer) estimateSize(
return size, nil
}
// createCommonReader creates a Reader for this file. isForeign, if true for
// virtual sstables, is passed into the vSSTable reader so its iterators can
// collapse obsolete points accordingly.
func createCommonReader(
v *tableCacheValue, file *fileMetadata, isForeign bool,
) sstable.CommonReader {
// TODO(bananabrick): We suffer an allocation if file is a virtual sstable.
var cr sstable.CommonReader = v.reader
if file.Virtual {
virtualReader := sstable.MakeVirtualReader(
v.reader, file.VirtualMeta(), isForeign,
)
cr = &virtualReader
}
return cr
}
func (c *tableCacheContainer) withCommonReader(
meta *fileMetadata, fn func(sstable.CommonReader) error,
) error {
s := c.tableCache.getShard(meta.FileBacking.DiskFileNum)
v := s.findNode(meta, &c.dbOpts)
defer s.unrefValue(v)
if v.err != nil {
return v.err
}
provider := c.dbOpts.objProvider
objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum)
if err != nil {
return err
}
return fn(createCommonReader(v, meta, provider.IsSharedForeign(objMeta)))
}
func (c *tableCacheContainer) withReader(meta physicalMeta, fn func(*sstable.Reader) error) error {
s := c.tableCache.getShard(meta.FileBacking.DiskFileNum)
v := s.findNode(meta.FileMetadata, &c.dbOpts)
@@ -216,7 +248,12 @@ func (c *tableCacheContainer) withVirtualReader(
if v.err != nil {
return v.err
}
return fn(sstable.MakeVirtualReader(v.reader, meta))
provider := c.dbOpts.objProvider
objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum)
if err != nil {
return err
}
return fn(sstable.MakeVirtualReader(v.reader, meta, provider.IsSharedForeign(objMeta)))
}
func (c *tableCacheContainer) iterCount() int64 {
@@ -402,6 +439,7 @@ func (c *tableCacheShard) newIters(
}
hideObsoletePoints := false
var pointKeyFilters []BlockPropertyFilter
if opts != nil {
// This code is appending (at most one filter) in-place to
// opts.PointKeyFilters even though the slice is shared for iterators in
@@ -417,7 +455,7 @@ func (c *tableCacheShard) newIters(
//
// An alternative would be to have different slices for different sstable
// iterators, but that requires more work to avoid allocations.
hideObsoletePoints, opts.PointKeyFilters =
hideObsoletePoints, pointKeyFilters =
v.reader.TryAddBlockPropertyFilterForHideObsoletePoints(
opts.snapshotForHideObsoletePoints, file.LargestSeqNum, opts.PointKeyFilters)
}
@@ -426,31 +464,13 @@ func (c *tableCacheShard) newIters(
var err error
if opts != nil {
ok, filterer, err = c.checkAndIntersectFilters(v, opts.TableFilter,
opts.PointKeyFilters, internalOpts.boundLimitedFilter)
pointKeyFilters, internalOpts.boundLimitedFilter)
}
if err != nil {
c.unrefValue(v)
return nil, nil, err
}
type iterCreator interface {
NewRawRangeDelIter() (keyspan.FragmentIterator, error)
NewIterWithBlockPropertyFiltersAndContextEtc(ctx context.Context, lower, upper []byte, filterer *sstable.BlockPropertiesFilterer, hideObsoletePoints, useFilterBlock bool, stats *base.InternalIteratorStats, rp sstable.ReaderProvider) (sstable.Iterator, error)
NewCompactionIter(
bytesIterated *uint64,
rp sstable.ReaderProvider,
) (sstable.Iterator, error)
}
// TODO(bananabrick): We suffer an allocation if file is a virtual sstable.
var ic iterCreator = v.reader
if file.Virtual {
virtualReader := sstable.MakeVirtualReader(
v.reader, file.VirtualMeta(),
)
ic = &virtualReader
}
provider := dbOpts.objProvider
// Check if this file is a foreign file.
objMeta, err := provider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
@@ -458,31 +478,12 @@ func (c *tableCacheShard) newIters(
return nil, nil, err
}
// Note: This suffers an allocation for virtual sstables.
cr := createCommonReader(v, file, provider.IsSharedForeign(objMeta))
// NB: range-del iterator does not maintain a reference to the table, nor
// does it need to read from it after creation.
var rangeDelIter keyspan.FragmentIterator
if provider.IsForeign(objMeta) {
if opts == nil {
panic("unexpected nil opts when reading foreign file")
}
if !file.Virtual {
// Foreign sstables must be virtual by definition.
panic(fmt.Sprintf("sstable is foreign but not virtual: %s", file.FileNum))
}
switch manifest.LevelToInt(opts.level) {
case 5:
rangeDelIter, err = ic.NewRawRangeDelIter()
case 6:
// Let rangeDelIter remain nil. We don't need to return rangedels from
// this file as they will not apply to any other files. For the purpose
// of collapsing rangedels within this file, we create another rangeDelIter
// below for use with the interleaving iter.
default:
panic(fmt.Sprintf("unexpected level for foreign sstable: %d", manifest.LevelToInt(opts.level)))
}
} else {
rangeDelIter, err = ic.NewRawRangeDelIter()
}
rangeDelIter, err := cr.NewRawRangeDelIter()
if err != nil {
c.unrefValue(v)
return nil, nil, err
@@ -519,10 +520,16 @@ func (c *tableCacheShard) newIters(
rp = &tableCacheShardReaderProvider{c: c, file: file, dbOpts: dbOpts}
}
if provider.IsSharedForeign(objMeta) {
if tableFormat < sstable.TableFormatPebblev4 {
return nil, nil, errors.New("pebble: shared foreign sstable has a lower table format than expected")
}
hideObsoletePoints = true
}
if internalOpts.bytesIterated != nil {
iter, err = ic.NewCompactionIter(internalOpts.bytesIterated, rp)
iter, err = cr.NewCompactionIter(internalOpts.bytesIterated, rp, internalOpts.bufferPool)
} else {
iter, err = ic.NewIterWithBlockPropertyFiltersAndContextEtc(
iter, err = cr.NewIterWithBlockPropertyFiltersAndContextEtc(
ctx, opts.GetLowerBound(), opts.GetUpperBound(), filterer, hideObsoletePoints, useFilter,
internalOpts.stats, rp)
}
@@ -536,35 +543,6 @@ func (c *tableCacheShard) newIters(
// NB: v.closeHook takes responsibility for calling unrefValue(v) here. Take
// care to avoid introducing an allocation here by adding a closure.
iter.SetCloseHook(v.closeHook)
if provider.IsForeign(objMeta) {
// NB: IsForeign() guarantees IsShared, so opts must not be nil as we've
// already panicked on the nil case above.
pointKeySeqNum := base.SeqNumForLevel(manifest.LevelToInt(opts.level))
pcIter := pointCollapsingIterator{
comparer: dbOpts.opts.Comparer,
merge: dbOpts.opts.Merge,
seqNum: math.MaxUint64,
elideRangeDeletes: true,
fixedSeqNum: pointKeySeqNum,
}
// Open a second rangedel iter. This is solely for the interleaving iter to
// be able to efficiently delete covered range deletes. We don't need to fix
// the sequence number in this iter, as these range deletes will not be
// exposed to anything other than the interleaving iter and
// pointCollapsingIter.
rangeDelIter, err := v.reader.NewRawRangeDelIter()
if err != nil {
c.unrefValue(v)
return nil, nil, err
}
if rangeDelIter == nil {
rangeDelIter = emptyKeyspanIter
}
pcIter.iter.Init(dbOpts.opts.Comparer, iter, rangeDelIter, nil /* mask */, opts.LowerBound, opts.UpperBound)
pcSSTIter := pcSSTIterPool.Get().(*pointCollapsingSSTIterator)
*pcSSTIter = pointCollapsingSSTIterator{pointCollapsingIterator: pcIter, childIter: iter}
iter = pcSSTIter
}
c.iterCount.Add(1)
dbOpts.iterCount.Add(1)
@@ -612,10 +590,15 @@ func (c *tableCacheShard) newRangeKeyIter(
var iter keyspan.FragmentIterator
if file.Virtual {
virtualReader := sstable.MakeVirtualReader(
v.reader, file.VirtualMeta(),
)
iter, err = virtualReader.NewRawRangeKeyIter()
provider := dbOpts.objProvider
var objMeta objstorage.ObjectMetadata
objMeta, err = provider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
if err == nil {
virtualReader := sstable.MakeVirtualReader(
v.reader, file.VirtualMeta(), provider.IsSharedForeign(objMeta),
)
iter, err = virtualReader.NewRawRangeKeyIter()
}
} else {
iter, err = v.reader.NewRawRangeKeyIter()
}
@@ -634,29 +617,6 @@ func (c *tableCacheShard) newRangeKeyIter(
return emptyKeyspanIter, nil
}
objMeta, err := dbOpts.objProvider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
if err != nil {
return nil, err
}
if dbOpts.objProvider.IsForeign(objMeta) {
if opts.Level == 0 {
panic("unexpected zero level when reading foreign file")
}
transform := &rangekey.ForeignSSTTransformer{
Comparer: dbOpts.opts.Comparer,
Level: manifest.LevelToInt(opts.Level),
}
if iter == nil {
iter = emptyKeyspanIter
}
transformIter := &keyspan.TransformerIter{
FragmentIterator: iter,
Transformer: transform,
Compare: dbOpts.opts.Comparer.Compare,
}
return transformIter, nil
}
return iter, nil
}
@@ -783,6 +743,29 @@ func (c *tableCacheShard) unrefValue(v *tableCacheValue) {
// that node if it didn't already exist. The caller is responsible for
// decrementing the returned node's refCount.
func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) *tableCacheValue {
v := c.findNodeInternal(meta, dbOpts)
// Loading a file before its global sequence number is known (eg,
// during ingest before entering the commit pipeline) can pollute
// the cache with incorrect state. In invariant builds, verify
// that the global sequence number of the returned reader matches.
if invariants.Enabled {
if v.reader != nil && meta.LargestSeqNum == meta.SmallestSeqNum &&
v.reader.Properties.GlobalSeqNum != meta.SmallestSeqNum {
panic(errors.AssertionFailedf("file %s loaded from table cache with the wrong global sequence number %d",
meta, v.reader.Properties.GlobalSeqNum))
}
}
return v
}
func (c *tableCacheShard) findNodeInternal(
meta *fileMetadata, dbOpts *tableCacheOpts,
) *tableCacheValue {
if refs := meta.Refs(); refs <= 0 {
panic(errors.AssertionFailedf("attempting to load file %s with refs=%d from table cache",
meta, refs))
}
// Fast-path for a hit in the cache.
c.mu.RLock()
key := tableCacheKey{dbOpts.cacheID, meta.FileBacking.DiskFileNum}
@@ -1108,20 +1091,16 @@ func (v *tableCacheValue) load(loadInfo loadInfo, c *tableCacheShard, dbOpts *ta
f, err = dbOpts.objProvider.OpenForReading(
context.TODO(), fileTypeTable, loadInfo.backingFileNum, objstorage.OpenOptions{MustExist: true},
)
if err != nil {
v.err = errors.Wrap(
err,
fmt.Sprintf("pebble: could not open backing file %s", errors.Safe(loadInfo.backingFileNum.FileNum())),
)
}
if v.err == nil {
if err == nil {
cacheOpts := private.SSTableCacheOpts(dbOpts.cacheID, loadInfo.backingFileNum).(sstable.ReaderOption)
v.reader, v.err = sstable.NewReader(f, dbOpts.opts, cacheOpts, dbOpts.filterMetrics)
v.reader, err = sstable.NewReader(f, dbOpts.opts, cacheOpts, dbOpts.filterMetrics)
}
if v.err == nil {
if loadInfo.smallestSeqNum == loadInfo.largestSeqNum {
v.reader.Properties.GlobalSeqNum = loadInfo.largestSeqNum
}
if err != nil {
v.err = errors.Wrapf(
err, "pebble: backing file %s error", errors.Safe(loadInfo.backingFileNum.FileNum()))
}
if v.err == nil && loadInfo.smallestSeqNum == loadInfo.largestSeqNum {
v.reader.Properties.GlobalSeqNum = loadInfo.largestSeqNum
}
if v.err != nil {
c.mu.Lock()

104
vendor/github.com/cockroachdb/pebble/table_stats.go generated vendored
View File

@@ -186,14 +186,9 @@ func (d *DB) loadNewFileStats(
continue
}
if nf.Meta.Virtual {
// cannot load virtual table stats
continue
}
stats, newHints, err := d.loadTableStats(
rs.current, nf.Level,
nf.Meta.PhysicalMeta(),
nf.Meta,
)
if err != nil {
d.opts.EventListener.BackgroundError(err)
@@ -222,12 +217,6 @@ func (d *DB) scanReadStateTableStats(
for l, levelMetadata := range rs.current.Levels {
iter := levelMetadata.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
// TODO(bananabrick): Support stats collection for virtual
// sstables.
continue
}
// NB: We're not holding d.mu which protects f.Stats, but only the
// active stats collection job updates f.Stats for active files,
// and we ensure only one goroutine runs it at a time through
@@ -236,11 +225,6 @@ func (d *DB) scanReadStateTableStats(
if f.StatsValid() {
continue
}
// TODO(bilal): Remove this guard when table stats collection is
// implemented for virtual sstables.
if f.Virtual {
continue
}
// Limit how much work we do per read state. The older the read
// state is, the higher the likelihood files are no longer being
@@ -253,7 +237,7 @@ func (d *DB) scanReadStateTableStats(
}
stats, newHints, err := d.loadTableStats(
rs.current, l, f.PhysicalMeta(),
rs.current, l, f,
)
if err != nil {
// Set `moreRemain` so we'll try again.
@@ -271,25 +255,22 @@ func (d *DB) scanReadStateTableStats(
return fill, hints, moreRemain
}
// loadTableStats currently only supports stats collection for physical
// sstables.
//
// TODO(bananabrick): Support stats collection for virtual sstables.
func (d *DB) loadTableStats(
v *version, level int, meta physicalMeta,
v *version, level int, meta *fileMetadata,
) (manifest.TableStats, []deleteCompactionHint, error) {
var stats manifest.TableStats
var compactionHints []deleteCompactionHint
err := d.tableCache.withReader(
meta, func(r *sstable.Reader) (err error) {
stats.NumEntries = r.Properties.NumEntries
stats.NumDeletions = r.Properties.NumDeletions
if r.Properties.NumPointDeletions() > 0 {
if err = d.loadTablePointKeyStats(r, v, level, meta, &stats); err != nil {
err := d.tableCache.withCommonReader(
meta, func(r sstable.CommonReader) (err error) {
props := r.CommonProperties()
stats.NumEntries = props.NumEntries
stats.NumDeletions = props.NumDeletions
if props.NumPointDeletions() > 0 {
if err = d.loadTablePointKeyStats(props, v, level, meta, &stats); err != nil {
return
}
}
if r.Properties.NumRangeDeletions > 0 || r.Properties.NumRangeKeyDels > 0 {
if props.NumRangeDeletions > 0 || props.NumRangeKeyDels > 0 {
if compactionHints, err = d.loadTableRangeDelStats(
r, v, level, meta, &stats,
); err != nil {
@@ -299,8 +280,8 @@ func (d *DB) loadTableStats(
// TODO(travers): Once we have real-world data, consider collecting
// additional stats that may provide improved heuristics for compaction
// picking.
stats.NumRangeKeySets = r.Properties.NumRangeKeySets
stats.ValueBlocksSize = r.Properties.ValueBlocksSize
stats.NumRangeKeySets = props.NumRangeKeySets
stats.ValueBlocksSize = props.ValueBlocksSize
return
})
if err != nil {
@@ -312,7 +293,11 @@ func (d *DB) loadTableStats(
// loadTablePointKeyStats calculates the point key statistics for the given
// table. The provided manifest.TableStats are updated.
func (d *DB) loadTablePointKeyStats(
r *sstable.Reader, v *version, level int, meta physicalMeta, stats *manifest.TableStats,
props *sstable.CommonProperties,
v *version,
level int,
meta *fileMetadata,
stats *manifest.TableStats,
) error {
// TODO(jackson): If the file has a wide keyspace, the average
// value size beneath the entire file might not be representative
@@ -320,21 +305,21 @@ func (d *DB) loadTablePointKeyStats(
// We could write the ranges of 'clusters' of point tombstones to
// a sstable property and call averageValueSizeBeneath for each of
// these narrower ranges to improve the estimate.
avgValLogicalSize, compressionRatio, err := d.estimateSizesBeneath(v, level, meta)
avgValLogicalSize, compressionRatio, err := d.estimateSizesBeneath(v, level, meta, props)
if err != nil {
return err
}
stats.PointDeletionsBytesEstimate =
pointDeletionsBytesEstimate(meta.Size, &r.Properties, avgValLogicalSize, compressionRatio)
pointDeletionsBytesEstimate(meta.Size, props, avgValLogicalSize, compressionRatio)
return nil
}
// loadTableRangeDelStats calculates the range deletion and range key deletion
// statistics for the given table.
func (d *DB) loadTableRangeDelStats(
r *sstable.Reader, v *version, level int, meta physicalMeta, stats *manifest.TableStats,
r sstable.CommonReader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats,
) ([]deleteCompactionHint, error) {
iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta.FileMetadata)
iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta)
if err != nil {
return nil, err
}
@@ -423,7 +408,7 @@ func (d *DB) loadTableRangeDelStats(
hintType: hintType,
start: make([]byte, len(start)),
end: make([]byte, len(end)),
tombstoneFile: meta.FileMetadata,
tombstoneFile: meta,
tombstoneLevel: level,
tombstoneLargestSeqNum: s.LargestSeqNum(),
tombstoneSmallestSeqNum: s.SmallestSeqNum(),
@@ -437,12 +422,21 @@ func (d *DB) loadTableRangeDelStats(
}
func (d *DB) estimateSizesBeneath(
v *version, level int, meta physicalMeta,
v *version, level int, meta *fileMetadata, fileProps *sstable.CommonProperties,
) (avgValueLogicalSize, compressionRatio float64, err error) {
// Find all files in lower levels that overlap with meta,
// summing their value sizes and entry counts.
file := meta.FileMetadata
file := meta
var fileSum, keySum, valSum, entryCount uint64
// Include the file itself. This is important because in some instances, the
// computed compression ratio is applied to the tombstones contained within
// `meta` itself. If there are no files beneath `meta` in the LSM, we would
// calculate a compression ratio of 0 which is not accurate for the file's
// own tombstones.
fileSum += file.Size
entryCount += fileProps.NumEntries
keySum += fileProps.RawKeySize
valSum += fileProps.RawValueSize
addPhysicalTableStats := func(r *sstable.Reader) (err error) {
fileSum += file.Size
@@ -459,15 +453,6 @@ func (d *DB) estimateSizesBeneath(
return nil
}
// Include the file itself. This is important because in some instances, the
// computed compression ratio is applied to the tombstones contained within
// `meta` itself. If there are no files beneath `meta` in the LSM, we would
// calculate a compression ratio of 0 which is not accurate for the file's
// own tombstones.
if err = d.tableCache.withReader(meta, addPhysicalTableStats); err != nil {
return 0, 0, err
}
for l := level + 1; l < numLevels; l++ {
overlaps := v.Overlaps(l, d.cmp, meta.Smallest.UserKey,
meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel())
@@ -637,8 +622,9 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
// doesn't require any additional IO and since the number of point
// deletions in the file is low, the error introduced by this crude
// estimate is expected to be small.
avgValSize, compressionRatio := estimatePhysicalSizes(meta.Size, props)
pointEstimate = pointDeletionsBytesEstimate(meta.Size, props, avgValSize, compressionRatio)
commonProps := &props.CommonProperties
avgValSize, compressionRatio := estimatePhysicalSizes(meta.Size, commonProps)
pointEstimate = pointDeletionsBytesEstimate(meta.Size, commonProps, avgValSize, compressionRatio)
}
meta.Stats.NumEntries = props.NumEntries
@@ -652,7 +638,7 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
}
func pointDeletionsBytesEstimate(
fileSize uint64, props *sstable.Properties, avgValLogicalSize, compressionRatio float64,
fileSize uint64, props *sstable.CommonProperties, avgValLogicalSize, compressionRatio float64,
) (estimate uint64) {
if props.NumEntries == 0 {
return 0
@@ -675,7 +661,7 @@ func pointDeletionsBytesEstimate(
// tombstones' encoded values.
//
// For un-sized point tombstones (DELs), we estimate assuming that each
// point tombstone on average covers 1 key and using average vaue sizes.
// point tombstone on average covers 1 key and using average value sizes.
// This is almost certainly an overestimate, but that's probably okay
// because point tombstones can slow range iterations even when they don't
// cover a key.
@@ -739,7 +725,7 @@ func pointDeletionsBytesEstimate(
}
func estimatePhysicalSizes(
fileSize uint64, props *sstable.Properties,
fileSize uint64, props *sstable.CommonProperties,
) (avgValLogicalSize, compressionRatio float64) {
// RawKeySize and RawValueSize are uncompressed totals. Scale according to
// the data size to account for compression, index blocks and metadata
@@ -812,7 +798,7 @@ func estimatePhysicalSizes(
// corresponding to the largest and smallest sequence numbers encountered across
// the range deletes and range keys deletes that comprised the merged spans.
func newCombinedDeletionKeyspanIter(
comparer *base.Comparer, r *sstable.Reader, m *fileMetadata,
comparer *base.Comparer, cr sstable.CommonReader, m *fileMetadata,
) (keyspan.FragmentIterator, error) {
// The range del iter and range key iter are each wrapped in their own
// defragmenting iter. For each iter, abutting spans can always be merged.
@@ -874,7 +860,7 @@ func newCombinedDeletionKeyspanIter(
})
mIter.Init(comparer.Compare, transform, new(keyspan.MergingBuffers))
iter, err := r.NewRawRangeDelIter()
iter, err := cr.NewRawRangeDelIter()
if err != nil {
return nil, err
}
@@ -886,12 +872,12 @@ func newCombinedDeletionKeyspanIter(
// See docs/range_deletions.md for why this is necessary.
iter = keyspan.Truncate(
comparer.Compare, iter, m.Smallest.UserKey, m.Largest.UserKey,
nil, nil, false, /* panicOnPartialOverlap */
nil, nil, false, /* panicOnUpperTruncate */
)
mIter.AddLevel(iter)
}
iter, err = r.NewRawRangeKeyIter()
iter, err = cr.NewRawRangeKeyIter()
if err != nil {
return nil, err
}
@@ -908,7 +894,7 @@ func newCombinedDeletionKeyspanIter(
out.Keys = append(out.Keys, k)
}
return len(out.Keys) > 0
})
}, comparer.Compare)
dIter := &keyspan.DefragmentingIter{}
dIter.Init(comparer, iter, equal, reducer, new(keyspan.DefragmentingBuffers))
iter = dIter

126
vendor/github.com/cockroachdb/pebble/version_set.go generated vendored
View File

@@ -8,7 +8,6 @@ import (
"bytes"
"fmt"
"io"
"math"
"sync"
"sync/atomic"
@@ -61,12 +60,11 @@ type versionSet struct {
// Immutable fields.
dirname string
// Set to DB.mu.
mu *sync.Mutex
opts *Options
fs vfs.FS
cmp Compare
cmpName string
diskAvailBytes func() uint64
mu *sync.Mutex
opts *Options
fs vfs.FS
cmp Compare
cmpName string
// Dynamic base level allows the dynamic base level computation to be
// disabled. Used by tests which want to create specific LSM structures.
dynamicBaseLevel bool
@@ -88,17 +86,21 @@ type versionSet struct {
// still referenced by an inuse iterator.
zombieTables map[base.DiskFileNum]uint64 // filenum -> size
// fileBackingMap is a map for the FileBacking which is supporting virtual
// sstables in the latest version. Once the file backing is backing no
// virtual sstables in the latest version, it is removed from this map and
// the corresponding state is added to the zombieTables map. Note that we
// don't keep track of file backing which supports a virtual sstable
// which is not in the latest version.
//
// fileBackingMap is protected by the versionSet.logLock. It's populated
// backingState is protected by the versionSet.logLock. It's populated
// during Open in versionSet.load, but it's not used concurrently during
// load.
fileBackingMap map[base.DiskFileNum]*fileBacking
backingState struct {
// fileBackingMap is a map for the FileBacking which is supporting virtual
// sstables in the latest version. Once the file backing is backing no
// virtual sstables in the latest version, it is removed from this map and
// the corresponding state is added to the zombieTables map. Note that we
// don't keep track of file backing which supports a virtual sstable
// which is not in the latest version.
fileBackingMap map[base.DiskFileNum]*fileBacking
// fileBackingSize is the sum of the sizes of the fileBackings in the
// fileBackingMap.
fileBackingSize uint64
}
// minUnflushedLogNum is the smallest WAL log file number corresponding to
// mutations that have not been flushed to an sstable.
@@ -112,9 +114,10 @@ type versionSet struct {
manifestFileNum FileNum
manifestMarker *atomicfs.Marker
manifestFile vfs.File
manifest *record.Writer
setCurrent func(FileNum) error
manifestFile vfs.File
manifest *record.Writer
setCurrent func(FileNum) error
getFormatMajorVersion func() FormatMajorVersion
writing bool
writerCond sync.Cond
@@ -127,6 +130,7 @@ func (vs *versionSet) init(
opts *Options,
marker *atomicfs.Marker,
setCurrent func(FileNum) error,
getFMV func() FormatMajorVersion,
mu *sync.Mutex,
) {
vs.dirname = dirname
@@ -140,13 +144,12 @@ func (vs *versionSet) init(
vs.versions.Init(mu)
vs.obsoleteFn = vs.addObsoleteLocked
vs.zombieTables = make(map[base.DiskFileNum]uint64)
vs.fileBackingMap = make(map[base.DiskFileNum]*fileBacking)
vs.backingState.fileBackingMap = make(map[base.DiskFileNum]*fileBacking)
vs.backingState.fileBackingSize = 0
vs.nextFileNum = 1
vs.manifestMarker = marker
vs.setCurrent = setCurrent
if vs.diskAvailBytes == nil {
vs.diskAvailBytes = func() uint64 { return math.MaxUint64 }
}
vs.getFormatMajorVersion = getFMV
}
// create creates a version set for a fresh DB.
@@ -156,15 +159,15 @@ func (vs *versionSet) create(
opts *Options,
marker *atomicfs.Marker,
setCurrent func(FileNum) error,
getFormatMajorVersion func() FormatMajorVersion,
mu *sync.Mutex,
) error {
vs.init(dirname, opts, marker, setCurrent, mu)
vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu)
newVersion := &version{}
vs.append(newVersion)
var err error
vs.picker = newCompactionPicker(newVersion, vs.opts, nil, vs.metrics.levelSizes(), vs.diskAvailBytes)
vs.picker = newCompactionPicker(newVersion, vs.opts, nil)
// Note that a "snapshot" version edit is written to the manifest when it is
// created.
vs.manifestFileNum = vs.getNextFileNum()
@@ -205,9 +208,10 @@ func (vs *versionSet) load(
manifestFileNum FileNum,
marker *atomicfs.Marker,
setCurrent func(FileNum) error,
getFormatMajorVersion func() FormatMajorVersion,
mu *sync.Mutex,
) error {
vs.init(dirname, opts, marker, setCurrent, mu)
vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu)
vs.manifestFileNum = manifestFileNum
manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum.DiskFileNum())
@@ -296,16 +300,17 @@ func (vs *versionSet) load(
// Populate the fileBackingMap and the FileBacking for virtual sstables since
// we have finished version edit accumulation.
for _, s := range bve.AddedFileBacking {
vs.fileBackingMap[s.DiskFileNum] = s
vs.addFileBacking(s)
}
for _, fileNum := range bve.RemovedFileBacking {
delete(vs.fileBackingMap, fileNum)
vs.removeFileBacking(fileNum)
}
newVersion, err := bve.Apply(
nil, vs.cmp, opts.Comparer.FormatKey, opts.FlushSplitBytes,
opts.Experimental.ReadCompactionRate, nil, /* zombies */
getFormatMajorVersion().orderingInvariants(),
)
if err != nil {
return err
@@ -320,7 +325,7 @@ func (vs *versionSet) load(
l.Size = int64(files.SizeSum())
}
vs.picker = newCompactionPicker(newVersion, vs.opts, nil, vs.metrics.levelSizes(), vs.diskAvailBytes)
vs.picker = newCompactionPicker(newVersion, vs.opts, nil)
return nil
}
@@ -363,6 +368,26 @@ func (vs *versionSet) logUnlock() {
vs.writerCond.Signal()
}
// Only call if the DiskFileNum doesn't exist in the fileBackingMap.
func (vs *versionSet) addFileBacking(backing *manifest.FileBacking) {
_, ok := vs.backingState.fileBackingMap[backing.DiskFileNum]
if ok {
panic("pebble: trying to add an existing file backing")
}
vs.backingState.fileBackingMap[backing.DiskFileNum] = backing
vs.backingState.fileBackingSize += backing.Size
}
// Only call if the the DiskFileNum exists in the fileBackingMap.
func (vs *versionSet) removeFileBacking(dfn base.DiskFileNum) {
backing, ok := vs.backingState.fileBackingMap[dfn]
if !ok {
panic("pebble: trying to remove an unknown file backing")
}
delete(vs.backingState.fileBackingMap, dfn)
vs.backingState.fileBackingSize -= backing.Size
}
// logAndApply logs the version edit to the manifest, applies the version edit
// to the current version, and installs the new version.
//
@@ -420,6 +445,8 @@ func (vs *versionSet) logAndApply(
}
currentVersion := vs.currentVersion()
fmv := vs.getFormatMajorVersion()
orderingInvariants := fmv.orderingInvariants()
var newVersion *version
// Generate a new manifest if we don't currently have one, or forceRotation
@@ -490,13 +517,14 @@ func (vs *versionSet) logAndApply(
defer vs.mu.Lock()
var err error
if vs.opts.FormatMajorVersion < ExperimentalFormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 {
if vs.getFormatMajorVersion() < FormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 {
return errors.AssertionFailedf("MANIFEST cannot contain virtual sstable records due to format major version")
}
newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE(
ve, currentVersion, vs.cmp, vs.opts.Comparer.FormatKey,
vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate,
vs.fileBackingMap,
vs.backingState.fileBackingMap, vs.addFileBacking, vs.removeFileBacking,
orderingInvariants,
)
if err != nil {
return errors.Wrap(err, "MANIFEST apply failed")
@@ -572,14 +600,6 @@ func (vs *versionSet) logAndApply(
for fileNum, size := range zombies {
vs.zombieTables[fileNum] = size
}
// Update the Deleted bools. We can't use the zombieTables struct for this
// as it works on FileBackings, not FileMetadatas.
for _, f := range ve.DeletedFiles {
f.Deleted = true
}
for i := range ve.NewFiles {
ve.NewFiles[i].Meta.Deleted = false
}
// Install the new version.
vs.append(newVersion)
@@ -601,30 +621,46 @@ func (vs *versionSet) logAndApply(
}
for i := range vs.metrics.Levels {
l := &vs.metrics.Levels[i]
l.NumFiles = int64(newVersion.Levels[i].Len())
l.NumVirtualFiles = newVersion.Levels[i].NumVirtual
l.VirtualSize = newVersion.Levels[i].VirtualSize
l.Size = int64(newVersion.Levels[i].Size())
l.Sublevels = 0
if l.NumFiles > 0 {
l.Sublevels = 1
}
if invariants.Enabled {
if count := int64(newVersion.Levels[i].Len()); l.NumFiles != count {
vs.opts.Logger.Fatalf("versionSet metrics L%d NumFiles = %d, actual count = %d", i, l.NumFiles, count)
}
levelFiles := newVersion.Levels[i].Slice()
if size := int64(levelFiles.SizeSum()); l.Size != size {
vs.opts.Logger.Fatalf("versionSet metrics L%d Size = %d, actual size = %d", i, l.Size, size)
}
if nVirtual := levelFiles.NumVirtual(); nVirtual != l.NumVirtualFiles {
vs.opts.Logger.Fatalf(
"versionSet metrics L%d NumVirtual = %d, actual NumVirtual = %d",
i, l.NumVirtualFiles, nVirtual,
)
}
if vSize := levelFiles.VirtualSizeSum(); vSize != l.VirtualSize {
vs.opts.Logger.Fatalf(
"versionSet metrics L%d Virtual size = %d, actual size = %d",
i, l.VirtualSize, vSize,
)
}
}
}
vs.metrics.Levels[0].Sublevels = int32(len(newVersion.L0SublevelFiles))
vs.picker = newCompactionPicker(newVersion, vs.opts, inProgress, vs.metrics.levelSizes(), vs.diskAvailBytes)
vs.picker = newCompactionPicker(newVersion, vs.opts, inProgress)
if !vs.dynamicBaseLevel {
vs.picker.forceBaseLevel1()
}
return nil
}
func (vs *versionSet) incrementCompactions(kind compactionKind, extraLevels []*compactionLevel) {
func (vs *versionSet) incrementCompactions(
kind compactionKind, extraLevels []*compactionLevel, pickerMetrics compactionPickerMetrics,
) {
switch kind {
case compactionKindDefault:
vs.metrics.Compact.Count++

Some files were not shown because too many files have changed in this diff Show More