aleesa-bot/aleesa-telegram-go
0
Fork 0
Code Activity

[WIP] lint a bit, update deps

Browse Source
This commit is contained in:
eleksir
2024-02-27 00:34:44 +03:00
parent 1892960896
commit 1f97424f31
201 changed files with 21474 additions and 9282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
vendor/github.com/NicoNex/echotron/v3/types.go generated vendored
View File

@@ -489,11 +489,13 @@ type Chat struct {
InviteLink string `json:"invite_link,omitempty"`
EmojiStatusCustomEmojiID string `json:"emoji_status_custom_emoji_id,omitempty"`
Type string `json:"type"`
CustomEmojiStickerSetName string `json:"custom_emoji_sticker_set_name,omitempty"`
AccentColorID int `json:"accent_color_id,omitempty"`
ProfileAccentColorID int `json:"profile_accent_color_id,omitempty"`
EmojiStatusExpirationDate int `json:"emoji_status_expiration_date,omitempty"`
MessageAutoDeleteTime int `json:"message_auto_delete_time,omitempty"`
SlowModeDelay int `json:"slow_mode_delay,omitempty"`
UnrestrictBoostCount int `json:"unrestrict_boost_count,omitempty"`
LinkedChatID int64 `json:"linked_chat_id,omitempty"`
ID int64 `json:"id"`
IsForum bool `json:"is_forum,omitempty"`
@@ -553,10 +555,12 @@ type Message struct {
UsersShared *UsersShared `json:"users_shared,omitempty"`
ChatShared *ChatShared `json:"chat_shared,omitempty"`
Story *Story `json:"story,omitempty"`
ReplyToStory *Story `json:"reply_to_story,omitempty"`
ExternalReply *ExternalReplyInfo `json:"external_reply,omitempty"`
Quote *TextQuote `json:"quote,omitempty"`
LinkPreviewOptions *LinkPreviewOptions `json:"link_preview_options,omitempty"`
ForwardOrigin *MessageOrigin `json:"forward_origin,omitempty"`
BoostAdded *ChatBoostAdded `json:"boost_added,omitempty"`
MediaGroupID string `json:"media_group_id,omitempty"`
ConnectedWebsite string `json:"connected_website,omitempty"`
NewChatTitle string `json:"new_chat_title,omitempty"`
@@ -575,6 +579,7 @@ type Message struct {
Date int `json:"date"`
MigrateToChatID int `json:"migrate_to_chat_id,omitempty"`
EditDate int `json:"edit_date,omitempty"`
SenderBoostCount int `json:"sender_boost_count,omitempty"`
DeleteChatPhoto bool `json:"delete_chat_photo,omitempty"`
IsTopicMessage bool `json:"is_topic_message,omitempty"`
IsAutomaticForward bool `json:"is_automatic_forward,omitempty"`
@@ -1173,6 +1178,11 @@ type ChatJoinRequest struct {
UserChatID int64 `json:"user_chat_id"`
}
// ChatBoostAdded represents a service message about a user boosting a chat.
type ChatBoostAdded struct {
BoostCount int `json:"boost_count"`
}
// ForumTopicCreated represents a service message about a new forum topic created in the chat.
type ForumTopicCreated struct {
Name string `json:"name"`
@@ -1238,9 +1248,11 @@ type ChatShared struct {
ChatID int64 `json:"chat_id"`
}
// Story represents a message about a forwarded story in the chat.
// Currently holds no information.
type Story struct{}
// Story represents a story.
type Story struct {
Chat Chat `json:"chat"`
ID int64 `json:"id"`
}
type ReactionType interface {
ImplementsReactionType()

3
vendor/github.com/cockroachdb/pebble/.gitignore generated vendored
View File

@@ -1,6 +1,9 @@
# Github action artifacts.
artifacts
# Profiling artifacts.
cpu.*.prof
heap.prof
mutex.prof
coverprofile.out
# Testing artifacts
meta.*.test

37
vendor/github.com/cockroachdb/pebble/Makefile generated vendored
View File

@@ -4,7 +4,8 @@ GOFLAGS :=
STRESSFLAGS :=
TAGS := invariants
TESTS := .
LATEST_RELEASE := $(shell git fetch origin && git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$$' | sort | tail -1)
PREV_RELEASE := crl-release-23.1
COVER_PROFILE := coverprofile.out
.PHONY: all
all:
@@ -15,7 +16,10 @@ all:
@echo " make stressrace"
@echo " make stressmeta"
@echo " make crossversion-meta"
@echo " make testcoverage"
@echo " make mod-update"
@echo " make generate"
@echo " make generate-test-data"
@echo " make clean"
override testflags :=
@@ -23,6 +27,10 @@ override testflags :=
test:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG}
.PHONY: testcoverage
testcoverage:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} -coverprofile ${COVER_PROFILE}
.PHONY: testrace
testrace: testflags += -race -timeout 20m
testrace: test
@@ -34,6 +42,10 @@ testmsan: export CC=clang
testmsan: testflags += -msan -timeout 20m
testmsan: test
.PHONY: testobjiotracing
testobjiotracing:
${GO} test -tags '$(TAGS) pebble_obj_io_tracing' ${testflags} -run ${TESTS} ./objstorage/objstorageprovider/objiotracing
.PHONY: lint
lint:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ./internal/lint
@@ -51,20 +63,31 @@ stressmeta: stress
.PHONY: crossversion-meta
crossversion-meta:
git checkout ${LATEST_RELEASE}; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \
git checkout ${PREV_RELEASE}; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${PREV_RELEASE}.test'; \
git checkout -; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${PREV_RELEASE},${PREV_RELEASE},${PREV_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
.PHONY: stress-crossversion
stress-crossversion:
STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 master
STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 crl-release-23.1 master
.PHONY: generate
generate:
${GO} generate ${PKG}
generate:
# Note that the output of generate-test-data is not deterministic. This should
# only be run manually as needed.
.PHONY: generate-test-data
generate-test-data:
${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
${GO} run -tags make_test_find_db ./tool/make_test_find_db.go
${GO} run -tags make_test_sstables ./tool/make_test_sstables.go
${GO} run -tags make_test_remotecat ./tool/make_test_remotecat.go
mod-update:
${GO} get -u
${GO} mod tidy
@@ -92,9 +115,11 @@ endif
@${GO} mod tidy
$(MAKE) git-clean-check
# TODO(radu): switch back to @latest once bogus doc changes are
# addressed; see https://github.com/cockroachdb/crlfmt/pull/44
.PHONY: format
format:
go install github.com/cockroachdb/crlfmt@latest && crlfmt -w -tab 2 .
go install github.com/cockroachdb/crlfmt@44a36ec7 && crlfmt -w -tab 2 .
.PHONY: format-check
format-check:

84
vendor/github.com/cockroachdb/pebble/README.md generated vendored
View File

@@ -1,4 +1,4 @@
# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble)
# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) <sup><sub><sub>[Coverage](https://storage.googleapis.com/crl-codecover-public/pebble/index.html)</sub></sub></sup>
#### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
@@ -8,9 +8,8 @@ RocksDB file formats and a few extensions such as range deletion
tombstones, table-level bloom filters, and updates to the MANIFEST
format.
Pebble intentionally does not aspire to include every feature in
RocksDB and is specifically targetting the use case and feature set
needed by CockroachDB:
Pebble intentionally does not aspire to include every feature in RocksDB and
specifically targets the use case and feature set needed by CockroachDB:
* Block-based tables
* Checkpoints
@@ -68,6 +67,17 @@ Pebble offers several improvements over RocksDB:
* Faster commit pipeline that achieves better concurrency.
* Seamless merged iteration of indexed batches. The mutations in the
batch conceptually occupy another memtable level.
* L0 sublevels and flush splitting for concurrent compactions out of L0 and
reduced read-amplification during heavy write load.
* Faster LSM edits in LSMs with large numbers of sstables through use of a
copy-on-write B-tree to hold file metadata.
* Delete-only compactions that drop whole sstables that fall within the bounds
of a range deletion.
* Block-property collectors and filters that enable iterators to skip tables,
index blocks and data blocks that are irrelevant, according to user-defined
properties over key-value pairs.
* Range keys API, allowing KV pairs defined over a range of keyspace with
user-defined semantics and interleaved during iteration.
* Smaller, more approachable code base.
See the [Pebble vs RocksDB: Implementation
@@ -100,10 +110,68 @@ known incompatibilities.
format.
* Plain table format. Pebble does not support the plain table sstable
format.
* SSTable format version 3 and 4. Pebble does not currently support
version 3 and version 4 format sstables. The sstable format version
is controlled by the `BlockBasedTableOptions::format_version`
option. See [#97](https://github.com/cockroachdb/pebble/issues/97).
* SSTable format version 3 and 4. Pebble does not support version 3
and version 4 format sstables. The sstable format version is
controlled by the `BlockBasedTableOptions::format_version` option.
See [#97](https://github.com/cockroachdb/pebble/issues/97).
## Format major versions
Over time Pebble has introduced new physical file formats. Backwards
incompatible changes are made through the introduction of 'format major
versions'. By default, when Pebble opens a database, it defaults to
`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB
6.2.1 (with the caveats described above).
To opt into new formats, a user may set `FormatMajorVersion` on the
[`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options)
supplied to
[`Open`](https://pkg.go.dev/github.com/cockroachdb/pebble#Open), or
upgrade the format major version at runtime using
[`DB.RatchetFormatMajorVersion`](https://pkg.go.dev/github.com/cockroachdb/pebble#DB.RatchetFormatMajorVersion).
Format major version upgrades are permanent; There is no option to
return to an earlier format.
The table below outlines the history of format major versions:
| Name | Value | Migration |
|------------------------------------|-------|------------|
| FormatMostCompatible | 1 | No |
| FormatVersioned | 3 | No |
| FormatSetWithDelete | 4 | No |
| FormatBlockPropertyCollector | 5 | No |
| FormatSplitUserKeysMarked | 6 | Background |
| FormatSplitUserKeysMarkedCompacted | 7 | Blocking |
| FormatRangeKeys | 8 | No |
| FormatMinTableFormatPebblev1 | 9 | No |
| FormatPrePebblev1Marked | 10 | Background |
| FormatSSTableValueBlocks | 12 | No |
| FormatFlushableIngest | 13 | No |
| FormatPrePebblev1MarkedCompacted | 14 | Blocking |
| FormatDeleteSizedAndObsolete | 15 | No |
| FormatVirtualSSTables | 16 | No |
Upgrading to a format major version with 'Background' in the migration
column may trigger background activity to rewrite physical file
formats, typically through compactions. Upgrading to a format major
version with 'Blocking' in the migration column will block until a
migration is complete. The database may continue to serve reads and
writes if upgrading a live database through
`RatchetFormatMajorVersion`, but the method call will not return until
the migration is complete.
For reference, the table below lists the range of supported Pebble format major
versions for CockroachDB releases.
| CockroachDB release | Earliest supported | Latest supported |
|---------------------|------------------------------------|---------------------------|
| 20.1 through 21.1 | FormatMostCompatible | FormatMostCompatible |
| 21.2 | FormatMostCompatible | FormatSetWithDelete |
| 21.2 | FormatMostCompatible | FormatSetWithDelete |
| 22.1 | FormatMostCompatible | FormatSplitUserKeysMarked |
| 22.2 | FormatMostCompatible | FormatPrePebblev1Marked |
| 23.1 | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest |
| 23.2 | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables |
## Pedigree

317
vendor/github.com/cockroachdb/pebble/batch.go generated vendored
View File

@@ -41,10 +41,10 @@ const (
var ErrNotIndexed = errors.New("pebble: batch not indexed")
// ErrInvalidBatch indicates that a batch is invalid or otherwise corrupted.
var ErrInvalidBatch = errors.New("pebble: invalid batch")
var ErrInvalidBatch = base.MarkCorruptionError(errors.New("pebble: invalid batch"))
// ErrBatchTooLarge indicates that a batch is invalid or otherwise corrupted.
var ErrBatchTooLarge = errors.Newf("pebble: batch too large: >= %s", humanize.Uint64(maxBatchSize))
var ErrBatchTooLarge = base.MarkCorruptionError(errors.Newf("pebble: batch too large: >= %s", humanize.Bytes.Uint64(maxBatchSize)))
// DeferredBatchOp represents a batch operation (eg. set, merge, delete) that is
// being inserted into the batch. Indexing is not performed on the specified key
@@ -183,6 +183,13 @@ func (d DeferredBatchOp) Finish() error {
// WAL, and thus stable. New record kinds may be added, but the existing ones
// will not be modified.
type Batch struct {
batchInternal
applied atomic.Bool
}
// batchInternal contains the set of fields within Batch that are non-atomic and
// capable of being reset using a *b = batchInternal{} struct copy.
type batchInternal struct {
// Data is the wire format of a batch's log entry:
// - 8 bytes for a sequence number of the first batch element,
// or zeroes if the batch has not yet been applied,
@@ -260,10 +267,12 @@ type Batch struct {
// memtable.
flushable *flushableBatch
// ingestedSSTBatch indicates that the batch contains one or more key kinds
// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
// then it will only contain key kinds of IngestSST.
ingestedSSTBatch bool
// minimumFormatMajorVersion indicates the format major version required in
// order to commit this batch. If an operation requires a particular format
// major version, it ratchets the batch's minimumFormatMajorVersion. When
// the batch is committed, this is validated against the database's current
// format major version.
minimumFormatMajorVersion FormatMajorVersion
// Synchronous Apply uses the commit WaitGroup for both publishing the
// seqnum and waiting for the WAL fsync (if needed). Asynchronous
@@ -285,7 +294,22 @@ type Batch struct {
commitStats BatchCommitStats
commitErr error
applied uint32 // updated atomically
// Position bools together to reduce the sizeof the struct.
// ingestedSSTBatch indicates that the batch contains one or more key kinds
// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
// then it will only contain key kinds of IngestSST.
ingestedSSTBatch bool
// committing is set to true when a batch begins to commit. It's used to
// ensure the batch is not mutated concurrently. It is not an atomic
// deliberately, so as to avoid the overhead on batch mutations. This is
// okay, because under correct usage this field will never be accessed
// concurrently. It's only under incorrect usage the memory accesses of this
// variable may violate memory safety. Since we don't use atomics here,
// false negatives are possible.
committing bool
}
// BatchCommitStats exposes stats related to committing a batch.
@@ -312,7 +336,11 @@ type BatchCommitStats struct {
// commitPipeline.Commit.
SemaphoreWaitDuration time.Duration
// WALQueueWaitDuration is the wait time for allocating memory blocks in the
// LogWriter (due to the LogWriter not writing fast enough).
// LogWriter (due to the LogWriter not writing fast enough). At the moment
// this is duration is always zero because a single WAL will allow
// allocating memory blocks up to the entire memtable size. In the future,
// we may pipeline WALs and bound the WAL queued blocks separately, so this
// field is preserved for that possibility.
WALQueueWaitDuration time.Duration
// MemTableWriteStallDuration is the wait caused by a write stall due to too
// many memtables (due to not flushing fast enough).
@@ -356,6 +384,14 @@ func newBatch(db *DB) *Batch {
return b
}
func newBatchWithSize(db *DB, size int) *Batch {
b := newBatch(db)
if cap(b.data) < size {
b.data = rawalloc.New(0, size)
}
return b
}
func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
i := indexedBatchPool.Get().(*indexedBatch)
i.batch.cmp = comparer.Compare
@@ -367,6 +403,14 @@ func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
return &i.batch
}
func newIndexedBatchWithSize(db *DB, comparer *Comparer, size int) *Batch {
b := newIndexedBatch(db, comparer)
if cap(b.data) < size {
b.data = rawalloc.New(0, size)
}
return b
}
// nextSeqNum returns the batch "sequence number" that will be given to the next
// key written to the batch. During iteration keys within an indexed batch are
// given a sequence number consisting of their offset within the batch combined
@@ -405,17 +449,21 @@ func (b *Batch) release() {
}
}
func (b *Batch) refreshMemTableSize() {
func (b *Batch) refreshMemTableSize() error {
b.memTableSize = 0
if len(b.data) < batchHeaderLen {
return
return nil
}
b.countRangeDels = 0
b.countRangeKeys = 0
b.minimumFormatMajorVersion = 0
for r := b.Reader(); ; {
kind, key, value, ok := r.Next()
kind, key, value, ok, err := r.Next()
if !ok {
if err != nil {
return err
}
break
}
switch kind {
@@ -423,12 +471,23 @@ func (b *Batch) refreshMemTableSize() {
b.countRangeDels++
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
b.countRangeKeys++
case InternalKeyKindDeleteSized:
if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
}
case InternalKeyKindIngestSST:
if b.minimumFormatMajorVersion < FormatFlushableIngest {
b.minimumFormatMajorVersion = FormatFlushableIngest
}
// This key kind doesn't contribute to the memtable size.
continue
}
b.memTableSize += memTableEntrySize(len(key), len(value))
}
if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys {
b.minimumFormatMajorVersion = FormatRangeKeys
}
return nil
}
// Apply the operations contained in the batch to the receiver batch.
@@ -442,7 +501,7 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
return nil
}
if len(batch.data) < batchHeaderLen {
return base.CorruptionErrorf("pebble: invalid batch")
return ErrInvalidBatch
}
offset := len(b.data)
@@ -459,8 +518,11 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
// order to update the index.
for iter := BatchReader(b.data[offset:]); len(iter) > 0; {
offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
kind, key, value, ok := iter.Next()
kind, key, value, ok, err := iter.Next()
if !ok {
if err != nil {
return err
}
break
}
switch kind {
@@ -516,6 +578,9 @@ func (b *Batch) Get(key []byte) ([]byte, io.Closer, error) {
}
func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind InternalKeyKind) {
if b.committing {
panic("pebble: batch already committing")
}
if len(b.data) == 0 {
b.init(keyLen + valueLen + 2*binary.MaxVarintLen64 + batchHeaderLen)
}
@@ -565,6 +630,9 @@ func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind Interna
}
func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
if b.committing {
panic("pebble: batch already committing")
}
if len(b.data) == 0 {
b.init(keyLen + binary.MaxVarintLen64 + batchHeaderLen)
}
@@ -598,6 +666,41 @@ func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
b.data = b.data[:pos+keyLen]
}
// AddInternalKey allows the caller to add an internal key of point key kinds to
// a batch. Passing in an internal key of kind RangeKey* or RangeDelete will
// result in a panic. Note that the seqnum in the internal key is effectively
// ignored, even though the Kind is preserved. This is because the batch format
// does not allow for a per-key seqnum to be specified, only a batch-wide one.
//
// Note that non-indexed keys (IngestKeyKind{LogData,IngestSST}) are not
// supported with this method as they require specialized logic.
func (b *Batch) AddInternalKey(key *base.InternalKey, value []byte, _ *WriteOptions) error {
keyLen := len(key.UserKey)
hasValue := false
switch key.Kind() {
case InternalKeyKindRangeDelete, InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
panic("unexpected range delete or range key kind in AddInternalKey")
case InternalKeyKindSingleDelete, InternalKeyKindDelete:
b.prepareDeferredKeyRecord(len(key.UserKey), key.Kind())
default:
b.prepareDeferredKeyValueRecord(keyLen, len(value), key.Kind())
hasValue = true
}
b.deferredOp.index = b.index
copy(b.deferredOp.Key, key.UserKey)
if hasValue {
copy(b.deferredOp.Value, value)
}
// TODO(peter): Manually inline DeferredBatchOp.Finish(). Mid-stack inlining
// in go1.13 will remove the need for this.
if b.index != nil {
if err := b.index.Add(b.deferredOp.offset); err != nil {
return err
}
}
return nil
}
// Set adds an action to the batch that sets the key to map to the value.
//
// It is safe to modify the contents of the arguments after Set returns.
@@ -680,6 +783,72 @@ func (b *Batch) DeleteDeferred(keyLen int) *DeferredBatchOp {
return &b.deferredOp
}
// DeleteSized behaves identically to Delete, but takes an additional
// argument indicating the size of the value being deleted. DeleteSized
// should be preferred when the caller has the expectation that there exists
// a single internal KV pair for the key (eg, the key has not been
// overwritten recently), and the caller knows the size of its value.
//
// DeleteSized will record the value size within the tombstone and use it to
// inform compaction-picking heuristics which strive to reduce space
// amplification in the LSM. This "calling your shot" mechanic allows the
// storage engine to more accurately estimate and reduce space amplification.
//
// It is safe to modify the contents of the arguments after DeleteSized
// returns.
func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions) error {
deferredOp := b.DeleteSizedDeferred(len(key), deletedValueSize)
copy(b.deferredOp.Key, key)
// TODO(peter): Manually inline DeferredBatchOp.Finish(). Check if in a
// later Go release this is unnecessary.
if b.index != nil {
if err := b.index.Add(deferredOp.offset); err != nil {
return err
}
}
return nil
}
// DeleteSizedDeferred is similar to DeleteSized in that it adds a sized delete
// operation to the batch, except it only takes in key length instead of a
// complete key slice, letting the caller encode into the DeferredBatchOp.Key
// slice and then call Finish() on the returned object.
func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp {
if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
}
// Encode the sum of the key length and the value in the value.
v := uint64(deletedValueSize) + uint64(keyLen)
// Encode `v` as a varint.
var buf [binary.MaxVarintLen64]byte
n := 0
{
x := v
for x >= 0x80 {
buf[n] = byte(x) | 0x80
x >>= 7
n++
}
buf[n] = byte(x)
n++
}
// NB: In batch entries and sstable entries, values are stored as
// varstrings. Here, the value is itself a simple varint. This results in an
// unnecessary double layer of encoding:
// varint(n) varint(deletedValueSize)
// The first varint will always be 1-byte, since a varint-encoded uint64
// will never exceed 128 bytes. This unnecessary extra byte and wrapping is
// preserved to avoid special casing across the database, and in particular
// in sstable block decoding which is performance sensitive.
b.prepareDeferredKeyValueRecord(keyLen, n, InternalKeyKindDeleteSized)
b.deferredOp.index = b.index
copy(b.deferredOp.Value, buf[:n])
return &b.deferredOp
}
// SingleDelete adds an action to the batch that single deletes the entry for key.
// See Writer.SingleDelete for more details on the semantics of SingleDelete.
//
@@ -782,6 +951,9 @@ func (b *Batch) rangeKeySetDeferred(startLen, internalValueLen int) *DeferredBat
func (b *Batch) incrementRangeKeysCount() {
b.countRangeKeys++
if b.minimumFormatMajorVersion < FormatRangeKeys {
b.minimumFormatMajorVersion = FormatRangeKeys
}
if b.index != nil {
b.rangeKeys = nil
b.rangeKeysSeqNum = 0
@@ -895,6 +1067,7 @@ func (b *Batch) ingestSST(fileNum base.FileNum) {
// is not reset because for the InternalKeyKindIngestSST the count is the
// number of sstable paths which have been added to the batch.
b.memTableSize = origMemTableSize
b.minimumFormatMajorVersion = FormatFlushableIngest
}
// Empty returns true if the batch is empty, and false otherwise.
@@ -930,11 +1103,12 @@ func (b *Batch) SetRepr(data []byte) error {
}
b.data = data
b.count = uint64(binary.LittleEndian.Uint32(b.countData()))
var err error
if b.db != nil {
// Only track memTableSize for batches that will be committed to the DB.
b.refreshMemTableSize()
err = b.refreshMemTableSize()
}
return nil
return err
}
// NewIter returns an iterator that is unpositioned (Iterator.Valid() will
@@ -944,8 +1118,8 @@ func (b *Batch) SetRepr(data []byte) error {
// The returned Iterator observes all of the Batch's existing mutations, but no
// later mutations. Its view can be refreshed via RefreshBatchSnapshot or
// SetOptions().
func (b *Batch) NewIter(o *IterOptions) *Iterator {
return b.NewIterWithContext(context.Background(), o)
func (b *Batch) NewIter(o *IterOptions) (*Iterator, error) {
return b.NewIterWithContext(context.Background(), o), nil
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
@@ -954,7 +1128,7 @@ func (b *Batch) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterato
if b.index == nil {
return &Iterator{err: ErrNotIndexed}
}
return b.db.newIter(ctx, b, nil /* snapshot */, o)
return b.db.newIter(ctx, b, snapshotIterOpts{}, o)
}
// newInternalIter creates a new internalIterator that iterates over the
@@ -1167,38 +1341,40 @@ func (b *Batch) Indexed() bool {
return b.index != nil
}
func (b *Batch) init(cap int) {
// init ensures that the batch data slice is initialized to meet the
// minimum required size and allocates space for the batch header.
func (b *Batch) init(size int) {
n := batchInitialSize
for n < cap {
for n < size {
n *= 2
}
b.data = rawalloc.New(batchHeaderLen, n)
b.setCount(0)
b.setSeqNum(0)
if cap(b.data) < n {
b.data = rawalloc.New(batchHeaderLen, n)
}
b.data = b.data[:batchHeaderLen]
// Zero the sequence number in the header.
for i := 0; i < len(b.data); i++ {
b.data[i] = 0
}
}
// Reset resets the batch for reuse. The underlying byte slice (that is
// returned by Repr()) is not modified. It is only necessary to call this
// returned by Repr()) may not be modified. It is only necessary to call this
// method if a batch is explicitly being reused. Close automatically takes are
// of releasing resources when appropriate for batches that are internally
// being reused.
func (b *Batch) Reset() {
b.count = 0
b.countRangeDels = 0
b.countRangeKeys = 0
b.memTableSize = 0
b.deferredOp = DeferredBatchOp{}
b.tombstones = nil
b.tombstonesSeqNum = 0
b.rangeKeys = nil
b.rangeKeysSeqNum = 0
b.flushable = nil
b.commit = sync.WaitGroup{}
b.fsyncWait = sync.WaitGroup{}
b.commitStats = BatchCommitStats{}
b.commitErr = nil
atomic.StoreUint32(&b.applied, 0)
// Zero out the struct, retaining only the fields necessary for manual
// reuse.
b.batchInternal = batchInternal{
data: b.data,
cmp: b.cmp,
formatKey: b.formatKey,
abbreviatedKey: b.abbreviatedKey,
index: b.index,
db: b.db,
}
b.applied.Store(false)
if b.data != nil {
if cap(b.data) > batchMaxRetainedSize {
// If the capacity of the buffer is larger than our maximum
@@ -1209,13 +1385,14 @@ func (b *Batch) Reset() {
} else {
// Otherwise, reset the buffer for re-use.
b.data = b.data[:batchHeaderLen]
b.setSeqNum(0)
// Zero the sequence number in the header.
for i := 0; i < len(b.data); i++ {
b.data[i] = 0
}
}
}
if b.index != nil {
b.index.Init(&b.data, b.cmp, b.abbreviatedKey)
b.rangeDelIndex = nil
b.rangeKeyIndex = nil
}
}
@@ -1287,6 +1464,11 @@ func (b *Batch) Reader() BatchReader {
}
func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) {
// TODO(jackson): This will index out of bounds if there's no varint or an
// invalid varint (eg, a single 0xff byte). Correcting will add a bit of
// overhead. We could avoid that overhead whenever len(data) >=
// binary.MaxVarint32?
var v uint32
var n int
ptr := unsafe.Pointer(&data[0])
@@ -1349,29 +1531,32 @@ func ReadBatch(repr []byte) (r BatchReader, count uint32) {
return repr[batchHeaderLen:], count
}
// Next returns the next entry in this batch. The final return value is false
// if the batch is corrupt. The end of batch is reached when len(r)==0.
func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool) {
// Next returns the next entry in this batch, if there is one. If the reader has
// reached the end of the batch, Next returns ok=false and a nil error. If the
// batch is corrupt and the next entry is illegible, Next returns ok=false and a
// non-nil error.
func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool, err error) {
if len(*r) == 0 {
return 0, nil, nil, false
return 0, nil, nil, false, nil
}
kind = InternalKeyKind((*r)[0])
if kind > InternalKeyKindMax {
return 0, nil, nil, false
return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "invalid key kind 0x%x", (*r)[0])
}
*r, ukey, ok = batchDecodeStr((*r)[1:])
if !ok {
return 0, nil, nil, false
return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding user key")
}
switch kind {
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
InternalKeyKindDeleteSized:
*r, value, ok = batchDecodeStr(*r)
if !ok {
return 0, nil, nil, false
return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding %s value", kind)
}
}
return kind, ukey, value, true
return kind, ukey, value, true, nil
}
// Note: batchIter mirrors the implementation of flushableBatchIter. Keep the
@@ -1500,7 +1685,8 @@ func (i *batchIter) value() []byte {
switch InternalKeyKind(data[offset]) {
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
InternalKeyKindDeleteSized:
_, value, ok := batchDecodeStr(data[keyEnd:])
if !ok {
return nil
@@ -1572,7 +1758,7 @@ var _ flushable = (*flushableBatch)(nil)
// interface. This allows the batch to act like a memtable and be placed in the
// queue of flushable memtables. Note that the flushable batch takes ownership
// of the batch data.
func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
func newFlushableBatch(batch *Batch, comparer *Comparer) (*flushableBatch, error) {
b := &flushableBatch{
data: batch.data,
cmp: comparer.Compare,
@@ -1593,8 +1779,11 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
var index uint32
for iter := BatchReader(b.data[batchHeaderLen:]); len(iter) > 0; index++ {
offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
kind, key, _, ok := iter.Next()
kind, key, _, ok, err := iter.Next()
if !ok {
if err != nil {
return nil, err
}
break
}
entry := flushableBatchEntry{
@@ -1666,7 +1855,7 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
}
fragmentRangeKeys(frag, it, len(rangeKeyOffsets))
}
return b
return b, nil
}
func (b *flushableBatch) setSeqNum(seqNum uint64) {
@@ -1715,6 +1904,7 @@ func (b *flushableBatch) Swap(i, j int) {
b.offsets[i], b.offsets[j] = b.offsets[j], b.offsets[i]
}
// newIter is part of the flushable interface.
func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
return &flushableBatchIter{
batch: b,
@@ -1727,6 +1917,7 @@ func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
}
}
// newFlushIter is part of the flushable interface.
func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
return &flushFlushableBatchIter{
flushableBatchIter: flushableBatchIter{
@@ -1740,6 +1931,7 @@ func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) inte
}
}
// newRangeDelIter is part of the flushable interface.
func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterator {
if len(b.tombstones) == 0 {
return nil
@@ -1747,6 +1939,7 @@ func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterato
return keyspan.NewIter(b.cmp, b.tombstones)
}
// newRangeKeyIter is part of the flushable interface.
func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
if len(b.rangeKeys) == 0 {
return nil
@@ -1754,17 +1947,23 @@ func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterato
return keyspan.NewIter(b.cmp, b.rangeKeys)
}
// containsRangeKeys is part of the flushable interface.
func (b *flushableBatch) containsRangeKeys() bool { return len(b.rangeKeys) > 0 }
// inuseBytes is part of the flushable interface.
func (b *flushableBatch) inuseBytes() uint64 {
return uint64(len(b.data) - batchHeaderLen)
}
// totalBytes is part of the flushable interface.
func (b *flushableBatch) totalBytes() uint64 {
return uint64(cap(b.data))
}
// readyForFlush is part of the flushable interface.
func (b *flushableBatch) readyForFlush() bool {
// A flushable batch is always ready for flush; it must be flushed together
// with the previous memtable.
return true
}
@@ -1944,7 +2143,8 @@ func (i *flushableBatchIter) value() base.LazyValue {
var ok bool
switch kind {
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
InternalKeyKindDeleteSized:
keyEnd := i.offsets[i.index].keyEnd
_, value, ok = batchDecodeStr(i.data[keyEnd:])
if !ok {
@@ -2081,7 +2281,10 @@ func batchSort(
rangeKeyIter := b.newRangeKeyIter(nil, math.MaxUint64)
return pointIter, rangeDelIter, rangeKeyIter
}
f := newFlushableBatch(b, b.db.opts.Comparer)
f, err := newFlushableBatch(b, b.db.opts.Comparer)
if err != nil {
panic(err)
}
return f.newIter(nil), f.newRangeDelIter(nil), f.newRangeKeyIter(nil)
}

52
vendor/github.com/cockroachdb/pebble/checkpoint.go generated vendored
View File

@@ -188,7 +188,10 @@ func (d *DB) Checkpoint(
manifestFileNum := d.mu.versions.manifestFileNum
manifestSize := d.mu.versions.manifest.Size()
optionsFileNum := d.optionsFileNum
virtualBackingFiles := make(map[base.DiskFileNum]struct{})
for diskFileNum := range d.mu.versions.backingState.fileBackingMap {
virtualBackingFiles[diskFileNum] = struct{}{}
}
// Release the manifest and DB.mu so we don't block other operations on
// the database.
d.mu.versions.logUnlock()
@@ -209,11 +212,7 @@ func (d *DB) Checkpoint(
}
if ckErr != nil {
// Attempt to cleanup on error.
paths, _ := fs.List(destDir)
for _, path := range paths {
_ = fs.Remove(path)
}
_ = fs.Remove(destDir)
_ = fs.RemoveAll(destDir)
}
}()
dir, ckErr = mkdirAllAndSyncParents(fs, destDir)
@@ -254,7 +253,9 @@ func (d *DB) Checkpoint(
}
var excludedFiles map[deletedFileEntry]*fileMetadata
// Set of FileBacking.DiskFileNum which will be required by virtual sstables
// in the checkpoint.
requiredVirtualBackingFiles := make(map[base.DiskFileNum]struct{})
// Link or copy the sstables.
for l := range current.Levels {
iter := current.Levels[l].Iter()
@@ -270,7 +271,15 @@ func (d *DB) Checkpoint(
continue
}
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, f.FileNum)
fileBacking := f.FileBacking
if f.Virtual {
if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok {
continue
}
requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{}
}
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
if ckErr != nil {
@@ -279,7 +288,19 @@ func (d *DB) Checkpoint(
}
}
ckErr = d.writeCheckpointManifest(fs, formatVers, destDir, dir, manifestFileNum, manifestSize, excludedFiles)
var removeBackingTables []base.DiskFileNum
for diskFileNum := range virtualBackingFiles {
if _, ok := requiredVirtualBackingFiles[diskFileNum]; !ok {
// The backing sstable associated with fileNum is no longer
// required.
removeBackingTables = append(removeBackingTables, diskFileNum)
}
}
ckErr = d.writeCheckpointManifest(
fs, formatVers, destDir, dir, manifestFileNum.DiskFileNum(), manifestSize,
excludedFiles, removeBackingTables,
)
if ckErr != nil {
return ckErr
}
@@ -292,7 +313,7 @@ func (d *DB) Checkpoint(
if logNum == 0 {
continue
}
srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum)
srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum.DiskFileNum())
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.Copy(fs, srcPath, destPath)
if ckErr != nil {
@@ -315,9 +336,10 @@ func (d *DB) writeCheckpointManifest(
formatVers FormatMajorVersion,
destDirPath string,
destDir vfs.File,
manifestFileNum FileNum,
manifestFileNum base.DiskFileNum,
manifestSize int64,
excludedFiles map[deletedFileEntry]*fileMetadata,
removeBackingTables []base.DiskFileNum,
) error {
// Copy the MANIFEST, and create a pointer to it. We copy rather
// than link because additional version edits added to the
@@ -346,7 +368,7 @@ func (d *DB) writeCheckpointManifest(
// need to append another record with the excluded files (we cannot simply
// append a record after a raw data copy; see
// https://github.com/cockroachdb/cockroach/issues/100935).
r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum)
r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum.FileNum())
w := record.NewWriter(dst)
for {
rr, err := r.Next()
@@ -369,8 +391,10 @@ func (d *DB) writeCheckpointManifest(
if len(excludedFiles) > 0 {
// Write out an additional VersionEdit that deletes the excluded SST files.
ve := versionEdit{
DeletedFiles: excludedFiles,
DeletedFiles: excludedFiles,
RemovedBackingTables: removeBackingTables,
}
rw, err := w.Next()
if err != nil {
return err
@@ -397,7 +421,7 @@ func (d *DB) writeCheckpointManifest(
if err != nil {
return err
}
if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum); err != nil {
if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum.FileNum()); err != nil {
return err
}
return manifestMarker.Close()

281
vendor/github.com/cockroachdb/pebble/cleaner.go generated vendored
View File

@@ -4,7 +4,18 @@
package pebble
import "github.com/cockroachdb/pebble/internal/base"
import (
"context"
"runtime/pprof"
"sync"
"time"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/tokenbucket"
)
// Cleaner exports the base.Cleaner type.
type Cleaner = base.Cleaner
@@ -14,3 +25,271 @@ type DeleteCleaner = base.DeleteCleaner
// ArchiveCleaner exports the base.ArchiveCleaner type.
type ArchiveCleaner = base.ArchiveCleaner
type cleanupManager struct {
opts *Options
objProvider objstorage.Provider
onTableDeleteFn func(fileSize uint64)
deletePacer *deletionPacer
// jobsCh is used as the cleanup job queue.
jobsCh chan *cleanupJob
// waitGroup is used to wait for the background goroutine to exit.
waitGroup sync.WaitGroup
mu struct {
sync.Mutex
// totalJobs is the total number of enqueued jobs (completed or in progress).
totalJobs int
completedJobs int
completedJobsCond sync.Cond
jobsQueueWarningIssued bool
}
}
// We can queue this many jobs before we have to block EnqueueJob.
const jobsQueueDepth = 1000
// obsoleteFile holds information about a file that needs to be deleted soon.
type obsoleteFile struct {
dir string
fileNum base.DiskFileNum
fileType fileType
fileSize uint64
}
type cleanupJob struct {
jobID int
obsoleteFiles []obsoleteFile
}
// openCleanupManager creates a cleanupManager and starts its background goroutine.
// The cleanupManager must be Close()d.
func openCleanupManager(
opts *Options,
objProvider objstorage.Provider,
onTableDeleteFn func(fileSize uint64),
getDeletePacerInfo func() deletionPacerInfo,
) *cleanupManager {
cm := &cleanupManager{
opts: opts,
objProvider: objProvider,
onTableDeleteFn: onTableDeleteFn,
deletePacer: newDeletionPacer(time.Now(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo),
jobsCh: make(chan *cleanupJob, jobsQueueDepth),
}
cm.mu.completedJobsCond.L = &cm.mu.Mutex
cm.waitGroup.Add(1)
go func() {
pprof.Do(context.Background(), gcLabels, func(context.Context) {
cm.mainLoop()
})
}()
return cm
}
// Close stops the background goroutine, waiting until all queued jobs are completed.
// Delete pacing is disabled for the remaining jobs.
func (cm *cleanupManager) Close() {
close(cm.jobsCh)
cm.waitGroup.Wait()
}
// EnqueueJob adds a cleanup job to the manager's queue.
func (cm *cleanupManager) EnqueueJob(jobID int, obsoleteFiles []obsoleteFile) {
job := &cleanupJob{
jobID: jobID,
obsoleteFiles: obsoleteFiles,
}
// Report deleted bytes to the pacer, which can use this data to potentially
// increase the deletion rate to keep up. We want to do this at enqueue time
// rather than when we get to the job, otherwise the reported bytes will be
// subject to the throttling rate which defeats the purpose.
var pacingBytes uint64
for _, of := range obsoleteFiles {
if cm.needsPacing(of.fileType, of.fileNum) {
pacingBytes += of.fileSize
}
}
if pacingBytes > 0 {
cm.deletePacer.ReportDeletion(time.Now(), pacingBytes)
}
cm.mu.Lock()
cm.mu.totalJobs++
cm.maybeLogLocked()
cm.mu.Unlock()
if invariants.Enabled && len(cm.jobsCh) >= cap(cm.jobsCh)-2 {
panic("cleanup jobs queue full")
}
cm.jobsCh <- job
}
// Wait until the completion of all jobs that were already queued.
//
// Does not wait for jobs that are enqueued during the call.
//
// Note that DB.mu should not be held while calling this method; the background
// goroutine needs to acquire DB.mu to update deleted table metrics.
func (cm *cleanupManager) Wait() {
cm.mu.Lock()
defer cm.mu.Unlock()
n := cm.mu.totalJobs
for cm.mu.completedJobs < n {
cm.mu.completedJobsCond.Wait()
}
}
// mainLoop runs the manager's background goroutine.
func (cm *cleanupManager) mainLoop() {
defer cm.waitGroup.Done()
var tb tokenbucket.TokenBucket
// Use a token bucket with 1 token / second refill rate and 1 token burst.
tb.Init(1.0, 1.0)
for job := range cm.jobsCh {
for _, of := range job.obsoleteFiles {
if of.fileType != fileTypeTable {
path := base.MakeFilepath(cm.opts.FS, of.dir, of.fileType, of.fileNum)
cm.deleteObsoleteFile(of.fileType, job.jobID, path, of.fileNum, of.fileSize)
} else {
cm.maybePace(&tb, of.fileType, of.fileNum, of.fileSize)
cm.onTableDeleteFn(of.fileSize)
cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.fileNum)
}
}
cm.mu.Lock()
cm.mu.completedJobs++
cm.mu.completedJobsCond.Broadcast()
cm.maybeLogLocked()
cm.mu.Unlock()
}
}
func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNum base.DiskFileNum) bool {
if fileType != fileTypeTable {
return false
}
meta, err := cm.objProvider.Lookup(fileType, fileNum)
if err != nil {
// The object was already removed from the provider; we won't actually
// delete anything, so we don't need to pace.
return false
}
// Don't throttle deletion of remote objects.
return !meta.IsRemote()
}
// maybePace sleeps before deleting an object if appropriate. It is always
// called from the background goroutine.
func (cm *cleanupManager) maybePace(
tb *tokenbucket.TokenBucket, fileType base.FileType, fileNum base.DiskFileNum, fileSize uint64,
) {
if !cm.needsPacing(fileType, fileNum) {
return
}
tokens := cm.deletePacer.PacingDelay(time.Now(), fileSize)
if tokens == 0.0 {
// The token bucket might be in debt; it could make us wait even for 0
// tokens. We don't want that if the pacer decided throttling should be
// disabled.
return
}
// Wait for tokens. We use a token bucket instead of sleeping outright because
// the token bucket accumulates up to one second of unused tokens.
for {
ok, d := tb.TryToFulfill(tokenbucket.Tokens(tokens))
if ok {
break
}
time.Sleep(d)
}
}
// deleteObsoleteFile deletes a (non-object) file that is no longer needed.
func (cm *cleanupManager) deleteObsoleteFile(
fileType fileType, jobID int, path string, fileNum base.DiskFileNum, fileSize uint64,
) {
// TODO(peter): need to handle this error, probably by re-adding the
// file that couldn't be deleted to one of the obsolete slices map.
err := cm.opts.Cleaner.Clean(cm.opts.FS, fileType, path)
if oserror.IsNotExist(err) {
return
}
switch fileType {
case fileTypeLog:
cm.opts.EventListener.WALDeleted(WALDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
case fileTypeManifest:
cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
case fileTypeTable:
panic("invalid deletion of object file")
}
}
func (cm *cleanupManager) deleteObsoleteObject(
fileType fileType, jobID int, fileNum base.DiskFileNum,
) {
if fileType != fileTypeTable {
panic("not an object")
}
var path string
meta, err := cm.objProvider.Lookup(fileType, fileNum)
if err != nil {
path = "<nil>"
} else {
path = cm.objProvider.Path(meta)
err = cm.objProvider.Remove(fileType, fileNum)
}
if cm.objProvider.IsNotExistError(err) {
return
}
switch fileType {
case fileTypeTable:
cm.opts.EventListener.TableDeleted(TableDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
}
}
// maybeLogLocked issues a log if the job queue gets 75% full and issues a log
// when the job queue gets back to less than 10% full.
//
// Must be called with cm.mu locked.
func (cm *cleanupManager) maybeLogLocked() {
const highThreshold = jobsQueueDepth * 3 / 4
const lowThreshold = jobsQueueDepth / 10
jobsInQueue := cm.mu.totalJobs - cm.mu.completedJobs
if !cm.mu.jobsQueueWarningIssued && jobsInQueue > highThreshold {
cm.mu.jobsQueueWarningIssued = true
cm.opts.Logger.Infof("cleanup falling behind; job queue has over %d jobs", highThreshold)
}
if cm.mu.jobsQueueWarningIssued && jobsInQueue < lowThreshold {
cm.mu.jobsQueueWarningIssued = false
cm.opts.Logger.Infof("cleanup back to normal; job queue has under %d jobs", lowThreshold)
}
}

61
vendor/github.com/cockroachdb/pebble/commit.go generated vendored
View File

@@ -9,7 +9,6 @@ import (
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/pebble/record"
)
@@ -33,13 +32,13 @@ type commitQueue struct {
//
// The head index is stored in the most-significant bits so that we can
// atomically add to it and the overflow is harmless.
headTail uint64
headTail atomic.Uint64
// slots is a ring buffer of values stored in this queue. The size must be a
// power of 2. A slot is in use until *both* the tail index has moved beyond
// it and the slot value has been set to nil. The slot value is set to nil
// atomically by the consumer and read atomically by the producer.
slots [record.SyncConcurrency]unsafe.Pointer
slots [record.SyncConcurrency]atomic.Pointer[Batch]
}
const dequeueBits = 32
@@ -58,7 +57,7 @@ func (q *commitQueue) pack(head, tail uint32) uint64 {
}
func (q *commitQueue) enqueue(b *Batch) {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
// Queue is full. This should never be reached because commitPipeline.commitQueueSem
@@ -67,8 +66,8 @@ func (q *commitQueue) enqueue(b *Batch) {
}
slot := &q.slots[head&uint32(len(q.slots)-1)]
// Check if the head slot has been released by dequeue.
for atomic.LoadPointer(slot) != nil {
// Check if the head slot has been released by dequeueApplied.
for slot.Load() != nil {
// Another goroutine is still cleaning up the tail, so the queue is
// actually still full. We spin because this should resolve itself
// momentarily.
@@ -76,16 +75,20 @@ func (q *commitQueue) enqueue(b *Batch) {
}
// The head slot is free, so we own it.
atomic.StorePointer(slot, unsafe.Pointer(b))
slot.Store(b)
// Increment head. This passes ownership of slot to dequeue and acts as a
// Increment head. This passes ownership of slot to dequeueApplied and acts as a
// store barrier for writing the slot.
atomic.AddUint64(&q.headTail, 1<<dequeueBits)
q.headTail.Add(1 << dequeueBits)
}
func (q *commitQueue) dequeue() *Batch {
// dequeueApplied removes the earliest enqueued Batch, if it is applied.
//
// Returns nil if the commit queue is empty or the earliest Batch is not yet
// applied.
func (q *commitQueue) dequeueApplied() *Batch {
for {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if tail == head {
// Queue is empty.
@@ -93,8 +96,8 @@ func (q *commitQueue) dequeue() *Batch {
}
slot := &q.slots[tail&uint32(len(q.slots)-1)]
b := (*Batch)(atomic.LoadPointer(slot))
if b == nil || atomic.LoadUint32(&b.applied) == 0 {
b := slot.Load()
if b == nil || !b.applied.Load() {
// The batch is not ready to be dequeued, or another goroutine has
// already dequeued it.
return nil
@@ -103,13 +106,13 @@ func (q *commitQueue) dequeue() *Batch {
// Confirm head and tail (for our speculative check above) and increment
// tail. If this succeeds, then we own the slot at tail.
ptrs2 := q.pack(head, tail+1)
if atomic.CompareAndSwapUint64(&q.headTail, ptrs, ptrs2) {
if q.headTail.CompareAndSwap(ptrs, ptrs2) {
// We now own slot.
//
// Tell enqueue that we're done with this slot. Zeroing the slot is also
// important so we don't leave behind references that could keep this object
// live longer than necessary.
atomic.StorePointer(slot, nil)
slot.Store(nil)
// At this point enqueue owns the slot.
return b
}
@@ -122,10 +125,10 @@ func (q *commitQueue) dequeue() *Batch {
type commitEnv struct {
// The next sequence number to give to a batch. Protected by
// commitPipeline.mu.
logSeqNum *uint64
logSeqNum *atomic.Uint64
// The visible sequence number at which reads should be performed. Ratcheted
// upwards atomically as batches are applied to the memtable.
visibleSeqNum *uint64
visibleSeqNum *atomic.Uint64
// Apply the batch to the specified memtable. Called concurrently.
apply func(b *Batch, mem *memTable) error
@@ -239,6 +242,7 @@ type commitPipeline struct {
// logSyncQSem are used for this reservation.
commitQueueSem chan struct{}
logSyncQSem chan struct{}
ingestSem chan struct{}
// The mutex to use for synchronizing access to logSeqNum and serializing
// calls to commitEnv.write().
mu sync.Mutex
@@ -263,6 +267,7 @@ func newCommitPipeline(env commitEnv) *commitPipeline {
// and sync the WAL.
commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
logSyncQSem: make(chan struct{}, record.SyncConcurrency-1),
ingestSem: make(chan struct{}, 1),
}
return p
}
@@ -381,12 +386,12 @@ func (p *commitPipeline) AllocateSeqNum(
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
logSeqNum := atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
logSeqNum := p.env.logSeqNum.Add(uint64(count)) - uint64(count)
seqNum := logSeqNum
if seqNum == 0 {
// We can't use the value 0 for the global seqnum during ingestion, because
// 0 indicates no global seqnum. So allocate one more seqnum.
atomic.AddUint64(p.env.logSeqNum, 1)
p.env.logSeqNum.Add(1)
seqNum++
}
b.setSeqNum(seqNum)
@@ -396,7 +401,7 @@ func (p *commitPipeline) AllocateSeqNum(
// writes that were sequenced before the ingestion. The spin loop is
// unfortunate, but obviates the need for additional synchronization.
for {
visibleSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
visibleSeqNum := p.env.visibleSeqNum.Load()
if visibleSeqNum == logSeqNum {
break
}
@@ -455,7 +460,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memT
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
b.setSeqNum(atomic.AddUint64(p.env.logSeqNum, n) - n)
b.setSeqNum(p.env.logSeqNum.Add(n) - n)
// Write the data to the WAL.
mem, err := p.env.write(b, syncWG, syncErr)
@@ -467,17 +472,17 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memT
func (p *commitPipeline) publish(b *Batch) {
// Mark the batch as applied.
atomic.StoreUint32(&b.applied, 1)
b.applied.Store(true)
// Loop dequeuing applied batches from the pending queue. If our batch was
// the head of the pending queue we are guaranteed that either we'll publish
// it or someone else will dequeue and publish it. If our batch is not the
// head of the queue then either we'll dequeue applied batches and reach our
// it or someone else will dequeueApplied and publish it. If our batch is not the
// head of the queue then either we'll dequeueApplied applied batches and reach our
// batch or there is an unapplied batch blocking us. When that unapplied
// batch applies it will go through the same process and publish our batch
// for us.
for {
t := p.pending.dequeue()
t := p.pending.dequeueApplied()
if t == nil {
// Wait for another goroutine to publish us. We might also be waiting for
// the WAL sync to finish.
@@ -486,7 +491,7 @@ func (p *commitPipeline) publish(b *Batch) {
b.commitStats.CommitWaitDuration += time.Since(now)
break
}
if atomic.LoadUint32(&t.applied) != 1 {
if !t.applied.Load() {
panic("not reached")
}
@@ -495,13 +500,13 @@ func (p *commitPipeline) publish(b *Batch) {
// number for a subsequent batch. That's ok as all we're guaranteeing is
// that the sequence number ratchets up.
for {
curSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
curSeqNum := p.env.visibleSeqNum.Load()
newSeqNum := t.SeqNum() + uint64(t.Count())
if newSeqNum <= curSeqNum {
// t's sequence number has already been published.
break
}
if atomic.CompareAndSwapUint64(p.env.visibleSeqNum, curSeqNum, newSeqNum) {
if p.env.visibleSeqNum.CompareAndSwap(curSeqNum, newSeqNum) {
// We successfully published t's sequence number.
break
}

1401
vendor/github.com/cockroachdb/pebble/compaction.go generated vendored
View File

File diff suppressed because it is too large Load Diff

700
vendor/github.com/cockroachdb/pebble/compaction_iter.go generated vendored
View File

@@ -6,6 +6,7 @@ package pebble
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"sort"
@@ -14,9 +15,9 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/rangekey"
"github.com/cockroachdb/redact"
)
// compactionIter provides a forward-only iterator that encapsulates the logic
@@ -205,7 +206,29 @@ type compactionIter struct {
// compaction iterator was only returned because an open snapshot prevents
// its elision. This field only applies to point keys, and not to range
// deletions or range keys.
//
// For MERGE, it is possible that doing the merge is interrupted even when
// the next point key is in the same stripe. This can happen if the loop in
// mergeNext gets interrupted by sameStripeNonSkippable.
// sameStripeNonSkippable occurs due to RANGEDELs that sort before
// SET/MERGE/DEL with the same seqnum, so the RANGEDEL does not necessarily
// delete the subsequent SET/MERGE/DEL keys.
snapshotPinned bool
// forceObsoleteDueToRangeDel is set to true in a subset of the cases that
// snapshotPinned is true. This value is true when the point is obsolete due
// to a RANGEDEL but could not be deleted due to a snapshot.
//
// NB: it may seem that the additional cases that snapshotPinned captures
// are harmless in that they can also be used to mark a point as obsolete
// (it is merely a duplication of some logic that happens in
// Writer.AddWithForceObsolete), but that is not quite accurate as of this
// writing -- snapshotPinned originated in stats collection and for a
// sequence MERGE, SET, where the MERGE cannot merge with the (older) SET
// due to a snapshot, the snapshotPinned value for the SET is true.
//
// TODO(sumeer,jackson): improve the logic of snapshotPinned and reconsider
// whether we need forceObsoleteDueToRangeDel.
forceObsoleteDueToRangeDel bool
// The index of the snapshot for the current key within the snapshots slice.
curSnapshotIdx int
curSnapshotSeqNum uint64
@@ -233,13 +256,19 @@ type compactionIter struct {
// The fragmented range keys.
rangeKeys []keyspan.Span
// Byte allocator for the tombstone keys.
alloc bytealloc.A
allowZeroSeqNum bool
elideTombstone func(key []byte) bool
elideRangeTombstone func(start, end []byte) bool
alloc bytealloc.A
allowZeroSeqNum bool
elideTombstone func(key []byte) bool
elideRangeTombstone func(start, end []byte) bool
ineffectualSingleDeleteCallback func(userKey []byte)
singleDeleteInvariantViolationCallback func(userKey []byte)
// The on-disk format major version. This informs the types of keys that
// may be written to disk during a compaction.
formatVersion FormatMajorVersion
stats struct {
// count of DELSIZED keys that were missized.
countMissizedDels uint64
}
}
func newCompactionIter(
@@ -254,20 +283,24 @@ func newCompactionIter(
allowZeroSeqNum bool,
elideTombstone func(key []byte) bool,
elideRangeTombstone func(start, end []byte) bool,
ineffectualSingleDeleteCallback func(userKey []byte),
singleDeleteInvariantViolationCallback func(userKey []byte),
formatVersion FormatMajorVersion,
) *compactionIter {
i := &compactionIter{
equal: equal,
merge: merge,
iter: iter,
snapshots: snapshots,
frontiers: frontiers{cmp: cmp},
rangeDelFrag: rangeDelFrag,
rangeKeyFrag: rangeKeyFrag,
allowZeroSeqNum: allowZeroSeqNum,
elideTombstone: elideTombstone,
elideRangeTombstone: elideRangeTombstone,
formatVersion: formatVersion,
equal: equal,
merge: merge,
iter: iter,
snapshots: snapshots,
frontiers: frontiers{cmp: cmp},
rangeDelFrag: rangeDelFrag,
rangeKeyFrag: rangeKeyFrag,
allowZeroSeqNum: allowZeroSeqNum,
elideTombstone: elideTombstone,
elideRangeTombstone: elideRangeTombstone,
ineffectualSingleDeleteCallback: ineffectualSingleDeleteCallback,
singleDeleteInvariantViolationCallback: singleDeleteInvariantViolationCallback,
formatVersion: formatVersion,
}
i.rangeDelFrag.Cmp = cmp
i.rangeDelFrag.Format = formatKey
@@ -306,21 +339,43 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
return nil, nil
}
// Prior to this call to `Next()` we are in one of three situations with
// Prior to this call to `Next()` we are in one of four situations with
// respect to `iterKey` and related state:
//
// - `!skip && pos == iterPosNext`: `iterKey` is already at the next key.
// - `!skip && pos == iterPosCur`: We are at the key that has been returned.
// - `!skip && pos == iterPosCurForward`: We are at the key that has been returned.
// To move forward we advance by one key, even if that lands us in the same
// snapshot stripe.
// - `skip && pos == iterPosCur`: We are at the key that has been returned.
// - `skip && pos == iterPosCurForward`: We are at the key that has been returned.
// To move forward we skip skippable entries in the stripe.
// - `skip && pos == iterPosNext && i.iterStripeChange == sameStripeNonSkippable`:
// This case may occur when skipping within a snapshot stripe and we
// encounter either:
// a) an invalid key kind; The previous call will have returned
// whatever key it was processing and deferred handling of the
// invalid key to this invocation of Next(). We're responsible for
// ignoring skip=true and falling into the invalid key kind case
// down below.
// b) an interleaved range delete; This is a wart of the current code
// structure. While skipping within a snapshot stripe, a range
// delete interleaved at its start key and sequence number
// interrupts the sequence of point keys. After we return the range
// delete to the caller, we need to pick up skipping at where we
// left off, so we preserve skip=true.
// TODO(jackson): This last case is confusing and can be removed if we
// interleave range deletions at the maximal sequence number using the
// keyspan interleaving iterator. This is the treatment given to range
// keys today.
if i.pos == iterPosCurForward {
if i.skip {
i.skipInStripe()
} else {
i.nextInStripe()
}
} else if i.skip {
if i.iterStripeChange != sameStripeNonSkippable {
panic(errors.AssertionFailedf("compaction iterator has skip=true, but iterator is at iterPosNext"))
}
}
i.pos = iterPosCurForward
@@ -374,6 +429,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
return &i.key, i.value
}
// TODO(sumeer): we could avoid calling Covers if i.iterStripeChange ==
// sameStripeSameKey since that check has already been done in
// nextInStripeHelper. However, we also need to handle the case of
// CoversInvisibly below.
if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly {
// A pending range deletion deletes this key. Skip it.
i.saveKey()
@@ -382,17 +441,37 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
} else if cover == keyspan.CoversInvisibly {
// i.iterKey would be deleted by a range deletion if there weren't
// any open snapshots. Mark it as pinned.
//
// NB: there are multiple places in this file where we call
// i.rangeDelFrag.Covers and this is the only one where we are writing
// to i.snapshotPinned. Those other cases occur in mergeNext where the
// caller is deciding whether the value should be merged or not, and the
// key is in the same snapshot stripe. Hence, snapshotPinned is by
// definition false in those cases.
i.snapshotPinned = true
i.forceObsoleteDueToRangeDel = true
} else {
i.forceObsoleteDueToRangeDel = false
}
switch i.iterKey.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
if i.elideTombstone(i.iterKey.UserKey) {
if i.curSnapshotIdx == 0 {
// If we're at the last snapshot stripe and the tombstone
// can be elided skip skippable keys in the same stripe.
i.saveKey()
i.skipInStripe()
if i.key.Kind() == InternalKeyKindSingleDelete {
i.skipDueToSingleDeleteElision()
} else {
i.skipInStripe()
if !i.skip && i.iterStripeChange != newStripeNewKey {
panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe disabled skip without advancing to new key"))
}
}
if i.iterStripeChange == newStripeSameKey {
panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe found a new stripe within the same key"))
}
continue
} else {
// We're not at the last snapshot stripe, so the tombstone
@@ -410,11 +489,23 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
i.skip = true
return &i.key, i.value
case InternalKeyKindDeleteSized:
// We may skip subsequent keys because of this tombstone. Scan
// ahead to see just how much data this tombstone drops and if
// the tombstone's value should be updated accordingly.
return i.deleteSizedNext()
case InternalKeyKindSingleDelete:
if i.singleDeleteNext() {
return &i.key, i.value
} else if i.err != nil {
return nil, nil
}
continue
default:
panic(errors.AssertionFailedf(
"unexpected kind %s", redact.SafeString(i.iterKey.Kind().String())))
}
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -424,6 +515,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
// preserving the original value, and potentially mutating the key
// kind.
i.setNext()
if i.err != nil {
return nil, nil
}
return &i.key, i.value
case InternalKeyKindMerge:
@@ -440,7 +534,15 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
if i.err == nil {
// includesBase is true whenever we've transformed the MERGE record
// into a SET.
includesBase := i.key.Kind() == InternalKeyKindSet
var includesBase bool
switch i.key.Kind() {
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
includesBase = true
case InternalKeyKindMerge:
default:
panic(errors.AssertionFailedf(
"unexpected kind %s", redact.SafeString(i.key.Kind().String())))
}
i.value, needDelete, i.valueCloser, i.err = finishValueMerger(valueMerger, includesBase)
}
if i.err == nil {
@@ -469,6 +571,8 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
}
if i.err != nil {
i.valid = false
// TODO(sumeer): why is MarkCorruptionError only being called for
// MERGE?
i.err = base.MarkCorruptionError(i.err)
}
return nil, nil
@@ -508,10 +612,16 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) {
return index, snapshots[index]
}
// skipInStripe skips over skippable keys in the same stripe and user key.
// skipInStripe skips over skippable keys in the same stripe and user key. It
// may set i.err, in which case i.iterKey will be nil.
func (i *compactionIter) skipInStripe() {
i.skip = true
// TODO(sumeer): we can avoid the overhead of calling i.rangeDelFrag.Covers,
// in this case of nextInStripe, since we are skipping all of them anyway.
for i.nextInStripe() == sameStripeSkippable {
if i.err != nil {
panic(i.err)
}
}
// Reset skip if we landed outside the original stripe. Otherwise, we landed
// in the same stripe on a non-skippable key. In that case we should preserve
@@ -549,11 +659,17 @@ const (
// nextInStripe advances the iterator and returns one of the above const ints
// indicating how its state changed.
//
// All sameStripeSkippable keys that are covered by a RANGEDEL will be skipped
// and not returned.
//
// Calls to nextInStripe must be preceded by a call to saveKey to retain a
// temporary reference to the original key, so that forward iteration can
// proceed with a reference to the original key. Care should be taken to avoid
// overwriting or mutating the saved key or value before they have been returned
// to the caller of the exported function (i.e. the caller of Next, First, etc.)
//
// nextInStripe may set i.err, in which case the return value will be
// newStripeNewKey, and i.iterKey will be nil.
func (i *compactionIter) nextInStripe() stripeChangeType {
i.iterStripeChange = i.nextInStripeHelper()
return i.iterStripeChange
@@ -562,58 +678,72 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
// nextInStripeHelper is an internal helper for nextInStripe; callers should use
// nextInStripe and not call nextInStripeHelper.
func (i *compactionIter) nextInStripeHelper() stripeChangeType {
if !i.iterNext() {
return newStripeNewKey
}
key := i.iterKey
origSnapshotIdx := i.curSnapshotIdx
for {
if !i.iterNext() {
return newStripeNewKey
}
key := i.iterKey
// NB: The below conditional is an optimization to avoid a user key
// comparison in many cases. Internal keys with the same user key are
// ordered in (strictly) descending order by trailer. If the new key has a
// greater or equal trailer, or the previous key had a zero sequence number,
// the new key must have a new user key.
//
// A couple things make these cases common:
// - Sequence-number zeroing ensures ~all of the keys in L6 have a zero
// sequence number.
// - Ingested sstables' keys all adopt the same sequence number.
if i.keyTrailer <= base.InternalKeyZeroSeqnumMaxTrailer || key.Trailer >= i.keyTrailer {
if invariants.Enabled && i.equal(i.key.UserKey, key.UserKey) {
if !i.equal(i.key.UserKey, key.UserKey) {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripeNewKey
}
// If i.key and key have the same user key, then
// 1. i.key must not have had a zero sequence number (or it would've be the last
// key with its user key).
// 2. i.key must have a strictly larger sequence number
// There's an exception in that either key may be a range delete. Range
// deletes may share a sequence number with a point key if the keys were
// ingested together. Range keys may also share the sequence number if they
// were ingested, but range keys are interleaved into the compaction
// iterator's input iterator at the maximal sequence number so their
// original sequence number will not be observed here.
if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) &&
i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete {
prevKey := i.key
prevKey.Trailer = i.keyTrailer
panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey))
panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key))
}
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripeNewKey
} else if !i.equal(i.key.UserKey, key.UserKey) {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripeNewKey
}
origSnapshotIdx := i.curSnapshotIdx
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
switch key.Kind() {
case InternalKeyKindRangeDelete:
// Range tombstones need to be exposed by the compactionIter to the upper level
// `compaction` object, so return them regardless of whether they are in the same
// snapshot stripe.
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
switch key.Kind() {
case InternalKeyKindRangeDelete:
// Range tombstones need to be exposed by the compactionIter to the upper level
// `compaction` object, so return them regardless of whether they are in the same
// snapshot stripe.
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripeSameKey
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
// Range keys are interleaved at the max sequence number for a given user
// key, so we should not see any more range keys in this stripe.
panic("unreachable")
case InternalKeyKindInvalid:
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripeSameKey
case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete,
InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
// Fall through
default:
i.iterKey = nil
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
return newStripeNewKey
}
return newStripeSameKey
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
// Range keys are interleaved at the max sequence number for a given user
// key, so we should not see any more range keys in this stripe.
panic("unreachable")
case InternalKeyKindInvalid:
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
// Same snapshot.
if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
continue
}
return sameStripeSkippable
}
return newStripeSameKey
}
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeSkippable
}
return newStripeSameKey
}
func (i *compactionIter) setNext() {
@@ -639,6 +769,10 @@ func (i *compactionIter) setNext() {
// Else, we continue to loop through entries in the stripe looking for a
// DEL. Note that we may stop *before* encountering a DEL, if one exists.
//
// NB: nextInStripe will skip sameStripeSkippable keys that are visibly
// covered by a RANGEDEL. This can include DELs -- this is fine since such
// DELs don't need to be combined with SET to make SETWITHDEL.
for {
switch i.nextInStripe() {
case newStripeNewKey, newStripeSameKey:
@@ -686,14 +820,19 @@ func (i *compactionIter) setNext() {
i.skip = true
return
case sameStripeSkippable:
// We're still in the same stripe. If this is a DEL/SINGLEDEL, we
// stop looking and emit a SETWITHDEL. Subsequent keys are
// eligible for skipping.
if i.iterKey.Kind() == InternalKeyKindDelete ||
i.iterKey.Kind() == InternalKeyKindSingleDelete {
// We're still in the same stripe. If this is a
// DEL/SINGLEDEL/DELSIZED, we stop looking and emit a SETWITHDEL.
// Subsequent keys are eligible for skipping.
switch i.iterKey.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
i.key.SetKind(InternalKeyKindSetWithDelete)
i.skip = true
return
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// Do nothing
default:
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
}
default:
panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange)))
@@ -713,13 +852,24 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
i.pos = iterPosNext
return i.iterStripeChange
}
if i.err != nil {
panic(i.err)
}
// NB: MERGE#10+RANGEDEL#9 stays a MERGE, since nextInStripe skips
// sameStripeSkippable keys that are visibly covered by a RANGEDEL. There
// may be MERGE#7 that is invisibly covered and will be preserved, but
// there is no risk that MERGE#10 and MERGE#7 will get merged in the
// future as the RANGEDEL still exists and will be used in user-facing
// reads that see MERGE#10, and will also eventually cause MERGE#7 to be
// deleted in a compaction.
key := i.iterKey
switch key.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
// We've hit a deletion tombstone. Return everything up to this point and
// then skip entries until the next snapshot stripe. We change the kind
// of the result key to a Set so that it shadows keys in lower
// levels. That is, MERGE+DEL -> SET.
// levels. That is, MERGE+DEL -> SETWITHDEL.
//
// We do the same for SingleDelete since SingleDelete is only
// permitted (with deterministic behavior) for keys that have been
// set once since the last SingleDelete/Delete, so everything
@@ -732,21 +882,11 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
// single Set, and then merge in any following Sets, but that is
// complicated wrt code and unnecessary given the narrow permitted
// use of SingleDelete.
i.key.SetKind(InternalKeyKindSet)
i.key.SetKind(InternalKeyKindSetWithDelete)
i.skip = true
return sameStripeSkippable
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
// MERGE+DEL.
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
}
// We've hit a Set or SetWithDel value. Merge with the existing
// value and return. We change the kind of the resulting key to a
// Set so that it shadows keys in lower levels. That is:
@@ -761,16 +901,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
return sameStripeSkippable
case InternalKeyKindMerge:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
// MERGE+DEL.
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
}
// We've hit another Merge value. Merge with the existing value and
// continue looping.
i.err = valueMerger.MergeOlder(i.iterValue)
@@ -787,6 +917,16 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
}
}
// singleDeleteNext processes a SingleDelete point tombstone. A SingleDelete, or
// SINGLEDEL, is unique in that it deletes exactly 1 internal key. It's a
// performance optimization when the client knows a user key has not been
// overwritten, allowing the elision of the tombstone earlier, avoiding write
// amplification.
//
// singleDeleteNext returns a boolean indicating whether or not the caller
// should yield the SingleDelete key to the consumer of the compactionIter. If
// singleDeleteNext returns false, the caller may consume/elide the
// SingleDelete.
func (i *compactionIter) singleDeleteNext() bool {
// Save the current key.
i.saveKey()
@@ -795,26 +935,85 @@ func (i *compactionIter) singleDeleteNext() bool {
// Loop until finds a key to be passed to the next level.
for {
// If we find a key that can't be skipped, return true so that the
// caller yields the SingleDelete to the caller.
if i.nextInStripe() != sameStripeSkippable {
// This defers additional error checking regarding single delete
// invariants to the compaction where the keys with the same user key as
// the single delete are in the same stripe.
i.pos = iterPosNext
return true
return i.err == nil
}
if i.err != nil {
panic(i.err)
}
// INVARIANT: sameStripeSkippable.
key := i.iterKey
switch key.Kind() {
case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// We've hit a Delete, Merge or SetWithDelete, transform the
// SingleDelete into a full Delete.
kind := key.Kind()
switch kind {
case InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
if (kind == InternalKeyKindDelete || kind == InternalKeyKindDeleteSized) &&
i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
// We've hit a Delete, DeleteSized, SetWithDelete, transform
// the SingleDelete into a full Delete.
i.key.SetKind(InternalKeyKindDelete)
i.skip = true
return true
case InternalKeyKindSet:
i.nextInStripe()
case InternalKeyKindSet, InternalKeyKindMerge:
// This SingleDelete deletes the Set/Merge, and we can now elide the
// SingleDel as well. We advance past the Set and return false to
// indicate to the main compaction loop that we should NOT yield the
// current SingleDel key to the compaction loop.
//
// NB: singleDeleteNext was called with i.pos == iterPosCurForward, and
// after the call to nextInStripe, we are still at iterPosCurForward,
// since we are at the key after the Set/Merge that was single deleted.
change := i.nextInStripe()
switch change {
case sameStripeSkippable, newStripeSameKey:
// On the same user key.
nextKind := i.iterKey.Kind()
switch nextKind {
case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
if i.singleDeleteInvariantViolationCallback != nil {
// sameStripeSkippable keys returned by nextInStripe() are already
// known to not be covered by a RANGEDEL, so it is an invariant
// violation. The rare case is newStripeSameKey, where it is a
// violation if not covered by a RANGEDEL.
if change == sameStripeSkippable ||
i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.NoCover {
i.singleDeleteInvariantViolationCallback(i.key.UserKey)
}
}
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
InternalKeyKindRangeDelete:
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
case sameStripeNonSkippable:
// No ability to check whether there is another Set/Merge below with
// the same user key.
//
// TODO(sumeer): once range deletions are interleaved at the maximal
// sequence number, this case will go away.
case newStripeNewKey:
default:
panic("unreachable")
}
i.valid = false
return false
case InternalKeyKindSingleDelete:
// Two single deletes met in a compaction. The first single delete is
// ineffectual.
if i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
// Continue to apply the second single delete.
continue
default:
@@ -825,6 +1024,303 @@ func (i *compactionIter) singleDeleteNext() bool {
}
}
// skipDueToSingleDeleteElision is called when the SingleDelete is being
// elided because it is in the final snapshot stripe and there are no keys
// with the same user key in lower levels in the LSM (below the files in this
// compaction).
//
// TODO(sumeer): the only difference between singleDeleteNext and
// skipDueToSingleDeleteElision is the fact that the caller knows it will be
// eliding the single delete in the latter case. There are some similar things
// happening in both implementations. My first attempt at combining them into
// a single method was hard to comprehend. Try again.
func (i *compactionIter) skipDueToSingleDeleteElision() {
for {
stripeChange := i.nextInStripe()
if i.err != nil {
panic(i.err)
}
switch stripeChange {
case newStripeNewKey:
// The single delete is only now being elided, meaning it did not elide
// any keys earlier in its descent down the LSM. We stepped onto a new
// user key, meaning that even now at its moment of elision, it still
// hasn't elided any other keys. The single delete was ineffectual (a
// no-op).
if i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
i.skip = false
return
case newStripeSameKey:
// This should be impossible. If we're eliding a single delete, we
// determined that the tombstone is in the final snapshot stripe, but we
// stepped into a new stripe of the same key.
panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
case sameStripeNonSkippable:
// There's a key that we cannot skip. There are two possible cases:
// a. The key is invalid. This is an error.
// b. The key is a range deletion.
// The second case may also be an ineffectual single delete. However, it
// is possible that there is a SET that is at the same seqnum as the
// RANGEDEL, and so is not deleted by that RANGEDEL, and will be deleted
// by this single delete. So we cannot be certain that this is an
// ineffectual single delete.
//
// TODO(sumeer): the existing todo to interleave range deletions at the
// maximal sequence number will allow us to address this ambiguity.
//
// TODO(sumeer): by setting skip to true, the compactionIter is making a
// single delete stronger (like a del), which will hide bugs in the use of
// single delete.
i.skip = true
return
case sameStripeSkippable:
kind := i.iterKey.Kind()
switch kind {
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
if i.ineffectualSingleDeleteCallback != nil {
i.ineffectualSingleDeleteCallback(i.key.UserKey)
}
switch kind {
case InternalKeyKindDelete, InternalKeyKindDeleteSized:
i.skipInStripe()
return
case InternalKeyKindSingleDelete:
// Repeat the same with this SingleDelete. We don't want to simply
// call skipInStripe(), since it increases the strength of the
// SingleDel, which hides bugs in the use of single delete.
continue
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
case InternalKeyKindSetWithDelete:
// The SingleDelete should behave like a Delete.
i.skipInStripe()
return
case InternalKeyKindSet, InternalKeyKindMerge:
// This SingleDelete deletes the Set/Merge, and we are eliding the
// SingleDel as well. Step to the next key (this is not deleted by the
// SingleDelete).
//
// NB: skipDueToSingleDeleteElision was called with i.pos ==
// iterPosCurForward, and after the call to nextInStripe, we are still
// at iterPosCurForward, since we are at the key after the Set/Merge
// that was single deleted.
change := i.nextInStripe()
if i.err != nil {
panic(i.err)
}
switch change {
case newStripeSameKey:
panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
case newStripeNewKey:
case sameStripeSkippable:
// On the same key.
nextKind := i.iterKey.Kind()
switch nextKind {
case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
if i.singleDeleteInvariantViolationCallback != nil {
i.singleDeleteInvariantViolationCallback(i.key.UserKey)
}
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
InternalKeyKindRangeDelete:
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
case sameStripeNonSkippable:
// No ability to check whether there is another Set/Merge below with
// the same user key.
//
// TODO(sumeer): once range deletions are interleaved at the maximal
// sequence number, this case will go away.
default:
panic("unreachable")
}
// Whether in same stripe or new stripe, this key is not consumed by
// the SingleDelete.
i.skip = false
return
default:
panic(errors.AssertionFailedf(
"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
}
default:
panic("unreachable")
}
}
}
// deleteSizedNext processes a DELSIZED point tombstone. Unlike ordinary DELs,
// these tombstones carry a value that's a varint indicating the size of the
// entry (len(key)+len(value)) that the tombstone is expected to delete.
//
// When a deleteSizedNext is encountered, we skip ahead to see which keys, if
// any, are elided as a result of the tombstone.
func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
i.saveKey()
i.valid = true
i.skip = true
// The DELSIZED tombstone may have no value at all. This happens when the
// tombstone has already deleted the key that the user originally predicted.
// In this case, we still peek forward in case there's another DELSIZED key
// with a lower sequence number, in which case we'll adopt its value.
if len(i.iterValue) == 0 {
i.value = i.valueBuf[:0]
} else {
i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
i.value = i.valueBuf
}
// Loop through all the keys within this stripe that are skippable.
i.pos = iterPosNext
for i.nextInStripe() == sameStripeSkippable {
if i.err != nil {
panic(i.err)
}
switch i.iterKey.Kind() {
case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
// We encountered a tombstone (DEL, or DELSIZED) that's deleted by
// the original DELSIZED tombstone. This can happen in two cases:
//
// (1) These tombstones were intended to delete two distinct values,
// and this DELSIZED has already dropped the relevant key. For
// example:
//
// a.DELSIZED.9 a.SET.7 a.DELSIZED.5 a.SET.4
//
// If a.DELSIZED.9 has already deleted a.SET.7, its size has
// already been zeroed out. In this case, we want to adopt the
// value of the DELSIZED with the lower sequence number, in
// case the a.SET.4 key has not yet been elided.
//
// (2) This DELSIZED was missized. The user thought they were
// deleting a key with this user key, but this user key had
// already been deleted.
//
// We can differentiate these two cases by examining the length of
// the DELSIZED's value. A DELSIZED's value holds the size of both
// the user key and value that it intends to delete. For any user
// key with a length > 0, a DELSIZED that has not deleted a key must
// have a value with a length > 0.
//
// We treat both cases the same functionally, adopting the identity
// of the lower-sequence numbered tombstone. However in the second
// case, we also increment the stat counting missized tombstones.
if len(i.value) > 0 {
// The original DELSIZED key was missized. The key that the user
// thought they were deleting does not exist.
i.stats.countMissizedDels++
}
i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
i.value = i.valueBuf
if i.iterKey.Kind() != InternalKeyKindDeleteSized {
// Convert the DELSIZED to a DEL—The DEL/SINGLEDEL we're eliding
// may not have deleted the key(s) it was intended to yet. The
// ordinary DEL compaction heuristics are better suited at that,
// plus we don't want to count it as a missized DEL. We early
// exit in this case, after skipping the remainder of the
// snapshot stripe.
i.key.SetKind(InternalKeyKindDelete)
// NB: We skipInStripe now, rather than returning leaving
// i.skip=true and returning early, because Next() requires
// that i.skip=true only if i.iterPos = iterPosCurForward.
//
// Ignore any error caused by skipInStripe since it does not affect
// the key/value being returned here, and the next call to Next() will
// expose it.
i.skipInStripe()
return &i.key, i.value
}
// Continue, in case we uncover another DELSIZED or a key this
// DELSIZED deletes.
case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// If the DELSIZED is value-less, it already deleted the key that it
// was intended to delete. This is possible with a sequence like:
//
// DELSIZED.8 SET.7 SET.3
//
// The DELSIZED only describes the size of the SET.7, which in this
// case has already been elided. We don't count it as a missizing,
// instead converting the DELSIZED to a DEL. Skip the remainder of
// the snapshot stripe and return.
if len(i.value) == 0 {
i.key.SetKind(InternalKeyKindDelete)
// NB: We skipInStripe now, rather than returning leaving
// i.skip=true and returning early, because Next() requires
// that i.skip=true only if i.iterPos = iterPosCurForward.
//
// Ignore any error caused by skipInStripe since it does not affect
// the key/value being returned here, and the next call to Next() will
// expose it.
i.skipInStripe()
return &i.key, i.value
}
// The deleted key is not a DEL, DELSIZED, and the DELSIZED in i.key
// has a positive size.
expectedSize, n := binary.Uvarint(i.value)
if n != len(i.value) {
i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value))
i.valid = false
return nil, nil
}
elidedSize := uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue))
if elidedSize != expectedSize {
// The original DELSIZED key was missized. It's unclear what to
// do. The user-provided size was wrong, so it's unlikely to be
// accurate or meaningful. We could:
//
// 1. return the DELSIZED with the original user-provided size unmodified
// 2. return the DELZIZED with a zeroed size to reflect that a key was
// elided, even if it wasn't the anticipated size.
// 3. subtract the elided size from the estimate and re-encode.
// 4. convert the DELSIZED into a value-less DEL, so that
// ordinary DEL heuristics apply.
//
// We opt for (4) under the rationale that we can't rely on the
// user-provided size for accuracy, so ordinary DEL heuristics
// are safer.
i.stats.countMissizedDels++
i.key.SetKind(InternalKeyKindDelete)
i.value = i.valueBuf[:0]
// NB: We skipInStripe now, rather than returning leaving
// i.skip=true and returning early, because Next() requires
// that i.skip=true only if i.iterPos = iterPosCurForward.
//
// Ignore any error caused by skipInStripe since it does not affect
// the key/value being returned here, and the next call to Next() will
// expose it.
i.skipInStripe()
return &i.key, i.value
}
// NB: We remove the value regardless of whether the key was sized
// appropriately. The size encoded is 'consumed' the first time it
// meets a key that it deletes.
i.value = i.valueBuf[:0]
default:
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
return nil, nil
}
}
// Reset skip if we landed outside the original stripe. Otherwise, we landed
// in the same stripe on a non-skippable key. In that case we should preserve
// `i.skip == true` such that later keys in the stripe will continue to be
// skipped.
if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey {
i.skip = false
}
if i.err != nil {
return nil, nil
}
return &i.key, i.value
}
func (i *compactionIter) saveKey() {
i.keyBuf = append(i.keyBuf[:0], i.iterKey.UserKey...)
i.key.UserKey = i.keyBuf
@@ -957,7 +1453,7 @@ func (i *compactionIter) maybeZeroSeqnum(snapshotIdx int) {
// This is not the last snapshot
return
}
i.key.SetSeqNum(0)
i.key.SetSeqNum(base.SeqNumZero)
}
// A frontier is used to monitor a compaction's progression across the user

685
vendor/github.com/cockroachdb/pebble/compaction_picker.go generated vendored
View File

File diff suppressed because it is too large Load Diff

982
vendor/github.com/cockroachdb/pebble/db.go generated vendored
View File

File diff suppressed because it is too large Load Diff

80
vendor/github.com/cockroachdb/pebble/event.go generated vendored
View File

@@ -39,10 +39,11 @@ func formatFileNums(tables []TableInfo) string {
return buf.String()
}
// LevelInfo contains info pertaining to a partificular level.
// LevelInfo contains info pertaining to a particular level.
type LevelInfo struct {
Level int
Tables []TableInfo
Score float64
}
func (i LevelInfo) String() string {
@@ -51,8 +52,11 @@ func (i LevelInfo) String() string {
// SafeFormat implements redact.SafeFormatter.
func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables))))
w.Printf("L%d [%s] (%s) Score=%.2f",
redact.Safe(i.Level),
redact.Safe(formatFileNums(i.Tables)),
redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))),
redact.Safe(i.Score))
}
// CompactionInfo contains the info for a compaction event.
@@ -75,6 +79,27 @@ type CompactionInfo struct {
TotalDuration time.Duration
Done bool
Err error
SingleLevelOverlappingRatio float64
MultiLevelOverlappingRatio float64
// Annotations specifies additional info to appear in a compaction's event log line
Annotations compactionAnnotations
}
type compactionAnnotations []string
// SafeFormat implements redact.SafeFormatter.
func (ca compactionAnnotations) SafeFormat(w redact.SafePrinter, _ rune) {
if len(ca) == 0 {
return
}
for i := range ca {
if i != 0 {
w.Print(" ")
}
w.Printf("%s", redact.SafeString(ca[i]))
}
}
func (i CompactionInfo) String() string {
@@ -90,20 +115,25 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
}
if !i.Done {
w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Print(levelInfos(i.Input))
w.Printf("[JOB %d] compacting(%s) ",
redact.Safe(i.JobID),
redact.SafeString(i.Reason))
w.Printf("%s", i.Annotations)
w.Printf("%s; ", levelInfos(i.Input))
w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio)
return
}
outputSize := tablesTotalSize(i.Output.Tables)
w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Printf("%s", i.Annotations)
w.Print(levelInfos(i.Input))
w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Output.Level),
redact.Safe(formatFileNums(i.Output.Tables)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(humanize.Bytes.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
type levelInfos []LevelInfo
@@ -129,6 +159,9 @@ type FlushInfo struct {
Reason string
// Input contains the count of input memtables that were flushed.
Input int
// InputBytes contains the total in-memory size of the memtable(s) that were
// flushed. This size includes skiplist indexing data structures.
InputBytes uint64
// Output contains the ouptut table generated by the flush. The output info
// is empty for the flush begin event.
Output []TableInfo
@@ -168,7 +201,7 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if !i.Ingest {
w.Printf("flushing %d memtable", redact.Safe(i.Input))
w.SafeString(plural)
w.Printf(" to L0")
w.Printf(" (%s) to L0", redact.Safe(humanize.Bytes.Uint64(i.InputBytes)))
} else {
w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural)
}
@@ -180,13 +213,14 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if invariants.Enabled && len(i.IngestLevels) > 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0"))
}
w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
w.Printf("[JOB %d] flushed %d memtable%s (%s) to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.JobID), redact.Safe(i.Input), plural,
redact.Safe(humanize.Bytes.Uint64(i.InputBytes)),
redact.Safe(formatFileNums(i.Output)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(humanize.Bytes.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
} else {
if invariants.Enabled && len(i.IngestLevels) == 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0"))
@@ -198,12 +232,12 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if j > 0 {
w.Printf(" +")
}
w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.IEC.Uint64(file.Size))
w.Printf(" L%d:%s (%s)", level, file.FileNum, humanize.Bytes.Uint64(file.Size))
}
w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
}
@@ -227,7 +261,7 @@ func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), i.FileNum)
}
// ManifestDeleteInfo contains the info for a Manifest deletion event.
@@ -249,7 +283,7 @@ func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), i.FileNum)
}
// TableCreateInfo contains the info for a table creation event.
@@ -269,7 +303,7 @@ func (i TableCreateInfo) String() string {
// SafeFormat implements redact.SafeFormatter.
func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] %s: sstable created %s",
redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
redact.Safe(i.JobID), redact.Safe(i.Reason), i.FileNum)
}
// TableDeleteInfo contains the info for a table deletion event.
@@ -288,10 +322,10 @@ func (i TableDeleteInfo) String() string {
func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] sstable delete error %s: %s",
redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
redact.Safe(i.JobID), i.FileNum, i.Err)
return
}
w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), i.FileNum)
}
// TableIngestInfo contains the info for a table ingestion event.
@@ -337,8 +371,8 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if !i.flushable {
levelStr = fmt.Sprintf("L%d:", t.Level)
}
w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum),
redact.Safe(humanize.Uint64(t.Size)))
w.Printf(" %s%s (%s)", redact.Safe(levelStr), t.FileNum,
redact.Safe(humanize.Bytes.Uint64(t.Size)))
}
}
@@ -399,12 +433,12 @@ func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
}
if i.RecycledFileNum == 0 {
w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), i.FileNum)
return
}
w.Printf("[JOB %d] WAL created %s (recycled %s)",
redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
redact.Safe(i.JobID), i.FileNum, i.RecycledFileNum)
}
// WALDeleteInfo contains the info for a WAL deletion event.
@@ -426,7 +460,7 @@ func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), i.FileNum)
}
// WriteStallBeginInfo contains the info for a write stall begin event.

28
vendor/github.com/cockroachdb/pebble/external_iterator.go generated vendored
View File

@@ -209,15 +209,15 @@ func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterato
pointIter internalIterator
err error
)
pointIter, err = r.NewIterWithBlockPropertyFiltersAndContext(
ctx,
it.opts.LowerBound,
it.opts.UpperBound,
nil, /* BlockPropertiesFilterer */
false, /* useFilterBlock */
&it.stats.InternalStats,
sstable.TrivialReaderProvider{Reader: r},
)
// We could set hideObsoletePoints=true, since we are reading at
// InternalKeySeqNumMax, but we don't bother since these sstables should
// not have obsolete points (so the performance optimization is
// unnecessary), and we don't want to bother constructing a
// BlockPropertiesFilterer that includes obsoleteKeyBlockPropertyFilter.
pointIter, err = r.NewIterWithBlockPropertyFiltersAndContextEtc(
ctx, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */
false /* hideObsoletePoints */, false, /* useFilterBlock */
&it.stats.InternalStats, sstable.TrivialReaderProvider{Reader: r})
if err != nil {
return nil, err
}
@@ -311,7 +311,7 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
base.InternalKeySeqNumMax,
it.opts.LowerBound, it.opts.UpperBound,
&it.hasPrefix, &it.prefixOrFullSeekKey,
true /* onlySets */, &it.rangeKey.internal,
false /* internalKeys */, &it.rangeKey.internal,
)
for i := range rangeKeyIters {
it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])
@@ -319,8 +319,12 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
}
}
if it.rangeKey != nil {
it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter, &it.rangeKeyMasking,
it.opts.LowerBound, it.opts.UpperBound)
it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter,
keyspan.InterleavingIterOpts{
Mask: &it.rangeKeyMasking,
LowerBound: it.opts.LowerBound,
UpperBound: it.opts.UpperBound,
})
it.iter = &it.rangeKey.iiter
}
}

2
vendor/github.com/cockroachdb/pebble/filenames.go generated vendored
View File

@@ -33,7 +33,7 @@ const (
// NB: This is a low-level routine and typically not what you want to
// use. Newer versions of Pebble running newer format major versions do
// not use the CURRENT file. See setCurrentFunc in version_set.go.
func setCurrentFile(dirname string, fs vfs.FS, fileNum FileNum) error {
func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error {
newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum)
oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum)
fs.Remove(oldFilename)

62
vendor/github.com/cockroachdb/pebble/flushable.go generated vendored
View File

@@ -60,20 +60,20 @@ type flushableEntry struct {
// flushable is a memTable, when the reader refs drops to zero, the writer
// refs will already be zero because the memtable will have been flushed and
// that only occurs once the writer refs drops to zero.
readerRefs int32
readerRefs atomic.Int32
// Closure to invoke to release memory accounting.
releaseMemAccounting func()
// unrefFiles, if not nil, should be invoked to decrease the ref count of
// files which are backing the flushable.
unrefFiles func() []*fileMetadata
unrefFiles func() []*fileBacking
// deleteFnLocked should be called if the caller is holding DB.mu.
deleteFnLocked func(obsolete []*fileMetadata)
deleteFnLocked func(obsolete []*fileBacking)
// deleteFn should be called if the caller is not holding DB.mu.
deleteFn func(obsolete []*fileMetadata)
deleteFn func(obsolete []*fileBacking)
}
func (e *flushableEntry) readerRef() {
switch v := atomic.AddInt32(&e.readerRefs, 1); {
switch v := e.readerRefs.Add(1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
@@ -90,9 +90,9 @@ func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) {
}
func (e *flushableEntry) readerUnrefHelper(
deleteFiles bool, deleteFn func(obsolete []*manifest.FileMetadata),
deleteFiles bool, deleteFn func(obsolete []*fileBacking),
) {
switch v := atomic.AddInt32(&e.readerRefs, -1); {
switch v := e.readerRefs.Add(-1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
@@ -116,9 +116,8 @@ type flushableList []*flushableEntry
// ingestedFlushable is the implementation of the flushable interface for the
// ingesting sstables which are added to the flushable list.
type ingestedFlushable struct {
files []*fileMetadata
cmp Compare
split Split
files []physicalMeta
comparer *Comparer
newIters tableNewIters
newRangeKeyIters keyspan.TableNewSpanIter
@@ -131,26 +130,27 @@ type ingestedFlushable struct {
func newIngestedFlushable(
files []*fileMetadata,
cmp Compare,
split Split,
comparer *Comparer,
newIters tableNewIters,
newRangeKeyIters keyspan.TableNewSpanIter,
) *ingestedFlushable {
var physicalFiles []physicalMeta
var hasRangeKeys bool
for _, f := range files {
if f.HasRangeKeys {
hasRangeKeys = true
}
physicalFiles = append(physicalFiles, f.PhysicalMeta())
}
ret := &ingestedFlushable{
files: files,
cmp: cmp,
split: split,
files: physicalFiles,
comparer: comparer,
newIters: newIters,
newRangeKeyIters: newRangeKeyIters,
// slice is immutable and can be set once and used many times.
slice: manifest.NewLevelSliceKeySorted(cmp, files),
}
for _, f := range files {
if f.HasRangeKeys {
ret.hasRangeKeys = true
break
}
slice: manifest.NewLevelSliceKeySorted(comparer.Compare, files),
hasRangeKeys: hasRangeKeys,
}
return ret
@@ -159,6 +159,7 @@ func newIngestedFlushable(
// TODO(sumeer): ingestedFlushable iters also need to plumb context for
// tracing.
// newIter is part of the flushable interface.
func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
var opts IterOptions
if o != nil {
@@ -169,10 +170,11 @@ func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
// aren't truly levels in the lsm. Right now, the encoding only supports
// L0 sublevels, and the rest of the levels in the lsm.
return newLevelIter(
opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), nil,
opts, s.comparer, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
)
}
// newFlushIter is part of the flushable interface.
func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
// newFlushIter is only used for writing memtables to disk as sstables.
// Since ingested sstables are already present on disk, they don't need to
@@ -181,7 +183,7 @@ func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) i
}
func (s *ingestedFlushable) constructRangeDelIter(
file *manifest.FileMetadata, _ *keyspan.SpanIterOptions,
file *manifest.FileMetadata, _ keyspan.SpanIterOptions,
) (keyspan.FragmentIterator, error) {
// Note that the keyspan level iter expects a non-nil iterator to be
// returned even if there is an error. So, we return the emptyKeyspanIter.
@@ -196,41 +198,47 @@ func (s *ingestedFlushable) constructRangeDelIter(
return rangeDelIter, nil
}
// newRangeDelIter is part of the flushable interface.
// TODO(bananabrick): Using a level iter instead of a keyspan level iter to
// surface range deletes is more efficient.
func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp,
keyspan.SpanIterOptions{}, s.comparer.Compare,
s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
manifest.KeyTypePoint,
)
}
// newRangeKeyIter is part of the flushable interface.
func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
if !s.containsRangeKeys() {
return nil
}
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters,
keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters,
s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
)
}
// containsRangeKeys is part of the flushable interface.
func (s *ingestedFlushable) containsRangeKeys() bool {
return s.hasRangeKeys
}
// inuseBytes is part of the flushable interface.
func (s *ingestedFlushable) inuseBytes() uint64 {
// inuseBytes is only used when memtables are flushed to disk as sstables.
panic("pebble: not implemented")
}
// totalBytes is part of the flushable interface.
func (s *ingestedFlushable) totalBytes() uint64 {
// We don't allocate additional bytes for the ingestedFlushable.
return 0
}
// readyForFlush is part of the flushable interface.
func (s *ingestedFlushable) readyForFlush() bool {
// ingestedFlushable should always be ready to flush. However, note that
// memtables before the ingested sstables in the memtable queue must be

106
vendor/github.com/cockroachdb/pebble/format_major_version.go generated vendored
View File

@@ -28,10 +28,13 @@ import (
// FormatVersion that the default corresponds to may change with time.
type FormatMajorVersion uint64
// SafeValue implements redact.SafeValue.
func (v FormatMajorVersion) SafeValue() {}
// String implements fmt.Stringer.
func (v FormatMajorVersion) String() string {
// NB: This must not change. It's used as the value for the the
// on-disk version marker file.
// NB: This must not change. It's used as the value for the on-disk
// version marker file.
//
// Specifically, this value must always parse as a base 10 integer
// that fits in a uint64. We format it as zero-padded, 3-digit
@@ -110,12 +113,12 @@ const (
// 23.1 versions.
// FormatUnusedPrePebblev1MarkedCompacted is an unused format major version.
// formatUnusedPrePebblev1MarkedCompacted is an unused format major version.
// This format major version was originally intended to ship in the 23.1
// release. It was later decided that this should be deferred until a
// subsequent release. The original ordering is preserved so as not to
// introduce breaking changes in Cockroach.
FormatUnusedPrePebblev1MarkedCompacted
formatUnusedPrePebblev1MarkedCompacted
// FormatSSTableValueBlocks is a format major version that adds support for
// storing values in value blocks in the sstable. Value block support is not
@@ -151,8 +154,28 @@ const (
// compactions for files marked for compaction are complete.
FormatPrePebblev1MarkedCompacted
// FormatDeleteSizedAndObsolete is a format major version that adds support
// for deletion tombstones that encode the size of the value they're
// expected to delete. This format major version is required before the
// associated key kind may be committed through batch applications or
// ingests. It also adds support for keys that are marked obsolete (see
// sstable/format.go for details).
FormatDeleteSizedAndObsolete
// FormatVirtualSSTables is a format major version that adds support for
// virtual sstables that can reference a sub-range of keys in an underlying
// physical sstable. This information is persisted through new,
// backward-incompatible fields in the Manifest, and therefore requires
// a format major version.
FormatVirtualSSTables
// internalFormatNewest holds the newest format major version, including
// experimental ones excluded from the exported FormatNewest constant until
// they've stabilized. Used in tests.
internalFormatNewest FormatMajorVersion = iota - 1
// FormatNewest always contains the most recent format major version.
FormatNewest FormatMajorVersion = iota - 1
FormatNewest FormatMajorVersion = internalFormatNewest
)
// MaxTableFormat returns the maximum sstable.TableFormat that can be used at
@@ -166,11 +189,12 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
FormatSplitUserKeysMarkedCompacted:
return sstable.TableFormatPebblev1
case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatUnusedPrePebblev1MarkedCompacted:
formatUnusedPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev2
case FormatSSTableValueBlocks, FormatFlushableIngest,
FormatPrePebblev1MarkedCompacted:
case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev3
case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
return sstable.TableFormatPebblev4
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
}
@@ -186,14 +210,27 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
FormatRangeKeys:
return sstable.TableFormatLevelDB
case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
return sstable.TableFormatPebblev1
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
}
}
// orderingInvariants returns an enum encoding the set of invariants that must
// hold within the receiver format major version. Invariants only get stricter
// as the format major version advances, so it is okay to retrieve the
// invariants from the current format major version and by the time the
// invariants are enforced, the format major version has advanced.
func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants {
if v < FormatSplitUserKeysMarkedCompacted {
return manifest.AllowSplitUserKeys
}
return manifest.ProhibitSplitUserKeys
}
// formatMajorVersionMigrations defines the migrations from one format
// major version to the next. Each migration is defined as a closure
// which will be invoked on the database before the new format major
@@ -219,7 +256,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
// guaranteed to exist, because we unconditionally locate it
// during Open.
manifestFileNum := d.mu.versions.manifestFileNum
filename := base.MakeFilename(fileTypeManifest, manifestFileNum)
filename := base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum())
if err := d.mu.versions.manifestMarker.Move(filename); err != nil {
return errors.Wrap(err, "moving manifest marker")
}
@@ -257,7 +294,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
// version that does not know about format major versions
// attempts to open the database, it will error avoiding
// accidental corruption.
if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, 0); err != nil {
if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatVersioned)
@@ -301,9 +338,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
},
FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
// Intentional no-op.
return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted)
return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted)
},
FormatSSTableValueBlocks: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
@@ -320,6 +357,12 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
},
FormatDeleteSizedAndObsolete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete)
},
FormatVirtualSSTables: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
},
}
const formatVersionMarkerName = `format-version`
@@ -342,7 +385,7 @@ func lookupFormatMajorVersion(
if vers == FormatDefault {
return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers)
}
if vers > FormatNewest {
if vers > internalFormatNewest {
return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers)
}
return vers, m, nil
@@ -376,8 +419,8 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
if d.opts.ReadOnly {
return ErrReadOnly
}
if formatVers > FormatNewest {
// Guard against accidentally forgetting to update FormatNewest.
if formatVers > internalFormatNewest {
// Guard against accidentally forgetting to update internalFormatNewest.
return errors.Errorf("pebble: unknown format version %d", formatVers)
}
if currentVers := d.FormatMajorVersion(); currentVers > formatVers {
@@ -519,17 +562,24 @@ var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc {
for l := numLevels - 1; l > 0; l-- {
iter := v.Levels[l].Iter()
for f := iter.First(); f != nil; f = iter.Next() {
err = tc.withReader(f, func(r *sstable.Reader) error {
tf, err := r.TableFormat()
if err != nil {
return err
}
if tf < sstable.TableFormatPebblev1 {
found = true
files[l] = append(files[l], f)
}
return nil
})
if f.Virtual {
// Any physical sstable which has been virtualized must
// have already undergone this migration, and we don't
// need to worry about the virtual sstable themselves.
panic("pebble: unexpected virtual sstable during migration")
}
err = tc.withReader(
f.PhysicalMeta(), func(r *sstable.Reader) error {
tf, err := r.TableFormat()
if err != nil {
return err
}
if tf < sstable.TableFormatPebblev1 {
found = true
files[l] = append(files[l], f)
}
return nil
})
if err != nil {
return
}

45
vendor/github.com/cockroachdb/pebble/get_iter.go generated vendored
View File

@@ -19,8 +19,7 @@ import (
// lazily.
type getIter struct {
logger Logger
cmp Compare
equal Equal
comparer *Comparer
newIters tableNewIters
snapshot uint64
key []byte
@@ -83,7 +82,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
// key. Every call to levelIter.Next() potentially switches to a new
// table and thus reinitializes rangeDelIter.
if g.rangeDelIter != nil {
g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key)
g.tombstone = keyspan.Get(g.comparer.Compare, g.rangeDelIter, g.key)
if g.err = g.rangeDelIter.Close(); g.err != nil {
return nil, base.LazyValue{}
}
@@ -101,7 +100,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
g.iter = nil
return nil, base.LazyValue{}
}
if g.equal(g.key, key.UserKey) {
if g.comparer.Equal(g.key, key.UserKey) {
if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) {
g.iterKey, g.iterValue = g.iter.Next()
continue
@@ -158,12 +157,25 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
if n := len(g.l0); n > 0 {
files := g.l0[n-1].Iter()
g.l0 = g.l0[:n-1]
iterOpts := IterOptions{logger: g.logger}
g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
files, manifest.L0Sublevel(n), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
bc := levelIterBoundaryContext{}
g.levelIter.initBoundaryContext(&bc)
g.iter = &g.levelIter
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
// Compute the key prefix for bloom filtering if split function is
// specified, or use the user key as default.
prefix := g.key
if g.comparer.Split != nil {
prefix = g.key[:g.comparer.Split(g.key)]
}
g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
g.iterKey = nil
g.iterValue = base.LazyValue{}
}
continue
}
g.level++
@@ -177,13 +189,26 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
continue
}
iterOpts := IterOptions{logger: g.logger}
g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
bc := levelIterBoundaryContext{}
g.levelIter.initBoundaryContext(&bc)
g.level++
g.iter = &g.levelIter
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
// Compute the key prefix for bloom filtering if split function is
// specified, or use the user key as default.
prefix := g.key
if g.comparer.Split != nil {
prefix = g.key[:g.comparer.Split(g.key)]
}
g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
g.iterKey = nil
g.iterValue = base.LazyValue{}
}
}
}

1458
vendor/github.com/cockroachdb/pebble/ingest.go generated vendored
View File

File diff suppressed because it is too large Load Diff

3
vendor/github.com/cockroachdb/pebble/internal.go generated vendored
View File

@@ -22,7 +22,8 @@ const (
InternalKeyKindRangeKeySet = base.InternalKeyKindRangeKeySet
InternalKeyKindRangeKeyUnset = base.InternalKeyKindRangeKeyUnset
InternalKeyKindRangeKeyDelete = base.InternalKeyKindRangeKeyDelete
InternalKeyKindIngestSST = base.InternalKeyIngestSST
InternalKeyKindIngestSST = base.InternalKeyKindIngestSST
InternalKeyKindDeleteSized = base.InternalKeyKindDeleteSized
InternalKeyKindInvalid = base.InternalKeyKindInvalid
InternalKeySeqNumBatch = base.InternalKeySeqNumBatch
InternalKeySeqNumMax = base.InternalKeySeqNumMax

56
vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go generated vendored
View File

@@ -18,22 +18,21 @@
package arenaskl
import (
"math"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/constants"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Arena is lock-free.
type Arena struct {
n uint64
n atomic.Uint64
buf []byte
}
const (
align4 = 3
)
const nodeAlignment = 4
var (
// ErrArenaFull indicates that the arena is full and cannot perform any more
@@ -44,20 +43,28 @@ var (
// NewArena allocates a new arena using the specified buffer as the backing
// store.
func NewArena(buf []byte) *Arena {
// Don't store data at position 0 in order to reserve offset=0 as a kind
// of nil pointer.
return &Arena{
n: 1,
if len(buf) > constants.MaxUint32OrInt {
if invariants.Enabled {
panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
}
buf = buf[:constants.MaxUint32OrInt]
}
a := &Arena{
buf: buf,
}
// We don't store data at position 0 in order to reserve offset=0 as a kind of
// nil pointer.
a.n.Store(1)
return a
}
// Size returns the number of bytes allocated by the arena.
func (a *Arena) Size() uint32 {
s := atomic.LoadUint64(&a.n)
if s > math.MaxUint32 {
// Saturate at MaxUint32.
return math.MaxUint32
s := a.n.Load()
if s > constants.MaxUint32OrInt {
// The last failed allocation can push the size higher than len(a.buf).
// Saturate at the maximum representable offset.
return constants.MaxUint32OrInt
}
return uint32(s)
}
@@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 {
return uint32(len(a.buf))
}
func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
// alloc allocates a buffer of the given size and with the given alignment
// (which must be a power of 2).
//
// If overflow is not 0, it also ensures that many bytes after the buffer are
// inside the arena (this is used for structures that are larger than the
// requested size but don't use those extra bytes).
func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
if invariants.Enabled && (alignment&(alignment-1)) != 0 {
panic(errors.AssertionFailedf("invalid alignment %d", alignment))
}
// Verify that the arena isn't already full.
origSize := atomic.LoadUint64(&a.n)
origSize := a.n.Load()
if int(origSize) > len(a.buf) {
return 0, 0, ErrArenaFull
}
// Pad the allocation with enough bytes to ensure the requested alignment.
padded := uint32(size) + align
padded := uint64(size) + uint64(alignment) - 1
newSize := atomic.AddUint64(&a.n, uint64(padded))
if int(newSize)+int(overflow) > len(a.buf) {
newSize := a.n.Add(padded)
if newSize+uint64(overflow) > uint64(len(a.buf)) {
return 0, 0, ErrArenaFull
}
// Return the aligned offset.
offset := (uint32(newSize) - padded + align) & ^align
return offset, padded, nil
offset := (uint32(newSize) - size) & ^(alignment - 1)
return offset, uint32(padded), nil
}
func (a *Arena) getBytes(offset uint32, size uint32) []byte {

14
vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go generated vendored
View File

@@ -18,7 +18,6 @@
package arenaskl
import (
"encoding/binary"
"sync"
"github.com/cockroachdb/pebble/internal/base"
@@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
}
func (it *Iterator) decodeKey() {
b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
// This is a manual inline of base.DecodeInternalKey, because the Go compiler
// seems to refuse to automatically inline it currently.
l := len(b) - 8
if l >= 0 {
it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
it.key.UserKey = b[:l:l]
} else {
it.key.Trailer = uint64(base.InternalKeyKindInvalid)
it.key.UserKey = nil
}
it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
it.key.Trailer = it.nd.keyTrailer
}
func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {

36
vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go generated vendored
View File

@@ -29,25 +29,27 @@ import (
// is used here. If a key/value overflows a uint32, it should not be added to
// the skiplist.
func MaxNodeSize(keySize, valueSize uint32) uint64 {
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
const maxPadding = nodeAlignment - 1
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
}
type links struct {
nextOffset uint32
prevOffset uint32
nextOffset atomic.Uint32
prevOffset atomic.Uint32
}
func (l *links) init(prevOffset, nextOffset uint32) {
l.nextOffset = nextOffset
l.prevOffset = prevOffset
l.nextOffset.Store(nextOffset)
l.prevOffset.Store(prevOffset)
}
type node struct {
// Immutable fields, so no need to lock to access key.
keyOffset uint32
keySize uint32
valueSize uint32
allocSize uint32
keyOffset uint32
keySize uint32
keyTrailer uint64
valueSize uint32
allocSize uint32
// Most nodes do not need to use the full height of the tower, since the
// probability of each successive level decreases exponentially. Because
@@ -65,7 +67,7 @@ func newNode(
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
keySize := key.Size()
keySize := len(key.UserKey)
if int64(keySize) > math.MaxUint32 {
panic("key is too large")
}
@@ -81,8 +83,8 @@ func newNode(
if err != nil {
return
}
key.Encode(nd.getKeyBytes(arena))
nd.keyTrailer = key.Trailer
copy(nd.getKeyBytes(arena), key.UserKey)
copy(nd.getValue(arena), value)
return
}
@@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod
unusedSize := uint32((maxHeight - int(height)) * linksSize)
nodeSize := uint32(maxNodeSize) - unusedSize
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
if err != nil {
return
}
@@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte {
}
func (n *node) nextOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].nextOffset)
return n.tower[h].nextOffset.Load()
}
func (n *node) prevOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].prevOffset)
return n.tower[h].prevOffset.Load()
}
func (n *node) casNextOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
return n.tower[h].nextOffset.CompareAndSwap(old, val)
}
func (n *node) casPrevOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
return n.tower[h].prevOffset.CompareAndSwap(old, val)
}

53
vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go generated vendored
View File

@@ -44,7 +44,6 @@ Key differences:
package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
import (
"encoding/binary"
"math"
"runtime"
"sync/atomic"
@@ -68,7 +67,7 @@ const (
// keys.
var ErrRecordExists = errors.New("record with this key already exists")
// Skiplist is a fast, cocnurrent skiplist implementation that supports forward
// Skiplist is a fast, concurrent skiplist implementation that supports forward
// and backward iteration. See batchskl.Skiplist for a non-concurrent
// skiplist. Keys and values are immutable once added to the skiplist and
// deletion is not supported. Instead, higher-level code is expected to add new
@@ -80,7 +79,7 @@ type Skiplist struct {
cmp base.Compare
head *node
tail *node
height uint32 // Current height. 1 <= height <= maxHeight. CAS.
height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
// If set to true by tests, then extra delays are added to make it easier to
// detect unusual race conditions.
@@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
headOffset := arena.getPointerOffset(unsafe.Pointer(head))
tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
for i := 0; i < maxHeight; i++ {
head.tower[i].nextOffset = tailOffset
tail.tower[i].prevOffset = headOffset
head.tower[i].nextOffset.Store(tailOffset)
tail.tower[i].prevOffset.Store(headOffset)
}
*s = Skiplist{
arena: arena,
cmp: cmp,
head: head,
tail: tail,
height: 1,
arena: arena,
cmp: cmp,
head: head,
tail: tail,
}
s.height.Store(1)
}
// Height returns the height of the highest tower within any of the nodes that
// have ever been allocated as part of this skiplist.
func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
func (s *Skiplist) Height() uint32 { return s.height.Load() }
// Arena returns the arena backing this skiplist.
func (s *Skiplist) Arena() *Arena { return s.arena }
@@ -327,7 +326,7 @@ func (s *Skiplist) newNode(
// Try to increase s.height via CAS.
listHeight := s.Height()
for height > listHeight {
if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
if s.height.CompareAndSwap(listHeight, height) {
// Successfully increased skiplist.height.
break
}
@@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel(
offset, size := next.keyOffset, next.keySize
nextKey := s.arena.buf[offset : offset+size]
n := int32(size) - 8
cmp := s.cmp(key.UserKey, nextKey[:n])
cmp := s.cmp(key.UserKey, nextKey)
if cmp < 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
if cmp == 0 {
// User-key equality.
var nextTrailer uint64
if n >= 0 {
nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
} else {
nextTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == nextTrailer {
if key.Trailer == next.keyTrailer {
// Internal key equality.
found = true
break
}
if key.Trailer > nextTrailer {
if key.Trailer > next.keyTrailer {
// We are done for this level, since prev.key < key < next.key.
break
}
@@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel(
func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
n := int32(nd.keySize) - 8
cmp := s.cmp(ndKey[:n], key.UserKey)
cmp := s.cmp(ndKey, key.UserKey)
if cmp < 0 {
return true
}
@@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
return false
}
// User-key equality.
var ndTrailer uint64
if n >= 0 {
ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
} else {
ndTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == ndTrailer {
if key.Trailer == nd.keyTrailer {
// Internal key equality.
return false
}
return key.Trailer < ndTrailer
return key.Trailer < nd.keyTrailer
}
func (s *Skiplist) getNext(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
offset := nd.tower[h].nextOffset.Load()
return (*node)(s.arena.getPointer(offset))
}
func (s *Skiplist) getPrev(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
offset := nd.tower[h].prevOffset.Load()
return (*node)(s.arena.getPointer(offset))
}

89
vendor/github.com/cockroachdb/pebble/internal/base/filenames.go generated vendored
View File

@@ -20,6 +20,41 @@ type FileNum uint64
// String returns a string representation of the file number.
func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
// SafeFormat implements redact.SafeFormatter.
func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("%06d", redact.SafeUint(fn))
}
// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
// called if the caller can ensure that the FileNum belongs to a physical file
// on disk. These could be manifests, log files, physical sstables on disk, the
// options file, but not virtual sstables.
func (fn FileNum) DiskFileNum() DiskFileNum {
return DiskFileNum{fn}
}
// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
// Note that a FileNum is an internal DB identifier and it could belong to files
// which don't exist on disk. An example would be virtual sstable FileNums.
// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
// FileNum to DiskFileNum may not be valid and care should be taken to prove
// that the FileNum actually exists on disk.
type DiskFileNum struct {
fn FileNum
}
func (dfn DiskFileNum) String() string { return dfn.fn.String() }
// SafeFormat implements redact.SafeFormatter.
func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
dfn.fn.SafeFormat(w, verb)
}
// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
func (dfn DiskFileNum) FileNum() FileNum {
return dfn.fn
}
// FileType enumerates the types of files found in a DB.
type FileType int
@@ -36,92 +71,92 @@ const (
)
// MakeFilename builds a filename from components.
func MakeFilename(fileType FileType, fileNum FileNum) string {
func MakeFilename(fileType FileType, dfn DiskFileNum) string {
switch fileType {
case FileTypeLog:
return fmt.Sprintf("%s.log", fileNum)
return fmt.Sprintf("%s.log", dfn)
case FileTypeLock:
return "LOCK"
case FileTypeTable:
return fmt.Sprintf("%s.sst", fileNum)
return fmt.Sprintf("%s.sst", dfn)
case FileTypeManifest:
return fmt.Sprintf("MANIFEST-%s", fileNum)
return fmt.Sprintf("MANIFEST-%s", dfn)
case FileTypeCurrent:
return "CURRENT"
case FileTypeOptions:
return fmt.Sprintf("OPTIONS-%s", fileNum)
return fmt.Sprintf("OPTIONS-%s", dfn)
case FileTypeOldTemp:
return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum)
return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
case FileTypeTemp:
return fmt.Sprintf("temporary.%s.dbtmp", fileNum)
return fmt.Sprintf("temporary.%s.dbtmp", dfn)
}
panic("unreachable")
}
// MakeFilepath builds a filepath from components.
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, fileNum))
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
}
// ParseFilename parses the components from a filename.
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) {
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
filename = fs.PathBase(filename)
switch {
case filename == "CURRENT":
return FileTypeCurrent, 0, true
return FileTypeCurrent, DiskFileNum{0}, true
case filename == "LOCK":
return FileTypeLock, 0, true
return FileTypeLock, DiskFileNum{0}, true
case strings.HasPrefix(filename, "MANIFEST-"):
fileNum, ok = parseFileNum(filename[len("MANIFEST-"):])
dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
if !ok {
break
}
return FileTypeManifest, fileNum, true
return FileTypeManifest, dfn, true
case strings.HasPrefix(filename, "OPTIONS-"):
fileNum, ok = parseFileNum(filename[len("OPTIONS-"):])
dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
if !ok {
break
}
return FileTypeOptions, fileNum, ok
return FileTypeOptions, dfn, ok
case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeOldTemp, fileNum, ok
return FileTypeOldTemp, dfn, ok
case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeTemp, fileNum, ok
return FileTypeTemp, dfn, ok
default:
i := strings.IndexByte(filename, '.')
if i < 0 {
break
}
fileNum, ok = parseFileNum(filename[:i])
dfn, ok = parseDiskFileNum(filename[:i])
if !ok {
break
}
switch filename[i+1:] {
case "sst":
return FileTypeTable, fileNum, true
return FileTypeTable, dfn, true
case "log":
return FileTypeLog, fileNum, true
return FileTypeLog, dfn, true
}
}
return 0, fileNum, false
return 0, dfn, false
}
func parseFileNum(s string) (fileNum FileNum, ok bool) {
func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
u, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return fileNum, false
return dfn, false
}
return FileNum(u), true
return DiskFileNum{FileNum(u)}, true
}
// A Fataler fatals a process with a message when called.

71
vendor/github.com/cockroachdb/pebble/internal/base/internal.go generated vendored
View File

@@ -9,6 +9,17 @@ import (
"fmt"
"strconv"
"strings"
"github.com/cockroachdb/redact"
)
const (
// SeqNumZero is the zero sequence number, set by compactions if they can
// guarantee there are no keys underneath an internal key.
SeqNumZero = uint64(0)
// SeqNumStart is the first sequence number assigned to a key. Sequence
// numbers 1-9 are reserved for potential future use.
SeqNumStart = uint64(10)
)
// InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
@@ -24,6 +35,14 @@ const (
//InternalKeyKindColumnFamilyDeletion InternalKeyKind = 4
//InternalKeyKindColumnFamilyValue InternalKeyKind = 5
//InternalKeyKindColumnFamilyMerge InternalKeyKind = 6
// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
// solely for compactions (to reduce write amp and space amp). Readers other
// than compactions should treat SINGLEDEL as equivalent to a DEL.
// Historically, it was simpler for readers other than compactions to treat
// SINGLEDEL as equivalent to DEL, but as of the introduction of
// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
// correctness.
InternalKeyKindSingleDelete InternalKeyKind = 7
//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
//InternalKeyKindBeginPrepareXID InternalKeyKind = 9
@@ -60,14 +79,21 @@ const (
InternalKeyKindRangeKeyUnset InternalKeyKind = 20
InternalKeyKindRangeKeySet InternalKeyKind = 21
// InternalKeyIngestSST is used to distinguish a batch that corresponds to
// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
// the WAL entry for ingested sstables that are added to the flushable
// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
// batch, or in an sstable.
InternalKeyIngestSST InternalKeyKind = 22
InternalKeyKindIngestSST InternalKeyKind = 22
// This maximum value isn't part of the file format. It's unlikely,
// but future extensions may increase this value.
// InternalKeyKindDeleteSized keys behave identically to
// InternalKeyKindDelete keys, except that they hold an associated uint64
// value indicating the (len(key)+len(value)) of the shadowed entry the
// tombstone is expected to delete. This value is used to inform compaction
// heuristics, but is not required to be accurate for correctness.
InternalKeyKindDeleteSized InternalKeyKind = 23
// This maximum value isn't part of the file format. Future extensions may
// increase this value.
//
// When constructing an internal key to pass to DB.Seek{GE,LE},
// internalKeyComparer sorts decreasing by kind (after sorting increasing by
@@ -75,14 +101,19 @@ const (
// which sorts 'less than or equal to' any other valid internalKeyKind, when
// searching for any kind of internal key formed by a certain user key and
// seqNum.
InternalKeyKindMax InternalKeyKind = 22
InternalKeyKindMax InternalKeyKind = 23
// Internal to the sstable format. Not exposed by any sstable iterator.
// Declared here to prevent definition of valid key kinds that set this bit.
InternalKeyKindSSTableInternalObsoleteBit InternalKeyKind = 64
InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
// zero sequence number.
InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid)
InternalKeyZeroSeqnumMaxTrailer = uint64(255)
// A marker for an invalid key.
InternalKeyKindInvalid InternalKeyKind = 255
InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
// which prevents those entries from being excluded from iteration.
@@ -105,6 +136,9 @@ const (
InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
)
// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
var internalKeyKindNames = []string{
InternalKeyKindDelete: "DEL",
InternalKeyKindSet: "SET",
@@ -117,7 +151,8 @@ var internalKeyKindNames = []string{
InternalKeyKindRangeKeySet: "RANGEKEYSET",
InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET",
InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
InternalKeyIngestSST: "INGESTSST",
InternalKeyKindIngestSST: "INGESTSST",
InternalKeyKindDeleteSized: "DELSIZED",
InternalKeyKindInvalid: "INVALID",
}
@@ -128,6 +163,11 @@ func (k InternalKeyKind) String() string {
return fmt.Sprintf("UNKNOWN:%d", k)
}
// SafeFormat implements redact.SafeFormatter.
func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
w.Print(redact.SafeString(k.String()))
}
// InternalKey is a key used for the in-memory and on-disk partial DBs that
// make up a pebble DB.
//
@@ -194,6 +234,7 @@ var kindsMap = map[string]InternalKeyKind{
"DEL": InternalKeyKindDelete,
"SINGLEDEL": InternalKeyKindSingleDelete,
"RANGEDEL": InternalKeyKindRangeDelete,
"LOGDATA": InternalKeyKindLogData,
"SET": InternalKeyKindSet,
"MERGE": InternalKeyKindMerge,
"INVALID": InternalKeyKindInvalid,
@@ -202,6 +243,8 @@ var kindsMap = map[string]InternalKeyKind{
"RANGEKEYSET": InternalKeyKindRangeKeySet,
"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
"RANGEKEYDEL": InternalKeyKindRangeKeyDelete,
"INGESTSST": InternalKeyKindIngestSST,
"DELSIZED": InternalKeyKindDeleteSized,
}
// ParseInternalKey parses the string representation of an internal key. The
@@ -338,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
return k.Trailer >> 8
}
// SeqNumFromTrailer returns the sequence number component of a trailer.
func SeqNumFromTrailer(t uint64) uint64 {
return t >> 8
}
// Visible returns true if the key is visible at the specified snapshot
// sequence number.
func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
@@ -395,6 +443,13 @@ func (k InternalKey) Clone() InternalKey {
}
}
// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
// reusing any space already used for the current UserKey.
func (k *InternalKey) CopyFrom(k2 InternalKey) {
k.UserKey = append(k.UserKey[:0], k2.UserKey...)
k.Trailer = k2.Trailer
}
// String returns a string representation of the key.
func (k InternalKey) String() string {
return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())

9
vendor/github.com/cockroachdb/pebble/internal/base/metrics.go generated vendored
View File

@@ -53,6 +53,15 @@ func (tm *ThroughputMetric) Rate() int64 {
float64(time.Second))
}
// Utilization returns a percent [0, 1.0] indicating the percent of time
// work was performed.
func (tm *ThroughputMetric) Utilization() float64 {
if tm.WorkDuration == 0 {
return 0
}
return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
}
// GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
// accumulating samples of that gauge.
type GaugeSampleMetric struct {

15
vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go generated vendored
View File

@@ -63,14 +63,15 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/constants"
"golang.org/x/exp/rand"
)
const (
maxHeight = 20
maxNodeSize = int(unsafe.Sizeof(node{}))
linksSize = int(unsafe.Sizeof(links{}))
maxNodesSize = math.MaxUint32
maxNodeSize = uint64(unsafe.Sizeof(node{}))
linksSize = uint64(unsafe.Sizeof(links{}))
maxNodesSize = constants.MaxUint32OrInt
)
var (
@@ -281,7 +282,7 @@ func (s *Skiplist) newNode(
panic("height cannot be less than one or greater than the max height")
}
unusedSize := (maxHeight - int(height)) * linksSize
unusedSize := uint64(maxHeight-int(height)) * linksSize
nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
if err != nil {
return 0, err
@@ -296,13 +297,13 @@ func (s *Skiplist) newNode(
}
func (s *Skiplist) alloc(size uint32) (uint32, error) {
offset := len(s.nodes)
offset := uint64(len(s.nodes))
// We only have a need for memory up to offset + size, but we never want
// to allocate a node whose tail points into unallocated memory.
minAllocSize := offset + maxNodeSize
if cap(s.nodes) < minAllocSize {
allocSize := cap(s.nodes) * 2
if uint64(cap(s.nodes)) < minAllocSize {
allocSize := uint64(cap(s.nodes)) * 2
if allocSize < minAllocSize {
allocSize = minAllocSize
}

208
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go generated vendored
View File

@@ -33,7 +33,7 @@ import (
type fileKey struct {
// id is the namespace for fileNums.
id uint64
fileNum base.FileNum
fileNum base.DiskFileNum
}
type key struct {
@@ -71,14 +71,12 @@ func (h Handle) Get() []byte {
// Release releases the reference to the cache entry.
func (h Handle) Release() {
if h.value != nil {
h.value.release()
}
h.value.release()
}
type shard struct {
hits int64
misses int64
hits atomic.Int64
misses atomic.Int64
mu sync.RWMutex
@@ -113,25 +111,25 @@ type shard struct {
countTest int64
}
func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
c.mu.RLock()
var value *Value
if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
value = e.acquireValue()
if value != nil {
atomic.StoreInt32(&e.referenced, 1)
e.referenced.Store(true)
}
}
c.mu.RUnlock()
if value == nil {
atomic.AddInt64(&c.misses, 1)
c.misses.Add(1)
return Handle{}
}
atomic.AddInt64(&c.hits, 1)
c.hits.Add(1)
return Handle{value: value}
}
func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
if n := value.refs(); n != 1 {
panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
}
@@ -160,7 +158,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
case e.peekValue() != nil:
// cache entry was a hot or cold page
e.setValue(value)
atomic.StoreInt32(&e.referenced, 1)
e.referenced.Store(true)
delta := int64(len(value.buf)) - e.size
e.size = int64(len(value.buf))
if e.ptype == etHot {
@@ -176,7 +174,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
// cache entry was a test page
c.sizeTest -= e.size
c.countTest--
c.metaDel(e)
c.metaDel(e).release()
c.metaCheck(e)
e.size = int64(len(value.buf))
@@ -185,7 +183,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
c.coldTarget = c.targetSize()
}
atomic.StoreInt32(&e.referenced, 0)
e.referenced.Store(false)
e.setValue(value)
e.ptype = etHot
if c.metaAdd(k, e) {
@@ -222,7 +220,7 @@ func (c *shard) checkConsistency() {
}
// Delete deletes the cached value for the specified file and offset.
func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
// The common case is there is nothing to delete, so do a quick check with
// shared lock.
k := key{fileKey{id, fileNum}, offset}
@@ -233,37 +231,75 @@ func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
return
}
c.mu.Lock()
defer c.mu.Unlock()
var deletedValue *Value
func() {
c.mu.Lock()
defer c.mu.Unlock()
e := c.blocks.Get(k)
if e == nil {
return
}
c.metaEvict(e)
c.checkConsistency()
e := c.blocks.Get(k)
if e == nil {
return
}
deletedValue = c.metaEvict(e)
c.checkConsistency()
}()
// Now that the mutex has been dropped, release the reference which will
// potentially free the memory associated with the previous cached value.
deletedValue.release()
}
// EvictFile evicts all of the cache values for the specified file.
func (c *shard) EvictFile(id uint64, fileNum base.FileNum) {
c.mu.Lock()
defer c.mu.Unlock()
func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
fkey := key{fileKey{id, fileNum}, 0}
for c.evictFileRun(fkey) {
// Sched switch to give another goroutine an opportunity to acquire the
// shard mutex.
runtime.Gosched()
}
}
func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
// If most of the file's blocks are held in the block cache, evicting all
// the blocks may take a while. We don't want to block the entire cache
// shard, forcing concurrent readers to wait until we're finished. We drop
// the mutex every [blocksPerMutexAcquisition] blocks to give other
// goroutines an opportunity to make progress.
const blocksPerMutexAcquisition = 5
c.mu.Lock()
// Releasing a value may result in free-ing it back to the memory allocator.
// This can have a nontrivial cost that we'd prefer to not pay while holding
// the shard mutex, so we collect the evicted values in a local slice and
// only release them in a defer after dropping the cache mutex.
var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
obsoleteValues := obsoleteValuesAlloc[:0]
defer func() {
c.mu.Unlock()
for _, v := range obsoleteValues {
v.release()
}
}()
blocks := c.files.Get(fkey)
if blocks == nil {
return
}
for b, n := blocks, (*entry)(nil); ; b = n {
n = b.fileLink.next
c.metaEvict(b)
if b == n {
break
}
// No blocks for this file.
return false
}
c.checkConsistency()
// b is the current head of the doubly linked list, and n is the entry after b.
for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
n = b.fileLink.next
obsoleteValues = append(obsoleteValues, c.metaEvict(b))
if b == n {
// b == n represents the case where b was the last entry remaining
// in the doubly linked list, which is why it pointed at itself. So
// no more entries left.
c.checkConsistency()
return false
}
}
// Exhausted blocksPerMutexAcquisition.
return true
}
func (c *shard) Free() {
@@ -274,7 +310,7 @@ func (c *shard) Free() {
// metaCheck call when the "invariants" build tag is specified.
for c.handHot != nil {
e := c.handHot
c.metaDel(c.handHot)
c.metaDel(c.handHot).release()
e.free()
}
@@ -359,12 +395,14 @@ func (c *shard) metaAdd(key key, e *entry) bool {
// Remove the entry from the cache. This removes the entry from the blocks map,
// the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
// entry.
func (c *shard) metaDel(e *entry) {
// entry. Returns the deleted value that must be released, if any.
func (c *shard) metaDel(e *entry) (deletedValue *Value) {
if value := e.peekValue(); value != nil {
value.ref.trace("metaDel")
}
e.setValue(nil)
// Remove the pointer to the value.
deletedValue = e.val
e.val = nil
c.blocks.Delete(e.key)
if entriesGoAllocated {
@@ -396,6 +434,7 @@ func (c *shard) metaDel(e *entry) {
} else {
c.files.Put(fkey, next)
}
return deletedValue
}
// Check that the specified entry is not referenced by the cache.
@@ -455,7 +494,7 @@ func (c *shard) metaCheck(e *entry) {
}
}
func (c *shard) metaEvict(e *entry) {
func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
switch e.ptype {
case etHot:
c.sizeHot -= e.size
@@ -467,9 +506,10 @@ func (c *shard) metaEvict(e *entry) {
c.sizeTest -= e.size
c.countTest--
}
c.metaDel(e)
evictedValue = c.metaDel(e)
c.metaCheck(e)
e.free()
return evictedValue
}
func (c *shard) evict() {
@@ -491,8 +531,8 @@ func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
e := c.handCold
if e.ptype == etCold {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
if e.referenced.Load() {
e.referenced.Store(false)
e.ptype = etHot
c.sizeCold -= e.size
c.countCold--
@@ -528,8 +568,8 @@ func (c *shard) runHandHot() {
e := c.handHot
if e.ptype == etHot {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
if e.referenced.Load() {
e.referenced.Store(false)
} else {
e.ptype = etCold
c.sizeHot -= e.size
@@ -564,7 +604,7 @@ func (c *shard) runHandTest() {
if c.coldTarget < 0 {
c.coldTarget = 0
}
c.metaDel(e)
c.metaDel(e).release()
c.metaCheck(e)
e.free()
}
@@ -587,7 +627,7 @@ type Metrics struct {
// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
// used for page replacement
// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
// order to provide better concurrency, 2 x NumCPUs shards are created, with
// order to provide better concurrency, 4 x NumCPUs shards are created, with
// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
// is run independently on each shard.
//
@@ -621,9 +661,9 @@ type Metrics struct {
// used in combination by specifying `-tags invariants,tracing`. Note that
// "tracing" produces a significant slowdown, while "invariants" does not.
type Cache struct {
refs int64
refs atomic.Int64
maxSize int64
idAlloc uint64
idAlloc atomic.Uint64
shards []shard
// Traces recorded by Cache.trace. Used for debugging.
@@ -643,17 +683,44 @@ type Cache struct {
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func New(size int64) *Cache {
return newShards(size, 2*runtime.GOMAXPROCS(0))
// How many cache shards should we create?
//
// Note that the probability two processors will try to access the same
// shard at the same time increases superlinearly with the number of
// processors (Eg, consider the brithday problem where each CPU is a person,
// and each shard is a possible birthday).
//
// We could consider growing the number of shards superlinearly, but
// increasing the shard count may reduce the effectiveness of the caching
// algorithm if frequently-accessed blocks are insufficiently distributed
// across shards. If a shard's size is smaller than a single frequently
// scanned sstable, then the shard will be unable to hold the entire
// frequently-scanned table in memory despite other shards still holding
// infrequently accessed blocks.
//
// Experimentally, we've observed contention contributing to tail latencies
// at 2 shards per processor. For now we use 4 shards per processor,
// recognizing this may not be final word.
m := 4 * runtime.GOMAXPROCS(0)
// In tests we can use large CPU machines with small cache sizes and have
// many caches in existence at a time. If sharding into m shards would
// produce too small shards, constrain the number of shards to 4.
const minimumShardSize = 4 << 20 // 4 MiB
if m > 4 && int(size)/m < minimumShardSize {
m = 4
}
return newShards(size, m)
}
func newShards(size int64, shards int) *Cache {
c := &Cache{
refs: 1,
maxSize: size,
idAlloc: 1,
shards: make([]shard, shards),
}
c.trace("alloc", c.refs)
c.refs.Store(1)
c.idAlloc.Store(1)
c.trace("alloc", c.refs.Load())
for i := range c.shards {
c.shards[i] = shard{
maxSize: size / int64(len(c.shards)),
@@ -669,7 +736,7 @@ func newShards(size int64, shards int) *Cache {
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(c, func(obj interface{}) {
c := obj.(*Cache)
if v := atomic.LoadInt64(&c.refs); v != 0 {
if v := c.refs.Load(); v != 0 {
c.tr.Lock()
fmt.Fprintf(os.Stderr,
"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
@@ -683,7 +750,7 @@ func newShards(size int64, shards int) *Cache {
return c
}
func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard {
func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
@@ -698,10 +765,11 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
h ^= uint64(id & 0xff)
id >>= 8
}
fileNumVal := uint64(fileNum.FileNum())
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(fileNum & 0xff)
fileNum >>= 8
h ^= uint64(fileNumVal) & 0xff
fileNumVal >>= 8
}
for i := 0; i < 8; i++ {
h *= prime64
@@ -715,7 +783,7 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
// Ref adds a reference to the cache. The cache only remains valid as long a
// reference is maintained to it.
func (c *Cache) Ref() {
v := atomic.AddInt64(&c.refs, 1)
v := c.refs.Add(1)
if v <= 1 {
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
@@ -724,7 +792,7 @@ func (c *Cache) Ref() {
// Unref releases a reference on the cache.
func (c *Cache) Unref() {
v := atomic.AddInt64(&c.refs, -1)
v := c.refs.Add(-1)
c.trace("unref", v)
switch {
case v < 0:
@@ -738,7 +806,7 @@ func (c *Cache) Unref() {
// Get retrieves the cache value for the specified file and offset, returning
// nil if no value is present.
func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
}
@@ -746,17 +814,17 @@ func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
// existing value if present. A Handle is returned which provides faster
// retrieval of the cached value than Get (lock-free and avoidance of the map
// lookup). The value must have been allocated by Cache.Alloc.
func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
}
// Delete deletes the cached value for the specified file and offset.
func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) {
func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
}
// EvictFile evicts all of the cache values for the specified file.
func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) {
func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
@@ -784,14 +852,14 @@ func (c *Cache) Size() int64 {
// manually managed. The caller MUST either add the value to the cache (via
// Cache.Set), or release the value (via Cache.Free). Failure to do so will
// result in a memory leak.
func (c *Cache) Alloc(n int) *Value {
func Alloc(n int) *Value {
return newValue(n)
}
// Free frees the specified value. The buffer associated with the value will
// possibly be reused, making it invalid to use the buffer after calling
// Free. Do not call Free on a value that has been added to the cache.
func (c *Cache) Free(v *Value) {
func Free(v *Value) {
if n := v.refs(); n > 1 {
panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
}
@@ -828,8 +896,8 @@ func (c *Cache) Metrics() Metrics {
m.Count += int64(s.blocks.Count())
m.Size += s.sizeHot + s.sizeCold
s.mu.RUnlock()
m.Hits += atomic.LoadInt64(&s.hits)
m.Misses += atomic.LoadInt64(&s.misses)
m.Hits += s.hits.Load()
m.Misses += s.misses.Load()
}
return m
}
@@ -837,5 +905,5 @@ func (c *Cache) Metrics() Metrics {
// NewID returns a new ID to be used as a namespace for cached file
// blocks.
func (c *Cache) NewID() uint64 {
return atomic.AddUint64(&c.idAlloc, 1)
return c.idAlloc.Add(1)
}

8
vendor/github.com/cockroachdb/pebble/internal/cache/entry.go generated vendored
View File

@@ -4,6 +4,8 @@
package cache
import "sync/atomic"
type entryType int8
const (
@@ -56,7 +58,7 @@ type entry struct {
ptype entryType
// referenced is atomically set to indicate that this entry has been accessed
// since the last time one of the clock hands swept it.
referenced int32
referenced atomic.Bool
shard *shard
// Reference count for the entry. The entry is freed when the reference count
// drops to zero.
@@ -137,9 +139,7 @@ func (e *entry) setValue(v *Value) {
}
old := e.val
e.val = v
if old != nil {
old.release()
}
old.release()
}
func (e *entry) peekValue() *Value {

2
vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go generated vendored
View File

@@ -24,7 +24,7 @@ func robinHoodHash(k key, shift uint32) uint32 {
const m = 11400714819323198485
h := hashSeed
h ^= k.id * m
h ^= uint64(k.fileNum) * m
h ^= uint64(k.fileNum.FileNum()) * m
h ^= k.offset * m
return uint32(h >> shift)
}

2
vendor/github.com/cockroachdb/pebble/internal/cache/value.go generated vendored
View File

@@ -40,7 +40,7 @@ func (v *Value) acquire() {
}
func (v *Value) release() {
if v.ref.release() {
if v != nil && v.ref.release() {
v.free()
}
}

17
vendor/github.com/cockroachdb/pebble/internal/constants/constants.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package constants
const (
// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
oneIf64Bit = ^uint(0) >> 63
// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
// - MaxUint32 on 64-bit platforms;
// - MaxInt on 32-bit platforms.
// It is used when slices are limited to Uint32 on 64-bit platforms (the
// length limit for slices is naturally MaxInt on 32-bit platforms).
MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
)

20
vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go generated vendored
View File

@@ -35,11 +35,11 @@ type config struct {
suffix []string
}
// IEC produces human readable representations of integer values in IEC units.
var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}}
// Bytes produces human readable representations of byte values in IEC units.
var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
// SI produces human readable representations of integer values in SI units.
var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}}
// Count produces human readable representations of unitless values in SI units.
var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
// Int64 produces a human readable representation of the value.
func (c *config) Int64(s int64) FormattedString {
@@ -54,18 +54,6 @@ func (c *config) Uint64(s uint64) FormattedString {
return FormattedString(humanate(s, c.base, c.suffix))
}
// Int64 produces a human readable representation of the value in IEC units
// (base 1024).
func Int64(s int64) FormattedString {
return IEC.Int64(s)
}
// Uint64 produces a human readable representation of the value in IEC units
// (base 1024).
func Uint64(s uint64) FormattedString {
return IEC.Uint64(s)
}
// FormattedString represents a human readable representation of a value. It
// implements the redact.SafeValue interface to signal that it represents a
// a string that does not need to be redacted.

162
vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go generated vendored Normal file
View File

@@ -0,0 +1,162 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package invalidating
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/fastrand"
"github.com/cockroachdb/pebble/internal/invariants"
)
// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
// MaybeWrapIfInvariants does nothing in non-invariant builds.
func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
if invariants.Enabled {
if fastrand.Uint32n(10) == 1 {
return NewIter(iter)
}
}
return iter
}
// iter tests unsafe key/value slice reuse by modifying the last
// returned key/value to all 1s.
type iter struct {
iter base.InternalIterator
lastKey *base.InternalKey
lastValue base.LazyValue
ignoreKinds [base.InternalKeyKindMax + 1]bool
err error
}
// Option configures the behavior of an invalidating iterator.
type Option interface {
apply(*iter)
}
type funcOpt func(*iter)
func (f funcOpt) apply(i *iter) { f(i) }
// IgnoreKinds constructs an Option that configures an invalidating iterator to
// skip trashing k/v pairs with the provided key kinds. Some iterators provided
// key stability guarantees for specific key kinds.
func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
return funcOpt(func(i *iter) {
for _, kind := range kinds {
i.ignoreKinds[kind] = true
}
})
}
// NewIter constructs a new invalidating iterator that wraps the provided
// iterator, trashing buffers for previously returned keys.
func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
i := &iter{iter: originalIterator}
for _, opt := range opts {
opt.apply(i)
}
return i
}
func (i *iter) update(
key *base.InternalKey, value base.LazyValue,
) (*base.InternalKey, base.LazyValue) {
i.trashLastKV()
if key == nil {
i.lastKey = nil
i.lastValue = base.LazyValue{}
return nil, base.LazyValue{}
}
i.lastKey = &base.InternalKey{}
*i.lastKey = key.Clone()
i.lastValue = base.LazyValue{
ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
}
if value.Fetcher != nil {
fetcher := new(base.LazyFetcher)
*fetcher = *value.Fetcher
i.lastValue.Fetcher = fetcher
}
return i.lastKey, i.lastValue
}
func (i *iter) trashLastKV() {
if i.lastKey == nil {
return
}
if i.ignoreKinds[i.lastKey.Kind()] {
return
}
if i.lastKey != nil {
for j := range i.lastKey.UserKey {
i.lastKey.UserKey[j] = 0xff
}
i.lastKey.Trailer = 0xffffffffffffffff
}
for j := range i.lastValue.ValueOrHandle {
i.lastValue.ValueOrHandle[j] = 0xff
}
if i.lastValue.Fetcher != nil {
// Not all the LazyFetcher fields are visible, so we zero out the last
// value's Fetcher struct entirely.
*i.lastValue.Fetcher = base.LazyFetcher{}
}
}
func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekGE(key, flags))
}
func (i *iter) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
}
func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.SeekLT(key, flags))
}
func (i *iter) First() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.First())
}
func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Last())
}
func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Next())
}
func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.Prev())
}
func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
return i.update(i.iter.NextPrefix(succKey))
}
func (i *iter) Error() error {
if err := i.iter.Error(); err != nil {
return err
}
return i.err
}
func (i *iter) Close() error {
return i.iter.Close()
}
func (i *iter) SetBounds(lower, upper []byte) {
i.iter.SetBounds(lower, upper)
}
func (i *iter) String() string {
return i.iter.String()
}

38
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored
View File

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
// fragments before this span. Defragment backward to find the start of the
// defragmented span.
i.defragmentBackward()
// Defragmenting backward may have stopped because it encountered an error.
// If so, we must not continue so that i.iter.Error() (and thus i.Error())
// yields the error.
if i.iterSpan == nil && i.iter.Error() != nil {
return nil
}
if i.iterPos == iterPosPrev {
// Next once back onto the span.
i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
// fragments after this span. Defragment forward to find the end of the
// defragmented span.
i.defragmentForward()
// Defragmenting forward may have stopped because it encountered an error.
// If so, we must not continue so that i.iter.Error() (and thus i.Error())
// yields the error.
if i.iterSpan == nil && i.iter.Error() != nil {
return nil
}
if i.iterPos == iterPosNext {
// Prev once back onto the span.
i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
// Next once to move onto y, defragment forward to land on the first z
// position.
i.iterSpan = i.iter.Next()
if invariants.Enabled && i.iterSpan == nil {
if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
panic("pebble: invariant violation: no next span while switching directions")
}
// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
// Prev once to move onto y, defragment backward to land on the last x
// position.
i.iterSpan = i.iter.Prev()
if invariants.Enabled && i.iterSpan == nil {
if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
panic("pebble: invariant violation: no previous span while switching directions")
}
// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Next()
}
// i.iterSpan == nil
//
// The inner iterator may return nil when it encounters an error. If there
// was an error, we don't know whether there is another span we should
// defragment or not. Return nil so that the caller knows they should check
// Error().
if i.iter.Error() != nil {
return nil
}
i.curr.Keys = i.keysBuf
return &i.curr
}
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Prev()
}
// i.iterSpan == nil
//
// The inner iterator may return nil when it encounters an error. If there
// was an error, we don't know whether there is another span we should
// defragment or not. Return nil so that the caller knows they should check
// Error().
if i.iter.Error() != nil {
return nil
}
i.curr.Keys = i.keysBuf
return &i.curr
}

25
vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go generated vendored
View File

@@ -4,6 +4,8 @@
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// FilterFunc defines a transform from the input Span into the output Span. The
// function returns true if the Span should be returned by the iterator, and
// false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
type filteringIter struct {
iter FragmentIterator
filterFn FilterFunc
cmp base.Compare
// span is a mutable Span passed to the filterFn. The filterFn is free to
// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
// Filter returns a new filteringIter that will filter the Spans from the
// provided child iterator using the provided FilterFunc.
func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter}
func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
}
// SeekGE implements FragmentIterator.
func (i *filteringIter) SeekGE(key []byte) *Span {
return i.filter(i.iter.SeekGE(key), +1)
span := i.filter(i.iter.SeekGE(key), +1)
// i.filter could return a span that's less than key, _if_ the filterFunc
// (which has no knowledge of the seek key) mutated the span to end at a key
// less than or equal to `key`. Detect this case and next/invalidate the iter.
if span != nil && i.cmp(span.End, key) <= 0 {
return i.Next()
}
return span
}
// SeekLT implements FragmentIterator.
func (i *filteringIter) SeekLT(key []byte) *Span {
return i.filter(i.iter.SeekLT(key), -1)
span := i.filter(i.iter.SeekLT(key), -1)
// i.filter could return a span that's >= key, _if_ the filterFunc (which has
// no knowledge of the seek key) mutated the span to start at a key greater
// than or equal to `key`. Detect this case and prev/invalidate the iter.
if span != nil && i.cmp(span.Start, key) >= 0 {
return i.Prev()
}
return span
}
// First implements FragmentIterator.

925
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored
View File

File diff suppressed because it is too large Load Diff

4
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored
View File

@@ -51,6 +51,8 @@ type FragmentIterator interface {
Prev() *Span
// Error returns any accumulated error.
//
// TODO(jackson): Lift errors into return values on the positioning methods.
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
@@ -62,7 +64,7 @@ type FragmentIterator interface {
// TableNewSpanIter creates a new iterator for range key spans for the given
// file.
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
// per-sstable span iterators.

4
vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go generated vendored
View File

@@ -98,7 +98,7 @@ func (l *LevelIter) Init(
) {
l.err = nil
l.level = level
l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
l.tableOpts = opts
l.cmp = cmp
l.iterFile = nil
l.newIter = newIter
@@ -179,7 +179,7 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur
return noFileLoaded
}
if indicator != fileAlreadyLoaded {
l.iter, l.err = l.newIter(file, &l.tableOpts)
l.iter, l.err = l.newIter(file, l.tableOpts)
indicator = newFileLoaded
}
if l.err != nil {

43
vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go generated vendored
View File

@@ -25,49 +25,6 @@ import (
// MergingIter implementation, but will require a bit of plumbing to thread the
// Equal function.
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
// provided output *Span pointer. The output Span's Keys slice may be reused
// by Transform to reduce allocations.
Transform(cmp base.Compare, in Span, out *Span) error
}
// The TransformerFunc type is an adapter to allow the use of ordinary functions
// as Transformers. If f is a function with the appropriate signature,
// TransformerFunc(f) is a Transformer that calls f.
type TransformerFunc func(base.Compare, Span, *Span) error
// Transform calls f(cmp, in, out).
func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
return tf(cmp, in, out)
}
var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
})
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
for _, k := range s.Keys {
// NB: The InternalKeySeqNumMax value is used for the batch snapshot
// because a batch's visible span keys are filtered when they're
// fragmented. There's no requirement to enforce visibility at
// iteration time.
if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
dst.Keys = append(dst.Keys, k)
}
}
return nil
})
}
// MergingIter merges spans across levels of the LSM, exposing an iterator over
// spans that yields sets of spans fragmented at unique user key boundaries.
//

50
vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go generated vendored Normal file
View File

@@ -0,0 +1,50 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
// provided output *Span pointer. The output Span's Keys slice may be reused
// by Transform to reduce allocations.
Transform(cmp base.Compare, in Span, out *Span) error
}
// The TransformerFunc type is an adapter to allow the use of ordinary functions
// as Transformers. If f is a function with the appropriate signature,
// TransformerFunc(f) is a Transformer that calls f.
type TransformerFunc func(base.Compare, Span, *Span) error
// Transform calls f(cmp, in, out).
func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
return tf(cmp, in, out)
}
var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
})
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
for _, k := range s.Keys {
// NB: The InternalKeySeqNumMax value is used for the batch snapshot
// because a batch's visible span keys are filtered when they're
// fragmented. There's no requirement to enforce visibility at
// iteration time.
if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
dst.Keys = append(dst.Keys, k)
}
}
return nil
})
}

19
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored
View File

@@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base"
// truncated to be contained within the range [lower, upper). If start and end
// are specified, filter out any spans that are completely outside those bounds.
func Truncate(
cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
cmp base.Compare,
iter FragmentIterator,
lower, upper []byte,
start, end *base.InternalKey,
panicOnUpperTruncate bool,
) FragmentIterator {
return Filter(iter, func(in *Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
// Ignore this span if it lies completely outside [start, end].
// Ignore this span if it lies completely outside start, end. Note that
// end endInclusive indicated whether end is inclusive.
//
// The comparison between s.End and start is by user key only, as
// the span is exclusive at s.End, so comparing by user keys
@@ -48,13 +53,21 @@ func Truncate(
// Wholly within the end bound. Keep it.
}
}
var truncated bool
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
out.Start = lower
}
if cmp(in.End, upper) > 0 {
truncated = true
out.End = upper
}
if panicOnUpperTruncate && truncated {
panic("pebble: upper bound should not be truncated")
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}, cmp)
}

34
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored
View File

@@ -108,7 +108,7 @@ type annotation struct {
}
type leafNode struct {
ref int32
ref atomic.Int32
count int16
leaf bool
// subtreeCount holds the count of files in the entire subtree formed by
@@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node {
func newLeafNode() *node {
n := leafToNode(new(leafNode))
n.leaf = true
n.ref = 1
n.ref.Store(1)
return n
}
func newNode() *node {
n := new(node)
n.ref = 1
n.ref.Store(1)
return n
}
@@ -159,7 +159,7 @@ func newNode() *node {
// When a node is cloned, the provided pointer will be redirected to the new
// mutable node.
func mut(n **node) *node {
if atomic.LoadInt32(&(*n).ref) == 1 {
if (*n).ref.Load() == 1 {
// Exclusive ownership. Can mutate in place.
// Whenever a node will be mutated, reset its annotations to be marked
@@ -186,7 +186,7 @@ func mut(n **node) *node {
// incRef acquires a reference to the node.
func (n *node) incRef() {
atomic.AddInt32(&n.ref, 1)
n.ref.Add(1)
}
// decRef releases a reference to the node. If requested, the method will unref
@@ -195,8 +195,8 @@ func (n *node) incRef() {
// new nodes pass contentsToo=false to preserve existing reference counts during
// operations that should yield a net-zero change to descendant refcounts.
// When a node is released, its contained files are dereferenced.
func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if atomic.AddInt32(&n.ref, -1) > 0 {
func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
if n.ref.Add(-1) > 0 {
// Other references remain. Can't free.
return
}
@@ -207,7 +207,7 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
// nodes, and they want to preserve the existing reference count.
if contentsToo {
for _, f := range n.items[:n.count] {
if atomic.AddInt32(&f.Refs, -1) == 0 {
if f.Unref() == 0 {
// There are two sources of node dereferences: tree mutations
// and Version dereferences. Files should only be made obsolete
// during Version dereferences, during which `obsolete` will be
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
if obsolete == nil {
panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
}
*obsolete = append(*obsolete, f)
// Reference counting is performed on the FileBacking. In the case
// of a virtual sstable, this reference counting is performed on
// a FileBacking which is shared by every single virtual sstable
// with the same backing sstable. If the reference count hits 0,
// then we know that the FileBacking won't be required by any
// sstable in Pebble, and that the backing sstable can be deleted.
*obsolete = append(*obsolete, f.FileBacking)
}
}
if !n.leaf {
@@ -241,7 +247,7 @@ func (n *node) clone() *node {
c.subtreeCount = n.subtreeCount
// Increase the refcount of each contained item.
for _, f := range n.items[:n.count] {
atomic.AddInt32(&f.Refs, 1)
f.Ref()
}
if !c.leaf {
// Copy children and increase each refcount.
@@ -761,8 +767,8 @@ type btree struct {
// Release dereferences and clears the root node of the btree, removing all
// items from the btree. In doing so, it decrements contained file counts.
// It returns a slice of newly obsolete files, if any.
func (t *btree) Release() (obsolete []*FileMetadata) {
// It returns a slice of newly obsolete backing files, if any.
func (t *btree) Release() (obsolete []*FileBacking) {
if t.root != nil {
t.root.decRef(true /* contentsToo */, &obsolete)
t.root = nil
@@ -800,7 +806,7 @@ func (t *btree) Delete(item *FileMetadata) (obsolete bool) {
return false
}
if out := mut(&t.root).Remove(t.cmp, item); out != nil {
obsolete = atomic.AddInt32(&out.Refs, -1) == 0
obsolete = out.Unref() == 0
}
if invariants.Enabled {
t.root.verifyInvariants()
@@ -832,7 +838,7 @@ func (t *btree) Insert(item *FileMetadata) error {
newRoot.subtreeCount = t.root.subtreeCount + splitNode.subtreeCount + 1
t.root = newRoot
}
atomic.AddInt32(&item.Refs, 1)
item.Ref()
err := mut(&t.root).Insert(t.cmp, item)
if invariants.Enabled {
t.root.verifyInvariants()

77
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored
View File

@@ -15,20 +15,28 @@ import (
// LevelMetadata contains metadata for all of the files within
// a level of the LSM.
type LevelMetadata struct {
level int
tree btree
level int
totalSize uint64
// NumVirtual is the number of virtual sstables in the level.
NumVirtual uint64
// VirtualSize is the size of the virtual sstables in the level.
VirtualSize uint64
tree btree
}
// clone makes a copy of the level metadata, implicitly increasing the ref
// count of every file contained within lm.
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
tree: lm.tree.Clone(),
level: lm.level,
totalSize: lm.totalSize,
NumVirtual: lm.NumVirtual,
VirtualSize: lm.VirtualSize,
tree: lm.tree.Clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
return lm.tree.Release()
}
@@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
var lm LevelMetadata
lm.level = level
lm.tree, _ = makeBTree(bcmp, files)
for _, f := range files {
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
}
return lm
}
@@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
return t, newLevelSlice(t.Iter())
}
func (lm *LevelMetadata) insert(f *FileMetadata) error {
if err := lm.tree.Insert(f); err != nil {
return err
}
lm.totalSize += f.Size
if f.Virtual {
lm.NumVirtual++
lm.VirtualSize += f.Size
}
return nil
}
func (lm *LevelMetadata) remove(f *FileMetadata) bool {
lm.totalSize -= f.Size
if f.Virtual {
lm.NumVirtual--
lm.VirtualSize -= f.Size
}
return lm.tree.Delete(f)
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.Count() == 0
@@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int {
return lm.tree.Count()
}
// Size returns the cumulative size of all the files within the level.
func (lm *LevelMetadata) Size() uint64 {
return lm.totalSize
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.Iter()}
@@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
return sum
}
// NumVirtual returns the number of virtual sstables in the level. Its runtime is
// linear in the length of the slice.
func (ls *LevelSlice) NumVirtual() uint64 {
var n uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
n++
}
}
return n
}
// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
// level.
func (ls *LevelSlice) VirtualSizeSum() uint64 {
var sum uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
sum += f.Size
}
}
return sum
}
// Reslice constructs a new slice backed by the same underlying level, with
// new start and end positions. Reslice invokes the provided function, passing
// two LevelIterators: one positioned to i's inclusive start and one

464
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored
View File

@@ -49,6 +49,9 @@ type TableStats struct {
// The number of point and range deletion entries in the table.
NumDeletions uint64
// NumRangeKeySets is the total number of range key sets in the table.
//
// NB: If there's a chance that the sstable contains any range key sets,
// then NumRangeKeySets must be > 0.
NumRangeKeySets uint64
// Estimate of the total disk space that may be dropped by this table's
// point deletions by compacting them.
@@ -60,9 +63,11 @@ type TableStats struct {
// overlapping data in L0 and ignores L0 sublevels, but the error that
// introduces is expected to be small.
//
// Tables in the bottommost level of the LSM may have a nonzero estimate
// if snapshots or move compactions prevented the elision of their range
// tombstones.
// Tables in the bottommost level of the LSM may have a nonzero estimate if
// snapshots or move compactions prevented the elision of their range
// tombstones. A table in the bottommost level that was ingested into L6
// will have a zero estimate, because the file's sequence numbers indicate
// that the tombstone cannot drop any data contained within the file itself.
RangeDeletionsBytesEstimate uint64
// Total size of value blocks and value index block.
ValueBlocksSize uint64
@@ -119,43 +124,72 @@ func (s CompactionState) String() string {
}
}
// FileMetadata holds the metadata for an on-disk table.
// FileMetadata is maintained for leveled-ssts, i.e., they belong to a level of
// some version. FileMetadata does not contain the actual level of the sst,
// since such leveled-ssts can move across levels in different versions, while
// sharing the same FileMetadata. There are two kinds of leveled-ssts, physical
// and virtual. Underlying both leveled-ssts is a backing-sst, for which the
// only state is FileBacking. A backing-sst is level-less. It is possible for a
// backing-sst to be referred to by a physical sst in one version and by one or
// more virtual ssts in one or more versions. A backing-sst becomes obsolete
// and can be deleted once it is no longer required by any physical or virtual
// sst in any version.
//
// We maintain some invariants:
//
// 1. Each physical and virtual sst will have a unique FileMetadata.FileNum,
// and there will be exactly one FileMetadata associated with the FileNum.
//
// 2. Within a version, a backing-sst is either only referred to by one
// physical sst or one or more virtual ssts.
//
// 3. Once a backing-sst is referred to by a virtual sst in the latest version,
// it cannot go back to being referred to by a physical sst in any future
// version.
//
// Once a physical sst is no longer needed by any version, we will no longer
// maintain the file metadata associated with it. We will still maintain the
// FileBacking associated with the physical sst if the backing sst is required
// by any virtual ssts in any version.
type FileMetadata struct {
// Atomic contains fields which are accessed atomically. Go allocations
// are guaranteed to be 64-bit aligned which we take advantage of by
// placing the 64-bit fields which we access atomically at the beginning
// of the FileMetadata struct. For more information, see
// https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
Atomic struct {
// AllowedSeeks is used to determine if a file should be picked for
// a read triggered compaction. It is decremented when read sampling
// in pebble.Iterator after every after every positioning operation
// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
AllowedSeeks int64
// AllowedSeeks is used to determine if a file should be picked for
// a read triggered compaction. It is decremented when read sampling
// in pebble.Iterator after every after every positioning operation
// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
AllowedSeeks atomic.Int64
// statsValid is 1 if stats have been loaded for the table. The
// TableStats structure is populated only if valid is 1.
statsValid uint32
}
// statsValid indicates if stats have been loaded for the table. The
// TableStats structure is populated only if valid is true.
statsValid atomic.Bool
// FileBacking is the state which backs either a physical or virtual
// sstables.
FileBacking *FileBacking
// InitAllowedSeeks is the inital value of allowed seeks. This is used
// to re-set allowed seeks on a file once it hits 0.
InitAllowedSeeks int64
// Reference count for the file: incremented when a file is added to a
// version and decremented when the version is unreferenced. The file is
// obsolete when the reference count falls to zero.
Refs int32
// FileNum is the file number.
//
// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
FileNum base.FileNum
// Size is the size of the file, in bytes.
// Size is the size of the file, in bytes. Size is an approximate value for
// virtual sstables.
//
// INVARIANTS:
// - When !FileMetadata.Virtual, Size == FileBacking.Size.
// - Size should be non-zero. Size 0 virtual sstables must not be created.
Size uint64
// File creation time in seconds since the epoch (1970-01-01 00:00:00
// UTC). For ingested sstables, this corresponds to the time the file was
// ingested.
// ingested. For virtual sstables, this corresponds to the wall clock time
// when the FileMetadata for the virtual sstable was first created.
CreationTime int64
// Smallest and largest sequence numbers in the table, across both point and
// range keys.
// Lower and upper bounds for the smallest and largest sequence numbers in
// the table, across both point and range keys. For physical sstables, these
// values are tight bounds. For virtual sstables, there is no guarantee that
// there will be keys with SmallestSeqNum or LargestSeqNum within virtual
// sstable bounds.
SmallestSeqNum uint64
LargestSeqNum uint64
// SmallestPointKey and LargestPointKey are the inclusive bounds for the
@@ -178,8 +212,17 @@ type FileMetadata struct {
Smallest InternalKey
Largest InternalKey
// Stats describe table statistics. Protected by DB.mu.
//
// For virtual sstables, set stats upon virtual sstable creation as
// asynchronous computation of stats is not currently supported.
//
// TODO(bananabrick): To support manifest replay for virtual sstables, we
// probably need to compute virtual sstable stats asynchronously. Otherwise,
// we'd have to write virtual sstable stats to the version edit.
Stats TableStats
// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
// pick L0 compactions. Only accurate for the most recent Version.
SubLevel int
L0Index int
minIntervalIndex int
@@ -188,9 +231,6 @@ type FileMetadata struct {
// NB: the alignment of this struct is 8 bytes. We pack all the bools to
// ensure an optimal packing.
// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
// pick L0 compactions. Only accurate for the most recent Version.
//
// IsIntraL0Compacting is set to True if this file is part of an intra-L0
// compaction. When it's true, IsCompacting must also return true. If
// Compacting is true and IsIntraL0Compacting is false for an L0 file, the
@@ -226,6 +266,212 @@ type FileMetadata struct {
// key type (point or range) corresponds to the smallest and largest overall
// table bounds.
boundTypeSmallest, boundTypeLargest boundType
// Virtual is true if the FileMetadata belongs to a virtual sstable.
Virtual bool
}
// PhysicalFileMeta is used by functions which want a guarantee that their input
// belongs to a physical sst and not a virtual sst.
//
// NB: This type should only be constructed by calling
// FileMetadata.PhysicalMeta.
type PhysicalFileMeta struct {
*FileMetadata
}
// VirtualFileMeta is used by functions which want a guarantee that their input
// belongs to a virtual sst and not a physical sst.
//
// A VirtualFileMeta inherits all the same fields as a FileMetadata. These
// fields have additional invariants imposed on them, and/or slightly varying
// meanings:
// - Smallest and Largest (and their counterparts
// {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a
// key at that exact bound. We make the effort to determine the next smallest
// or largest key in an sstable after virtualizing it, to maintain this
// tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it
// could mean that a rangedel or range key ends at that user key, or has been
// truncated to that user key.
// - One invariant is that if a rangedel or range key is truncated on its
// upper bound, the virtual sstable *must* have a rangedel or range key
// sentinel key as its upper bound. This is because truncation yields
// an exclusive upper bound for the rangedel/rangekey, and if there are
// any points at that exclusive upper bound within the same virtual
// sstable, those could get uncovered by this truncation. We enforce this
// invariant in calls to keyspan.Truncate.
// - Size is an estimate of the size of the virtualized portion of this sstable.
// The underlying file's size is stored in FileBacking.Size, though it could
// also be estimated or could correspond to just the referenced portion of
// a file (eg. if the file originated on another node).
// - Size must be > 0.
// - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
// This means that all keys in the virtual sstable must have seqnums within
// [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
// a key with a seqnum at either of the bounds. Calculating tight seqnum
// bounds would be too expensive and deliver little value.
//
// NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
type VirtualFileMeta struct {
*FileMetadata
}
// PhysicalMeta should be the only source of creating the PhysicalFileMeta
// wrapper type.
func (m *FileMetadata) PhysicalMeta() PhysicalFileMeta {
if m.Virtual {
panic("pebble: file metadata does not belong to a physical sstable")
}
return PhysicalFileMeta{
m,
}
}
// VirtualMeta should be the only source of creating the VirtualFileMeta wrapper
// type.
func (m *FileMetadata) VirtualMeta() VirtualFileMeta {
if !m.Virtual {
panic("pebble: file metadata does not belong to a virtual sstable")
}
return VirtualFileMeta{
m,
}
}
// FileBacking either backs a single physical sstable, or one or more virtual
// sstables.
//
// See the comment above the FileMetadata type for sstable terminology.
type FileBacking struct {
// Reference count for the backing file on disk: incremented when a
// physical or virtual sstable which is backed by the FileBacking is
// added to a version and decremented when the version is unreferenced.
// We ref count in order to determine when it is safe to delete a
// backing sst file from disk. The backing file is obsolete when the
// reference count falls to zero.
refs atomic.Int32
// latestVersionRefs are the references to the FileBacking in the
// latest version. This reference can be through a single physical
// sstable in the latest version, or one or more virtual sstables in the
// latest version.
//
// INVARIANT: latestVersionRefs <= refs.
latestVersionRefs atomic.Int32
// VirtualizedSize is set iff the backing sst is only referred to by
// virtual ssts in the latest version. VirtualizedSize is the sum of the
// virtual sstable sizes of all of the virtual sstables in the latest
// version which are backed by the physical sstable. When a virtual
// sstable is removed from the latest version, we will decrement the
// VirtualizedSize. During compaction picking, we'll compensate a
// virtual sstable file size by
// (FileBacking.Size - FileBacking.VirtualizedSize) / latestVersionRefs.
// The intuition is that if FileBacking.Size - FileBacking.VirtualizedSize
// is high, then the space amplification due to virtual sstables is
// high, and we should pick the virtual sstable with a higher priority.
//
// TODO(bananabrick): Compensate the virtual sstable file size using
// the VirtualizedSize during compaction picking and test.
VirtualizedSize atomic.Uint64
DiskFileNum base.DiskFileNum
Size uint64
}
// InitPhysicalBacking allocates and sets the FileBacking which is required by a
// physical sstable FileMetadata.
//
// Ensure that the state required by FileBacking, such as the FileNum, is
// already set on the FileMetadata before InitPhysicalBacking is called.
// Calling InitPhysicalBacking only after the relevant state has been set in the
// FileMetadata is not necessary in tests which don't rely on FileBacking.
func (m *FileMetadata) InitPhysicalBacking() {
if m.Virtual {
panic("pebble: virtual sstables should use a pre-existing FileBacking")
}
if m.FileBacking == nil {
m.FileBacking = &FileBacking{Size: m.Size, DiskFileNum: m.FileNum.DiskFileNum()}
}
}
// InitProviderBacking creates a new FileBacking for a file backed by
// an objstorage.Provider.
func (m *FileMetadata) InitProviderBacking(fileNum base.DiskFileNum) {
if !m.Virtual {
panic("pebble: provider-backed sstables must be virtual")
}
if m.FileBacking == nil {
m.FileBacking = &FileBacking{DiskFileNum: fileNum}
}
}
// ValidateVirtual should be called once the FileMetadata for a virtual sstable
// is created to verify that the fields of the virtual sstable are sound.
func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {
if !m.Virtual {
panic("pebble: invalid virtual sstable")
}
if createdFrom.SmallestSeqNum != m.SmallestSeqNum {
panic("pebble: invalid smallest sequence number for virtual sstable")
}
if createdFrom.LargestSeqNum != m.LargestSeqNum {
panic("pebble: invalid largest sequence number for virtual sstable")
}
if createdFrom.FileBacking != nil && createdFrom.FileBacking != m.FileBacking {
panic("pebble: invalid physical sstable state for virtual sstable")
}
if m.Size == 0 {
panic("pebble: virtual sstable size must be set upon creation")
}
}
// Refs returns the refcount of backing sstable.
func (m *FileMetadata) Refs() int32 {
return m.FileBacking.refs.Load()
}
// Ref increments the ref count associated with the backing sstable.
func (m *FileMetadata) Ref() {
m.FileBacking.refs.Add(1)
}
// Unref decrements the ref count associated with the backing sstable.
func (m *FileMetadata) Unref() int32 {
v := m.FileBacking.refs.Add(-1)
if invariants.Enabled && v < 0 {
panic("pebble: invalid FileMetadata refcounting")
}
return v
}
// LatestRef increments the latest ref count associated with the backing
// sstable.
func (m *FileMetadata) LatestRef() {
m.FileBacking.latestVersionRefs.Add(1)
if m.Virtual {
m.FileBacking.VirtualizedSize.Add(m.Size)
}
}
// LatestUnref decrements the latest ref count associated with the backing
// sstable.
func (m *FileMetadata) LatestUnref() int32 {
if m.Virtual {
m.FileBacking.VirtualizedSize.Add(-m.Size)
}
v := m.FileBacking.latestVersionRefs.Add(-1)
if invariants.Enabled && v < 0 {
panic("pebble: invalid FileMetadata latest refcounting")
}
return v
}
// LatestRefs returns the latest ref count associated with the backing sstable.
func (m *FileMetadata) LatestRefs() int32 {
return m.FileBacking.latestVersionRefs.Load()
}
// SetCompactionState transitions this file's compaction state to the given
@@ -263,22 +509,14 @@ func (m *FileMetadata) IsCompacting() bool {
// returns true, the Stats field may be read (with or without holding the
// database mutex).
func (m *FileMetadata) StatsValid() bool {
return atomic.LoadUint32(&m.Atomic.statsValid) == 1
}
// StatsValidLocked returns true if the table stats have been populated.
// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the
// overhead of an atomic load. This is possible because table stats validity is
// only set while DB.mu is held.
func (m *FileMetadata) StatsValidLocked() bool {
return m.Atomic.statsValid == 1
return m.statsValid.Load()
}
// StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu
// while populating TableStats and calling StatsMarkValud. Once stats are
// populated, they must not be mutated.
func (m *FileMetadata) StatsMarkValid() {
atomic.StoreUint32(&m.Atomic.statsValid, 1)
m.statsValid.Store(true)
}
// ExtendPointKeyBounds attempts to extend the lower and upper point key bounds
@@ -369,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive
return true
}
// ContainedWithinSpan returns true if the file key range completely overlaps with the
// given range ("end" is assumed to exclusive).
func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool {
lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end)
return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel()))
}
// ContainsKeyType returns whether or not the file contains keys of the provided
// type.
func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
@@ -466,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
if !verbose {
return b.String()
}
fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum)
if m.HasPointKeys {
fmt.Fprintf(&b, " points:[%s-%s]",
m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format))
@@ -479,9 +725,9 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
// ParseFileMetadataDebug parses a FileMetadata from its DebugString
// representation.
func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
// Split lines of the form:
// 000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
// 000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...]
fields := strings.FieldsFunc(s, func(c rune) bool {
switch c {
case ':', '[', '-', ']':
@@ -491,10 +737,24 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
}
})
if len(fields)%3 != 0 {
return m, errors.Newf("malformed input: %s", s)
return nil, errors.Newf("malformed input: %s", s)
}
m := &FileMetadata{}
for len(fields) > 0 {
prefix := fields[0]
if prefix == "seqnums" {
smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64)
if err != nil {
return m, errors.Newf("malformed input: %s: %s", s, err)
}
m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum
fields = fields[3:]
continue
}
smallest := base.ParsePrettyInternalKey(fields[1])
largest := base.ParsePrettyInternalKey(fields[2])
switch prefix {
@@ -522,7 +782,8 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
m.HasPointKeys = true
}
return
m.InitPhysicalBacking()
return m, nil
}
// Validate validates the metadata for consistency with itself, returning an
@@ -584,6 +845,11 @@ func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error {
}
}
// Ensure that FileMetadata.Init was called.
if m.FileBacking == nil {
return base.CorruptionErrorf("file metadata FileBacking not set")
}
return nil
}
@@ -756,6 +1022,9 @@ func NewVersion(
} else {
v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp)
}
for _, f := range files[l] {
v.Levels[l].totalSize += f.Size
}
}
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
panic(err)
@@ -788,7 +1057,7 @@ func NewVersion(
// key in a higher level table that has both the same user key and a higher
// sequence number.
type Version struct {
refs int32
refs atomic.Int32
// The level 0 sstables are organized in a series of sublevels. Similar to
// the seqnum invariant in normal levels, there is no internal key in a
@@ -805,7 +1074,7 @@ type Version struct {
// usage.
//
// L0Sublevels.Levels contains L0 files ordered by sublevels. All the files
// in Files[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
// in Levels[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
// a reference to that slice, as that slice is necessary for iterator
// creation and needs to outlast L0Sublevels.
L0Sublevels *L0Sublevels
@@ -820,7 +1089,7 @@ type Version struct {
// The callback to invoke when the last reference to a version is
// removed. Will be called with list.mu held.
Deleted func(obsolete []*FileMetadata)
Deleted func(obsolete []*FileBacking)
// Stats holds aggregated stats about the version maintained from
// version to version.
@@ -849,15 +1118,21 @@ func (v *Version) DebugString(format base.FormatKey) string {
return v.string(format, true)
}
func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string {
var buf bytes.Buffer
for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- {
fmt.Fprintf(&buf, "0.%d:\n", sublevel)
sublevels[sublevel].Each(func(f *FileMetadata) {
fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose))
})
}
return buf.String()
}
func (v *Version) string(format base.FormatKey, verbose bool) string {
var buf bytes.Buffer
if len(v.L0SublevelFiles) > 0 {
for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
fmt.Fprintf(&buf, "0.%d:\n", sublevel)
v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) {
fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose))
})
}
fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles))
}
for level := 1; level < NumLevels; level++ {
if v.Levels[level].Empty() {
@@ -898,7 +1173,7 @@ func ParseVersionDebug(
m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
m.HasPointKeys = true
}
files[level] = append(files[level], &m)
files[level] = append(files[level], m)
}
}
// Reverse the order of L0 files. This ensures we construct the same
@@ -912,12 +1187,12 @@ func ParseVersionDebug(
// Refs returns the number of references to the version.
func (v *Version) Refs() int32 {
return atomic.LoadInt32(&v.refs)
return v.refs.Load()
}
// Ref increments the version refcount.
func (v *Version) Ref() {
atomic.AddInt32(&v.refs, 1)
v.refs.Add(1)
}
// Unref decrements the version refcount. If the last reference to the version
@@ -925,12 +1200,11 @@ func (v *Version) Ref() {
// Deleted callback is invoked. Requires that the VersionList mutex is NOT
// locked.
func (v *Version) Unref() {
if atomic.AddInt32(&v.refs, -1) == 0 {
obsolete := v.unrefFiles()
if v.refs.Add(-1) == 0 {
l := v.list
l.mu.Lock()
l.Remove(v)
v.Deleted(obsolete)
v.Deleted(v.unrefFiles())
l.mu.Unlock()
}
}
@@ -940,14 +1214,14 @@ func (v *Version) Unref() {
// the Deleted callback is invoked. Requires that the VersionList mutex is
// already locked.
func (v *Version) UnrefLocked() {
if atomic.AddInt32(&v.refs, -1) == 0 {
if v.refs.Add(-1) == 0 {
v.list.Remove(v)
v.Deleted(v.unrefFiles())
}
}
func (v *Version) unrefFiles() []*FileMetadata {
var obsolete []*FileMetadata
func (v *Version) unrefFiles() []*FileBacking {
var obsolete []*FileBacking
for _, lm := range v.Levels {
obsolete = append(obsolete, lm.release()...)
}
@@ -1079,16 +1353,20 @@ func (v *Version) Overlaps(
// CheckOrdering checks that the files are consistent with respect to
// increasing file numbers (for level 0 files) and increasing and non-
// overlapping internal key ranges (for level non-0 files).
func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
func (v *Version) CheckOrdering(
cmp Compare, format base.FormatKey, order OrderingInvariants,
) error {
for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
sublevelIter := v.L0SublevelFiles[sublevel].Iter()
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
// Sublevels have NEVER allowed split user keys, so we can pass
// ProhibitSplitUserKeys.
if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
for level, lm := range v.Levels {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
}
}
@@ -1157,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
v.list = nil // avoid memory leaks
}
// OrderingInvariants dictates the file ordering invariants active.
type OrderingInvariants int8
const (
// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
// contain the same user key.
ProhibitSplitUserKeys OrderingInvariants = iota
// AllowSplitUserKeys indicates that adjacent files within a level may
// contain the same user key. This is only allowed by historical format
// major versions.
//
// TODO(jackson): Remove.
AllowSplitUserKeys
)
// CheckOrdering checks that the files are consistent with respect to
// seqnums (for level 0 files -- see detailed comment below) and increasing and non-
// overlapping internal key ranges (for non-level 0 files).
func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
//
// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
// inclusive of the same user key. Pebble no longer creates version edits
// installing such files, and Pebble databases with sufficiently high format
// major version should no longer have any such files within their LSM.
// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
// earlier format major versions.
func CheckOrdering(
cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
) error {
// The invariants to check for L0 sublevels are the same as the ones to
// check for all other levels. However, if L0 is not organized into
// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1238,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
// What's considered "overlapping" is dependent on the format
// major version. If ordering=ProhibitSplitUserKeys, then both
// files cannot contain keys with the same user keys. If the
// bounds have the same user key, the previous file's boundary
// must have a Trailer indicating that it's exclusive.
switch ordering {
case AllowSplitUserKeys:
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
case ProhibitSplitUserKeys:
if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
default:
panic("unreachable")
}
}
}

403
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored
View File

@@ -11,11 +11,11 @@ import (
"fmt"
"io"
"sort"
"sync/atomic"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// TODO(peter): describe the MANIFEST file format, independently of the C++
@@ -51,7 +51,9 @@ const (
tagMaxColumnFamily = 203
// Pebble tags.
tagNewFile5 = 104 // Range keys.
tagNewFile5 = 104 // Range keys.
tagCreatedBackingTable = 105
tagRemovedBackingTable = 106
// The custom tags sub-format used by tagNewFile4 and above.
customTagTerminate = 1
@@ -59,6 +61,7 @@ const (
customTagCreationTime = 6
customTagPathID = 65
customTagNonSafeIgnoreMask = 1 << 6
customTagVirtual = 66
)
// DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -73,6 +76,9 @@ type DeletedFileEntry struct {
type NewFileEntry struct {
Level int
Meta *FileMetadata
// BackingFileNum is only set during manifest replay, and only for virtual
// sstables.
BackingFileNum base.DiskFileNum
}
// VersionEdit holds the state for an edit to a Version along with other
@@ -110,9 +116,42 @@ type VersionEdit struct {
// found that there was no overlapping file at the higher level).
DeletedFiles map[DeletedFileEntry]*FileMetadata
NewFiles []NewFileEntry
// CreatedBackingTables can be used to preserve the FileBacking associated
// with a physical sstable. This is useful when virtual sstables in the
// latest version are reconstructed during manifest replay, and we also need
// to reconstruct the FileBacking which is required by these virtual
// sstables.
//
// INVARIANT: The FileBacking associated with a physical sstable must only
// be added as a backing file in the same version edit where the physical
// sstable is first virtualized. This means that the physical sstable must
// be present in DeletedFiles and that there must be at least one virtual
// sstable with the same FileBacking as the physical sstable in NewFiles. A
// file must be present in CreatedBackingTables in exactly one version edit.
// The physical sstable associated with the FileBacking must also not be
// present in NewFiles.
CreatedBackingTables []*FileBacking
// RemovedBackingTables is used to remove the FileBacking associated with a
// virtual sstable. Note that a backing sstable can be removed as soon as
// there are no virtual sstables in the latest version which are using the
// backing sstable, but the backing sstable doesn't necessarily have to be
// removed atomically with the version edit which removes the last virtual
// sstable associated with the backing sstable. The removal can happen in a
// future version edit.
//
// INVARIANT: A file must only be added to RemovedBackingTables if it was
// added to CreateBackingTables in a prior version edit. The same version
// edit also cannot have the same file present in both CreateBackingTables
// and RemovedBackingTables. A file must be present in RemovedBackingTables
// in exactly one version edit.
RemovedBackingTables []base.DiskFileNum
}
// Decode decodes an edit from the specified reader.
//
// Note that the Decode step will not set the FileBacking for virtual sstables
// and the responsibility is left to the caller. However, the Decode step will
// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles.
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(byteReader)
if !ok {
@@ -165,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
// NB: RocksDB does not use compaction pointers anymore.
case tagRemovedBackingTable:
n, err := d.readUvarint()
if err != nil {
return err
}
v.RemovedBackingTables = append(
v.RemovedBackingTables, base.FileNum(n).DiskFileNum(),
)
case tagCreatedBackingTable:
dfn, err := d.readUvarint()
if err != nil {
return err
}
size, err := d.readUvarint()
if err != nil {
return err
}
fileBacking := &FileBacking{
DiskFileNum: base.FileNum(dfn).DiskFileNum(),
Size: size,
}
v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking)
case tagDeletedFile:
level, err := d.readLevel()
if err != nil {
@@ -270,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
var markedForCompaction bool
var creationTime uint64
virtualState := struct {
virtual bool
backingFileNum uint64
}{}
if tag == tagNewFile4 || tag == tagNewFile5 {
for {
customTag, err := d.readUvarint()
@@ -278,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
if customTag == customTagTerminate {
break
} else if customTag == customTagVirtual {
virtualState.virtual = true
n, err := d.readUvarint()
if err != nil {
return err
}
virtualState.backingFileNum = n
continue
}
field, err := d.readBytes()
if err != nil {
return err
@@ -314,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
SmallestSeqNum: smallestSeqNum,
LargestSeqNum: largestSeqNum,
MarkedForCompaction: markedForCompaction,
Virtual: virtualState.virtual,
}
if tag != tagNewFile5 { // no range keys present
m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
@@ -345,10 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error {
}
}
m.boundsSet = true
v.NewFiles = append(v.NewFiles, NewFileEntry{
if !virtualState.virtual {
m.InitPhysicalBacking()
}
nfe := NewFileEntry{
Level: level,
Meta: m,
})
}
if virtualState.virtual {
nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum()
}
v.NewFiles = append(v.NewFiles, nfe)
case tagPrevLogNumber:
n, err := d.readUvarint()
@@ -367,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
return nil
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
var buf bytes.Buffer
if v.ComparerName != "" {
fmt.Fprintf(&buf, " comparer: %s", v.ComparerName)
@@ -399,7 +481,12 @@ func (v *VersionEdit) String() string {
fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum)
}
for _, nf := range v.NewFiles {
fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String())
fmt.Fprintf(&buf, " added: L%d", nf.Level)
if verbose {
fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
} else {
fmt.Fprintf(&buf, " %s", nf.Meta.String())
}
if nf.Meta.CreationTime != 0 {
fmt.Fprintf(&buf, " (%s)",
time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -409,6 +496,16 @@ func (v *VersionEdit) String() string {
return buf.String()
}
// DebugString is a more verbose version of String(). Use this in tests.
func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
return v.string(true /* verbose */, fmtKey)
}
// String implements fmt.Stringer for a VersionEdit.
func (v *VersionEdit) String() string {
return v.string(false /* verbose */, base.DefaultFormatter)
}
// Encode encodes an edit to the specified writer.
func (v *VersionEdit) Encode(w io.Writer) error {
e := versionEditEncoder{new(bytes.Buffer)}
@@ -429,6 +526,15 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(tagNextFileNumber)
e.writeUvarint(uint64(v.NextFileNum))
}
for _, dfn := range v.RemovedBackingTables {
e.writeUvarint(tagRemovedBackingTable)
e.writeUvarint(uint64(dfn.FileNum()))
}
for _, fileBacking := range v.CreatedBackingTables {
e.writeUvarint(tagCreatedBackingTable)
e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum()))
e.writeUvarint(fileBacking.Size)
}
// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
// even though its value is zero. We detect this by encoding LastSeqNum when
// ComparerName is set.
@@ -442,7 +548,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(uint64(x.FileNum))
}
for _, x := range v.NewFiles {
customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
var tag uint64
switch {
case x.Meta.HasRangeKeys:
@@ -495,6 +601,10 @@ func (v *VersionEdit) Encode(w io.Writer) error {
e.writeUvarint(customTagNeedsCompaction)
e.writeBytes([]byte{1})
}
if x.Meta.Virtual {
e.writeUvarint(customTagVirtual)
e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum()))
}
e.writeUvarint(customTagTerminate)
}
}
@@ -502,6 +612,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
return err
}
// versionEditDecoder should be used to decode version edits.
type versionEditDecoder struct {
byteReader
}
@@ -581,10 +692,30 @@ func (e versionEditEncoder) writeUvarint(u uint64) {
// BulkVersionEdit summarizes the files added and deleted from a set of version
// edits.
//
// INVARIANTS:
// No file can be added to a level more than once. This is true globally, and
// also true for all of the calls to Accumulate for a single bulk version edit.
//
// No file can be removed from a level more than once. This is true globally,
// and also true for all of the calls to Accumulate for a single bulk version
// edit.
//
// A file must not be added and removed from a given level in the same version
// edit.
//
// A file that is being removed from a level must have been added to that level
// before (in a prior version edit). Note that a given file can be deleted from
// a level and added to another level in a single version edit
type BulkVersionEdit struct {
Added [NumLevels][]*FileMetadata
Added [NumLevels]map[base.FileNum]*FileMetadata
Deleted [NumLevels]map[base.FileNum]*FileMetadata
// AddedFileBacking is a map to support lookup so that we can populate the
// FileBacking of virtual sstables during manifest replay.
AddedFileBacking map[base.DiskFileNum]*FileBacking
RemovedFileBacking []base.DiskFileNum
// AddedByFileNum maps file number to file metadata for all added files
// from accumulated version edits. AddedByFileNum is only populated if set
// to non-nil by a caller. It must be set to non-nil when replaying
@@ -603,6 +734,19 @@ type BulkVersionEdit struct {
// Accumulate adds the file addition and deletions in the specified version
// edit to the bulk edit's internal state.
//
// INVARIANTS:
// If a file is added to a given level in a call to Accumulate and then removed
// from that level in a subsequent call, the file will not be present in the
// resulting BulkVersionEdit.Deleted for that level.
//
// After accumulation of version edits, the bulk version edit may have
// information about a file which has been deleted from a level, but it may
// not have information about the same file added to the same level. The add
// could've occurred as part of a previous bulk version edit. In this case,
// the deleted file must be present in BulkVersionEdit.Deleted, at the end
// of the accumulation, because we need to decrease the refcount of the
// deleted file in Apply.
func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
for df, m := range ve.DeletedFiles {
dmap := b.Deleted[df.Level]
@@ -624,7 +768,28 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
if m.MarkedForCompaction {
b.MarkedForCompactionCountDiff--
}
dmap[df.FileNum] = m
if _, ok := b.Added[df.Level][df.FileNum]; !ok {
dmap[df.FileNum] = m
} else {
// Present in b.Added for the same level.
delete(b.Added[df.Level], df.FileNum)
}
}
// Generate state for Added backing files. Note that these must be generated
// before we loop through the NewFiles, because we need to populate the
// FileBackings which might be used by the NewFiles loop.
if b.AddedFileBacking == nil {
b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
}
for _, fb := range ve.CreatedBackingTables {
if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
// There is already a FileBacking associated with fb.DiskFileNum.
// This should never happen. There must always be only one FileBacking
// associated with a backing sstable.
panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
}
b.AddedFileBacking[fb.DiskFileNum] = fb
}
for _, nf := range ve.NewFiles {
@@ -635,7 +800,21 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
}
}
b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta)
if nf.Meta.Virtual && nf.Meta.FileBacking == nil {
// FileBacking for a virtual sstable must only be nil if we're performing
// manifest replay.
nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum]
if nf.Meta.FileBacking == nil {
return errors.Errorf("FileBacking for virtual sstable must not be nil")
}
} else if nf.Meta.FileBacking == nil {
return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum)
}
if b.Added[nf.Level] == nil {
b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
}
b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta
if b.AddedByFileNum != nil {
b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta
}
@@ -643,36 +822,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
b.MarkedForCompactionCountDiff++
}
}
// Since a file can be removed from backing files in exactly one version
// edit it is safe to just append without any de-duplication.
b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)
return nil
}
// AccumulateIncompleteAndApplySingleVE should be called if a single version edit
// is to be applied to the provided curr Version and if the caller needs to
// update the versionSet.zombieTables map. This function exists separately from
// BulkVersionEdit.Apply because it is easier to reason about properties
// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we
// know that exactly one version edit is being accumulated.
//
// Note that the version edit passed into this function may be incomplete
// because compactions don't have the ref counting information necessary to
// populate VersionEdit.RemovedBackingTables. This function will complete such a
// version edit by populating RemovedBackingTables.
//
// Invariant: Any file being deleted through ve must belong to the curr Version.
// We can't have a delete for some arbitrary file which does not exist in curr.
func AccumulateIncompleteAndApplySingleVE(
ve *VersionEdit,
curr *Version,
cmp Compare,
formatKey base.FormatKey,
flushSplitBytes int64,
readCompactionRate int64,
backingStateMap map[base.DiskFileNum]*FileBacking,
addBackingFunc func(*FileBacking),
removeBackingFunc func(base.DiskFileNum),
orderingInvariants OrderingInvariants,
) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
if len(ve.RemovedBackingTables) != 0 {
panic("pebble: invalid incomplete version edit")
}
var b BulkVersionEdit
err := b.Accumulate(ve)
if err != nil {
return nil, nil, err
}
zombies = make(map[base.DiskFileNum]uint64)
v, err := b.Apply(
curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
)
if err != nil {
return nil, nil, err
}
for _, s := range b.AddedFileBacking {
addBackingFunc(s)
}
for fileNum := range zombies {
if _, ok := backingStateMap[fileNum]; ok {
// This table was backing some virtual sstable in the latest version,
// but is now a zombie. We add RemovedBackingTables entries for
// these, before the version edit is written to disk.
ve.RemovedBackingTables = append(
ve.RemovedBackingTables, fileNum,
)
removeBackingFunc(fileNum)
}
}
return v, zombies, nil
}
// Apply applies the delta b to the current version to produce a new
// version. The new version is consistent with respect to the comparer cmp.
//
// curr may be nil, which is equivalent to a pointer to a zero version.
//
// On success, a map of zombie files containing the file numbers and sizes of
// deleted files is returned. These files are considered zombies because they
// are no longer referenced by the returned Version, but cannot be deleted from
// disk as they are still in use by the incoming Version.
// On success, if a non-nil zombies map is provided to Apply, the map is updated
// with file numbers and files sizes of deleted files. These files are
// considered zombies because they are no longer referenced by the returned
// Version, but cannot be deleted from disk as they are still in use by the
// incoming Version.
func (b *BulkVersionEdit) Apply(
curr *Version,
cmp Compare,
formatKey base.FormatKey,
flushSplitBytes int64,
readCompactionRate int64,
) (_ *Version, zombies map[base.FileNum]uint64, _ error) {
addZombie := func(fileNum base.FileNum, size uint64) {
if zombies == nil {
zombies = make(map[base.FileNum]uint64)
}
zombies[fileNum] = size
}
// The remove zombie function is used to handle tables that are moved from
// one level to another during a version edit (i.e. a "move" compaction).
removeZombie := func(fileNum base.FileNum) {
zombies map[base.DiskFileNum]uint64,
orderingInvariants OrderingInvariants,
) (*Version, error) {
addZombie := func(state *FileBacking) {
if zombies != nil {
delete(zombies, fileNum)
zombies[state.DiskFileNum] = state.Size
}
}
removeZombie := func(state *FileBacking) {
if zombies != nil {
delete(zombies, state.DiskFileNum)
}
}
@@ -684,7 +928,7 @@ func (b *BulkVersionEdit) Apply(
}
v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff
if v.Stats.MarkedForCompaction < 0 {
return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
}
for level := range v.Levels {
@@ -705,7 +949,7 @@ func (b *BulkVersionEdit) Apply(
// Initialize L0Sublevels.
if curr == nil || curr.L0Sublevels == nil {
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
} else {
v.L0Sublevels = curr.L0Sublevels
@@ -718,47 +962,70 @@ func (b *BulkVersionEdit) Apply(
// Some edits on this level.
lm := &v.Levels[level]
lmRange := &v.RangeKeyLevels[level]
addedFiles := b.Added[level]
deletedMap := b.Deleted[level]
if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
return nil, nil, base.CorruptionErrorf(
addedFilesMap := b.Added[level]
deletedFilesMap := b.Deleted[level]
if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 {
return nil, base.CorruptionErrorf(
"pebble: internal error: No current or added files but have deleted files: %d",
errors.Safe(len(deletedMap)))
errors.Safe(len(deletedFilesMap)))
}
// NB: addedFiles may be empty and it also is not necessarily
// internally consistent: it does not reflect deletions in deletedMap.
// NB: addedFilesMap may be empty. If a file is present in addedFilesMap
// for a level, it won't be present in deletedFilesMap for the same
// level.
for _, f := range deletedMap {
addZombie(f.FileNum, f.Size)
if obsolete := v.Levels[level].tree.Delete(f); obsolete {
for _, f := range deletedFilesMap {
if obsolete := v.Levels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
// file's reference count dropping to zero.
err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
return nil, nil, err
return nil, err
}
if f.HasRangeKeys {
if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
// file's reference count dropping to zero.
err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum)
return nil, nil, err
return nil, err
}
}
// Note that a backing sst will only become a zombie if the
// references to it in the latest version is 0. We will remove the
// backing sst from the zombie list in the next loop if one of the
// addedFiles in any of the levels is referencing the backing sst.
// This is possible if a physical sstable is virtualized, or if it
// is moved.
latestRefCount := f.LatestRefs()
if latestRefCount <= 0 {
// If a file is present in deletedFilesMap for a level, then it
// must have already been added to the level previously, which
// means that its latest ref count cannot be 0.
err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum)
return nil, err
} else if f.LatestUnref() == 0 {
addZombie(f.FileBacking)
}
}
addedFiles := make([]*FileMetadata, 0, len(addedFilesMap))
for _, f := range addedFilesMap {
addedFiles = append(addedFiles, f)
}
// Sort addedFiles by file number. This isn't necessary, but tests which
// replay invalid manifests check the error output, and the error output
// depends on the order in which files are added to the btree.
sort.Slice(addedFiles, func(i, j int) bool {
return addedFiles[i].FileNum < addedFiles[j].FileNum
})
var sm, la *FileMetadata
for _, f := range addedFiles {
if _, ok := deletedMap[f.FileNum]; ok {
// Already called addZombie on this file in the preceding
// loop, so we don't need to do it here.
continue
}
// NB: allowedSeeks is used for read triggered compactions. It is set using
// Options.Experimental.ReadCompactionRate which defaults to 32KB.
var allowedSeeks int64
@@ -768,20 +1035,23 @@ func (b *BulkVersionEdit) Apply(
if allowedSeeks < 100 {
allowedSeeks = 100
}
atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks)
f.AllowedSeeks.Store(allowedSeeks)
f.InitAllowedSeeks = allowedSeeks
err := lm.tree.Insert(f)
err := lm.insert(f)
// We're adding this file to the new version, so increment the
// latest refs count.
f.LatestRef()
if err != nil {
return nil, nil, errors.Wrap(err, "pebble")
return nil, errors.Wrap(err, "pebble")
}
if f.HasRangeKeys {
err = lmRange.tree.Insert(f)
err = lmRange.insert(f)
if err != nil {
return nil, nil, errors.Wrap(err, "pebble")
return nil, errors.Wrap(err, "pebble")
}
}
removeZombie(f.FileNum)
removeZombie(f.FileBacking)
// Track the keys with the smallest and largest keys, so that we can
// check consistency of the modified span.
if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 {
@@ -793,27 +1063,36 @@ func (b *BulkVersionEdit) Apply(
}
if level == 0 {
if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 {
if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 {
// Flushes and ingestions that do not delete any L0 files do not require
// a regeneration of L0Sublevels from scratch. We can instead generate
// it incrementally.
var err error
// AddL0Files requires addedFiles to be sorted in seqnum order.
addedFiles = append([]*FileMetadata(nil), addedFiles...)
SortBySeqNum(addedFiles)
v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0])
if errors.Is(err, errInvalidL0SublevelsOpt) {
err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes)
} else if invariants.Enabled && err == nil {
copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes)
if err != nil {
panic(fmt.Sprintf("error when regenerating sublevels: %s", err))
}
s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels)
s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels)
if s1 != s2 {
panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2))
}
}
if err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
v.L0SublevelFiles = v.L0Sublevels.Levels
} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
return nil, errors.Wrap(err, "pebble: internal error")
}
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
continue
}
@@ -833,10 +1112,10 @@ func (b *BulkVersionEdit) Apply(
end.Prev()
}
})
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
return nil, errors.Wrap(err, "pebble: internal error")
}
}
}
return v, zombies, nil
return v, nil
}

2
vendor/github.com/cockroachdb/pebble/internal/private/sstable.go generated vendored
View File

@@ -8,7 +8,7 @@ import "github.com/cockroachdb/pebble/internal/base"
// SSTableCacheOpts is a hook for specifying cache options to
// sstable.NewReader.
var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{}
var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{}
// SSTableRawTombstonesOpt is a sstable.Reader option for disabling
// fragmentation of the range tombstones returned by

71
vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go generated vendored
View File

@@ -52,15 +52,15 @@ import (
// │
// ╰── <?>.Next
type UserIteratorConfig struct {
snapshot uint64
comparer *base.Comparer
miter keyspan.MergingIter
biter keyspan.BoundedIter
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
onlySets bool
bufs *Buffers
snapshot uint64
comparer *base.Comparer
miter keyspan.MergingIter
biter keyspan.BoundedIter
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
internalKeys bool
bufs *Buffers
}
// Buffers holds various buffers used for range key iteration. They're exposed
@@ -79,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
// Init initializes the range key iterator stack for user iteration. The
// resulting fragment iterator applies range key semantics, defragments spans
// according to their user-observable state and, if onlySets = true, removes all
// according to their user-observable state and, if !internalKeys, removes all
// Keys other than RangeKeySets describing the current state of range keys. The
// resulting spans contain Keys sorted by Suffix.
// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
// in which case they remain sorted by trailer descending).
//
// The snapshot sequence number parameter determines which keys are visible. Any
// keys not visible at the provided snapshot are ignored.
@@ -91,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
lower, upper []byte,
hasPrefix *bool,
prefix *[]byte,
onlySets bool,
internalKeys bool,
bufs *Buffers,
iters ...keyspan.FragmentIterator,
) keyspan.FragmentIterator {
ui.snapshot = snapshot
ui.comparer = comparer
ui.onlySets = onlySets
ui.internalKeys = internalKeys
ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
if internalKeys {
ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
} else {
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
}
ui.litersUsed = 0
ui.bufs = bufs
return &ui.diter
@@ -136,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
// of unset keys, removal of keys overwritten by a set at the same suffix, etc)
// and then non-RangeKeySet keys are removed. The resulting transformed spans
// only contain RangeKeySets describing the state visible at the provided
// sequence number, and hold their Keys sorted by Suffix.
// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
// is true, then keys remain sorted by trailer.
func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
// Apply shadowing of keys.
dst.Start = s.Start
@@ -148,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
return err
}
// During user iteration over range keys, unsets and deletes don't matter.
// Remove them if onlySets = true. This step helps logical defragmentation
// during iteration.
if ui.internalKeys {
if s.KeysOrder != keyspan.ByTrailerDesc {
panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
}
dst.Keys = ui.bufs.sortBuf.Keys
keyspan.SortKeysByTrailer(&dst.Keys)
return nil
}
// During user iteration over range keys, unsets and deletes don't matter. This
// step helps logical defragmentation during iteration.
keys := ui.bufs.sortBuf.Keys
dst.Keys = dst.Keys[:0]
for i := range keys {
@@ -164,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
// Skip.
continue
case base.InternalKeyKindRangeKeyDelete:
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
// Skip.
continue
default:
return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
}
@@ -193,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
// sequence numbers). It's intended for use during user iteration, when the
// wrapped keyspan iterator is merging spans across all levels of the LSM.
func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
// This method is not called with internalKeys = true.
if ui.internalKeys {
panic("unexpected call to ShouldDefragment with internalKeys = true")
}
// This implementation must only be used on spans that have transformed by
// ui.Transform. The transform applies shadowing, removes all keys besides
// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -208,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
ret := true
for i := range a.Keys {
if invariants.Enabled {
if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
panic("pebble: unexpected non-RangeKeySet during defragmentation")
}
if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||

387
vendor/github.com/cockroachdb/pebble/internal/rate/rate.go generated vendored
View File

@@ -1,387 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package rate provides a rate limiter.
package rate // import "github.com/cockroachdb/pebble/internal/rate"
import (
"context"
"math"
"sync"
"time"
"github.com/cockroachdb/errors"
)
// Limit defines the maximum frequency of some events.
// Limit is represented as number of events per second.
// A zero Limit allows no events.
type Limit float64
// Inf is the infinite rate limit; it allows all events (even if burst is zero).
const Inf = Limit(math.MaxFloat64)
// Every converts a minimum time interval between events to a Limit.
func Every(interval time.Duration) Limit {
if interval <= 0 {
return Inf
}
return 1 / Limit(interval.Seconds())
}
// A Limiter controls how frequently events are allowed to happen.
// It implements a "token bucket" of size b, initially full and refilled
// at rate r tokens per second.
// Informally, in any large enough time interval, the Limiter limits the
// rate to r tokens per second, with a maximum burst size of b events.
// As a special case, if r == Inf (the infinite rate), b is ignored.
// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
//
// The zero value is a valid Limiter, but it will reject all events.
// Use NewLimiter to create non-zero Limiters.
//
// Limiter has three main methods, Allow, Reserve, and Wait.
// Most callers should use Wait.
//
// Each of the three methods consumes a single token.
// They differ in their behavior when no token is available.
// If no token is available, Allow returns false.
// If no token is available, Reserve returns a reservation for a future token
// and the amount of time the caller must wait before using it.
// If no token is available, Wait blocks until one can be obtained
// or its associated context.Context is canceled.
//
// The methods AllowN, ReserveN, and WaitN consume n tokens.
type Limiter struct {
limit Limit
burst int
mu sync.Mutex
tokens float64
// last is the last time the limiter's tokens field was updated
last time.Time
// lastEvent is the latest time of a rate-limited event (past or future)
lastEvent time.Time
}
// Limit returns the maximum overall event rate.
func (lim *Limiter) Limit() Limit {
lim.mu.Lock()
defer lim.mu.Unlock()
return lim.limit
}
// Burst returns the maximum burst size. Burst is the maximum number of tokens
// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
// Burst values allow more events to happen at once.
// A zero Burst allows no events, unless limit == Inf.
func (lim *Limiter) Burst() int {
return lim.burst
}
// NewLimiter returns a new Limiter that allows events up to rate r and permits
// bursts of at most b tokens.
func NewLimiter(r Limit, b int) *Limiter {
return &Limiter{
limit: r,
burst: b,
}
}
// Allow is shorthand for AllowN(time.Now(), 1).
func (lim *Limiter) Allow() bool {
return lim.AllowN(time.Now(), 1)
}
// AllowN reports whether n events may happen at time now.
// Use this method if you intend to drop / skip events that exceed the rate limit.
// Otherwise use Reserve or Wait.
func (lim *Limiter) AllowN(now time.Time, n int) bool {
return lim.reserveN(now, n, 0).ok
}
// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
// A Reservation may be canceled, which may enable the Limiter to permit additional events.
type Reservation struct {
ok bool
lim *Limiter
tokens int
timeToAct time.Time
// This is the Limit at reservation time, it can change later.
limit Limit
}
// OK returns whether the limiter can provide the requested number of tokens
// within the maximum wait time. If OK is false, Delay returns InfDuration, and
// Cancel does nothing.
func (r *Reservation) OK() bool {
return r.ok
}
// Delay is shorthand for DelayFrom(time.Now()).
func (r *Reservation) Delay() time.Duration {
return r.DelayFrom(time.Now())
}
// InfDuration is the duration returned by Delay when a Reservation is not OK.
const InfDuration = time.Duration(1<<63 - 1)
// DelayFrom returns the duration for which the reservation holder must wait
// before taking the reserved action. Zero duration means act immediately.
// InfDuration means the limiter cannot grant the tokens requested in this
// Reservation within the maximum wait time.
func (r *Reservation) DelayFrom(now time.Time) time.Duration {
if !r.ok {
return InfDuration
}
delay := r.timeToAct.Sub(now)
if delay < 0 {
return 0
}
return delay
}
// Cancel is shorthand for CancelAt(time.Now()).
func (r *Reservation) Cancel() {
r.CancelAt(time.Now())
}
// CancelAt indicates that the reservation holder will not perform the reserved action
// and reverses the effects of this Reservation on the rate limit as much as possible,
// considering that other reservations may have already been made.
func (r *Reservation) CancelAt(now time.Time) {
if !r.ok {
return
}
r.lim.mu.Lock()
defer r.lim.mu.Unlock()
if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
return
}
// calculate tokens to restore
// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
// after r was obtained. These tokens should not be restored.
restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
if restoreTokens <= 0 {
return
}
// advance time to now
now, _, tokens := r.lim.advance(now)
// calculate new number of tokens
tokens += restoreTokens
if burst := float64(r.lim.burst); tokens > burst {
tokens = burst
}
// update state
r.lim.last = now
r.lim.tokens = tokens
if r.timeToAct == r.lim.lastEvent {
prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
if !prevEvent.Before(now) {
r.lim.lastEvent = prevEvent
}
}
}
// Reserve is shorthand for ReserveN(time.Now(), 1).
func (lim *Limiter) Reserve() Reservation {
return lim.ReserveN(time.Now(), 1)
}
// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
// The Limiter takes this Reservation into account when allowing future events.
// ReserveN returns false if n exceeds the Limiter's burst size.
// Usage example:
//
// r := lim.ReserveN(time.Now(), 1)
// if !r.OK() {
// // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
// return
// }
// time.Sleep(r.Delay())
// Act()
//
// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
// If you need to respect a deadline or cancel the delay, use Wait instead.
// To drop or skip events exceeding rate limit, use Allow instead.
func (lim *Limiter) ReserveN(now time.Time, n int) Reservation {
return lim.reserveN(now, n, InfDuration)
}
// Wait is shorthand for WaitN(ctx, 1).
func (lim *Limiter) Wait(ctx context.Context) (err error) {
return lim.WaitN(ctx, 1)
}
// WaitN blocks until lim permits n events to happen.
// It returns an error if n exceeds the Limiter's burst size, the Context is
// canceled, or the expected wait time exceeds the Context's Deadline.
func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
if n > lim.burst && lim.limit != Inf {
return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst))
}
// Check if ctx is already cancelled
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// Determine wait limit
now := time.Now()
waitLimit := InfDuration
if deadline, ok := ctx.Deadline(); ok {
waitLimit = deadline.Sub(now)
}
// Reserve
r := lim.reserveN(now, n, waitLimit)
if !r.ok {
return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n))
}
// Wait
d := r.DelayFrom(now)
if d <= 0 {
return nil
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
// We can proceed.
return nil
case <-ctx.Done():
// Context was canceled before we could proceed. Cancel the
// reservation, which may permit other events to proceed sooner.
r.Cancel()
return ctx.Err()
}
}
// Delay is shorthand for DelayN(time.Now(), 1).
func (lim *Limiter) Delay() time.Duration {
return lim.DelayN(time.Now(), 1)
}
// DelayN returns the delay to wait to permit n events to happen. Zero duration
// means act immediately. InfDuration means the limiter cannot grant the tokens
// requested within the maximum wait time.
func (lim *Limiter) DelayN(now time.Time, n int) time.Duration {
r := lim.reserveN(now, n, InfDuration)
return r.DelayFrom(now)
}
// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
func (lim *Limiter) SetLimit(newLimit Limit) {
lim.SetLimitAt(time.Now(), newLimit)
}
// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
// before SetLimitAt was called.
func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
lim.last = now
lim.tokens = tokens
lim.limit = newLimit
}
// reserveN is a helper method for AllowN, ReserveN, and WaitN.
// maxFutureReserve specifies the maximum reservation wait duration allowed.
// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
lim.mu.Lock()
if lim.limit == Inf {
lim.mu.Unlock()
return Reservation{
ok: true,
lim: lim,
tokens: n,
timeToAct: now,
}
}
now, last, tokens := lim.advance(now)
// Calculate the remaining number of tokens resulting from the request.
tokens -= float64(n)
// Calculate the wait duration
var waitDuration time.Duration
if tokens < 0 {
waitDuration = lim.limit.durationFromTokens(-tokens)
}
// Decide result
ok := n <= lim.burst && waitDuration <= maxFutureReserve
// Prepare reservation
r := Reservation{
ok: ok,
lim: lim,
limit: lim.limit,
}
if ok {
r.tokens = n
r.timeToAct = now.Add(waitDuration)
}
// Update state
if ok {
lim.last = now
lim.tokens = tokens
lim.lastEvent = r.timeToAct
} else {
lim.last = last
}
lim.mu.Unlock()
return r
}
// advance calculates and returns an updated state for lim resulting from the passage of time.
// lim is not changed.
func (lim *Limiter) advance(
now time.Time,
) (newNow time.Time, newLast time.Time, newTokens float64) {
last := lim.last
if now.Before(last) {
last = now
}
// Avoid making delta overflow below when last is very old.
maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
elapsed := now.Sub(last)
if elapsed > maxElapsed {
elapsed = maxElapsed
}
// Calculate the new number of tokens, due to time that passed.
delta := lim.limit.tokensFromDuration(elapsed)
tokens := lim.tokens + delta
if burst := float64(lim.burst); tokens > burst {
tokens = burst
}
return now, last, tokens
}
// durationFromTokens is a unit conversion function from the number of tokens to the duration
// of time it takes to accumulate them at a rate of limit tokens per second.
func (limit Limit) durationFromTokens(tokens float64) time.Duration {
seconds := tokens / float64(limit)
return time.Nanosecond * time.Duration(1e9*seconds)
}
// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
// which could be accumulated during that duration at a rate of limit tokens per second.
func (limit Limit) tokensFromDuration(d time.Duration) float64 {
return d.Seconds() * float64(limit)
}

229
vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go generated vendored
View File

@@ -20,17 +20,27 @@ import (
"strings"
"github.com/cockroachdb/pebble/internal/base"
"golang.org/x/exp/constraints"
"golang.org/x/exp/rand"
)
const alpha = "abcdefghijklmnopqrstuvwxyz"
const suffixDelim = '@'
var inverseAlphabet = make(map[byte]int64, len(alpha))
func init() {
for i := range alpha {
inverseAlphabet[alpha[i]] = int64(i)
}
}
// MaxSuffixLen is the maximum length of a suffix generated by this package.
var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64))
var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64)))
// Comparer is the comparer for test keys generated by this package.
var Comparer *base.Comparer = &base.Comparer{
var Comparer = &base.Comparer{
Compare: compare,
Equal: func(a, b []byte) bool { return compare(a, b) == 0 },
AbbreviatedKey: func(k []byte) uint64 {
@@ -136,7 +146,7 @@ func compareTimestamps(a, b []byte) int {
// Keyspace describes a finite keyspace of unsuffixed test keys.
type Keyspace interface {
// Count returns the number of keys that exist within this keyspace.
Count() int
Count() int64
// MaxLen returns the maximum length, in bytes, of a key within this
// keyspace. This is only guaranteed to return an upper bound.
@@ -144,20 +154,21 @@ type Keyspace interface {
// Slice returns the sub-keyspace from index i, inclusive, to index j,
// exclusive. The receiver is unmodified.
Slice(i, j int) Keyspace
Slice(i, j int64) Keyspace
// EveryN returns a key space that includes 1 key for every N keys in the
// original keyspace. The receiver is unmodified.
EveryN(n int) Keyspace
EveryN(n int64) Keyspace
key(buf []byte, i int) int
// key writes the i-th key to the buffer and returns the length.
key(buf []byte, i int64) int
}
// Divvy divides the provided keyspace into N equal portions, containing
// disjoint keys evenly distributed across the keyspace.
func Divvy(ks Keyspace, n int) []Keyspace {
func Divvy(ks Keyspace, n int64) []Keyspace {
ret := make([]Keyspace, n)
for i := 0; i < n; i++ {
for i := int64(0); i < n; i++ {
ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
}
return ret
@@ -175,7 +186,7 @@ func Alpha(maxLength int) Keyspace {
// KeyAt returns the i-th key within the keyspace with a suffix encoding the
// timestamp t.
func KeyAt(k Keyspace, i int, t int) []byte {
func KeyAt(k Keyspace, i int64, t int64) []byte {
b := make([]byte, k.MaxLen()+MaxSuffixLen)
return b[:WriteKeyAt(b, k, i, t)]
}
@@ -183,20 +194,20 @@ func KeyAt(k Keyspace, i int, t int) []byte {
// WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
// suffix encoding the timestamp t suffix. It returns the number of bytes
// written.
func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int {
func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int {
n := WriteKey(dst, k, i)
n += WriteSuffix(dst[n:], t)
return n
}
// Suffix returns the test keys suffix representation of timestamp t.
func Suffix(t int) []byte {
func Suffix(t int64) []byte {
b := make([]byte, MaxSuffixLen)
return b[:WriteSuffix(b, t)]
}
// SuffixLen returns the exact length of the given suffix when encoded.
func SuffixLen(t int) int {
func SuffixLen(t int64) int {
// Begin at 1 for the '@' delimiter, 1 for a single digit.
n := 2
t /= 10
@@ -208,42 +219,42 @@ func SuffixLen(t int) int {
}
// ParseSuffix returns the integer representation of the encoded suffix.
func ParseSuffix(s []byte) (int, error) {
return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
func ParseSuffix(s []byte) (int64, error) {
return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64)
}
// WriteSuffix writes the test keys suffix representation of timestamp t to dst,
// returning the number of bytes written.
func WriteSuffix(dst []byte, t int) int {
func WriteSuffix(dst []byte, t int64) int {
dst[0] = suffixDelim
n := 1
n += len(strconv.AppendInt(dst[n:n], int64(t), 10))
n += len(strconv.AppendInt(dst[n:n], t, 10))
return n
}
// Key returns the i-th unsuffixed key within the keyspace.
func Key(k Keyspace, i int) []byte {
func Key(k Keyspace, i int64) []byte {
b := make([]byte, k.MaxLen())
return b[:k.key(b, i)]
}
// WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
// returns the number of bytes written.
func WriteKey(dst []byte, k Keyspace, i int) int {
func WriteKey(dst []byte, k Keyspace, i int64) int {
return k.key(dst, i)
}
type alphabet struct {
alphabet []byte
maxLength int
headSkip int
tailSkip int
increment int
headSkip int64
tailSkip int64
increment int64
}
func (a alphabet) Count() int {
func (a alphabet) Count() int64 {
// Calculate the total number of keys, ignoring the increment.
total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip)
total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip
// The increment dictates that we take every N keys, where N = a.increment.
// Consider a total containing the 5 keys:
@@ -263,20 +274,25 @@ func (a alphabet) MaxLen() int {
return a.maxLength
}
func (a alphabet) Slice(i, j int) Keyspace {
func (a alphabet) Slice(i, j int64) Keyspace {
s := a
s.headSkip += i
s.tailSkip += a.Count() - j
return s
}
func (a alphabet) EveryN(n int) Keyspace {
func (a alphabet) EveryN(n int64) Keyspace {
s := a
s.increment *= n
return s
}
func keyCount(n, l int) int {
func keyCount(n, l int) int64 {
if n == 0 {
return 0
} else if n == 1 {
return int64(l)
}
// The number of representable keys in the keyspace is a function of the
// length of the alphabet n and the max key length l. Consider how the
// number of representable keys grows as l increases:
@@ -286,10 +302,10 @@ func keyCount(n, l int) int {
// l = 3: n + n^2 + n^3
// ...
// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1)
return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1)
}
func (a alphabet) key(buf []byte, idx int) int {
func (a alphabet) key(buf []byte, idx int64) int {
// This function generates keys of length 1..maxKeyLength, pulling
// characters from the alphabet. The idx determines which key to generate,
// generating the i-th lexicographically next key.
@@ -308,14 +324,14 @@ func (a alphabet) key(buf []byte, idx int) int {
keyCount(len(a.alphabet), a.maxLength))
}
func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int {
if keyCount == 0 || i > keyCount || i < 0 {
return 0
}
// Of the keyCount keys in the generative keyspace, how many are there
// starting with a particular character?
keysPerCharacter := keyCount / len(alphabet)
keysPerCharacter := keyCount / int64(len(alphabet))
// Find the character that the key at index i starts with and set it.
characterIdx := i / keysPerCharacter
@@ -347,3 +363,156 @@ func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
i = i - keysPerCharacter*characterIdx - 1
return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
}
// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey,
// returning the index of a particular key, given the provided alphabet and max
// length of a key.
//
// len(key) must be ≥ 1.
func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
i, ok := alphabet[key[0]]
if !ok {
panic(fmt.Sprintf("unrecognized alphabet character %v", key[0]))
}
// How many keys exist that start with the preceding i characters? Each of
// the i characters themselves are a key, plus the count of all the keys
// with one less character for each.
ret := i + i*keyCount(len(alphabet), n-1)
if len(key) > 1 {
ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1)
}
return ret
}
func abs(a int64) int64 {
if a < 0 {
return -a
}
return a
}
// RandomSeparator returns a random alphabetic key k such that a < k < b,
// pulling randomness from the provided random number generator. If dst is
// provided and the generated key fits within dst's capacity, the returned slice
// will use dst's memory.
//
// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
// will consist of the prefix P appended with the provided suffix. A zero suffix
// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b,
// but the generated key will not use the provided suffix. Note that it's
// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case
// RandomSeparator returns nil.
//
// If RandomSeparator generates a new prefix, the generated prefix will have
// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))).
//
// RandomSeparator panics if a or b fails to decode.
func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
if Comparer.Compare(a, b) >= 0 {
return nil
}
// Determine both keys' logical prefixes and suffixes.
ai := Comparer.Split(a)
bi := Comparer.Split(b)
ap := a[:ai]
bp := b[:bi]
maxLength = max[int](maxLength, max[int](len(ap), len(bp)))
var as, bs int64
var err error
if ai != len(a) {
as, err = ParseSuffix(a[ai:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", a))
}
}
if bi != len(b) {
bs, err = ParseSuffix(b[bi:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", b))
}
}
apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength)
bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength)
diff := bpIdx - apIdx
generatedIdx := bpIdx
if diff > 0 {
var add int64 = diff + 1
var start int64 = apIdx
if as == 1 {
// There's no expressible key with prefix a greater than a@1. So,
// exclude ap.
start = apIdx + 1
add = diff
}
if bs == 0 {
// No key with prefix b can sort before b@0. We don't want to pick b.
add--
}
// We're allowing generated id to be in the range [start, start + add - 1].
if start > start+add-1 {
return nil
}
// If we can generate a key which is actually in the middle of apIdx
// and bpIdx use it so that we don't have to bother about timestamps.
generatedIdx = rng.Int63n(add) + start
for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
generatedIdx = rng.Int63n(add) + start
}
}
switch {
case generatedIdx == apIdx && generatedIdx == bpIdx:
if abs(bs-as) <= 1 {
// There's no expressible suffix between the two, and there's no
// possible separator key.
return nil
}
// The key b is >= key a, but has the same prefix, so b must have the
// smaller timestamp, unless a has timestamp of 0.
//
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if as == 0 {
// bs > as
suffix = bs + rng.Int63n(10) + 1
} else {
// bs < as.
// Generate suffix in range [bs + 1, as - 1]
suffix = bs + 1 + rng.Int63n(as-bs-1)
}
case generatedIdx == apIdx:
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if as == 0 && suffix == 0 {
suffix++
} else if as != 0 && suffix >= as {
suffix = rng.Int63n(as)
}
case generatedIdx == bpIdx:
if suffix <= bs {
suffix = bs + rng.Int63n(10) + 1
}
}
if sz := maxLength + SuffixLen(suffix); cap(dst) < sz {
dst = make([]byte, sz)
} else {
dst = dst[:cap(dst)]
}
var w int
if suffix == 0 {
w = WriteKey(dst, Alpha(maxLength), generatedIdx)
} else {
w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
}
return dst[:w]
}
func max[I constraints.Ordered](a, b I) I {
if b > a {
return b
}
return a
}

332
vendor/github.com/cockroachdb/pebble/internal_iterator.go generated vendored
View File

@@ -1,332 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"context"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
)
// scanInternalIterator is an iterator that returns all internal keys instead of
// collapsing them by user keys. For instance, an InternalKeyKindDelete would be
// returned as an InternalKeyKindDelete instead of the iterator skipping over to
// the next key. Useful if an external user of Pebble needs to observe and
// rebuild Pebble's history of internal keys, such as in node-to-node
// replication. For use with {db,snapshot}.ScanInternal().
//
// scanInternalIterator is allowed to ignorepoint keys deleted by range deletions,
// and range keys shadowed by a range key unset or delete, however it *must*
// return the range delete as well as the range key unset/delete that did the
// shadowing.
type scanInternalIterator struct {
opts IterOptions
comparer *base.Comparer
iter internalIterator
readState *readState
rangeKey *iteratorRangeKeyState
pointKeyIter keyspan.InterleavingIter
iterKey *InternalKey
iterValue LazyValue
alloc *iterAlloc
newIters tableNewIters
newIterRangeKey keyspan.TableNewSpanIter
seqNum uint64
// boundsBuf holds two buffers used to store the lower and upper bounds.
// Whenever the InternalIterator's bounds change, the new bounds are copied
// into boundsBuf[boundsBufIdx]. The two bounds share a slice to reduce
// allocations. opts.LowerBound and opts.UpperBound point into this slice.
boundsBuf [2][]byte
boundsBufIdx int
}
// TODO(sumeer): scanInternalImpl is the implementation for the user-facing
// ScanInternal. Make it accept a context parameter, and plumb it through for
// tracing support.
func scanInternalImpl(
lower []byte,
iter *scanInternalIterator,
visitPointKey func(key *InternalKey, value LazyValue) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []keyspan.Key) error,
) error {
for valid := iter.seekGE(lower); valid && iter.error() == nil; valid = iter.next() {
key := iter.unsafeKey()
switch key.Kind() {
case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
span := iter.unsafeSpan()
if err := visitRangeKey(span.Start, span.End, span.Keys); err != nil {
return err
}
case InternalKeyKindRangeDelete:
rangeDel := iter.unsafeRangeDel()
if err := visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil {
return err
}
default:
val := iter.lazyValue()
if s := iter.pointKeyIter.Span(); s != nil && s.CoversAt(iter.seqNum, key.SeqNum()) {
// Key deleted by a range deletion. Skip it.
continue
}
if err := visitPointKey(key, val); err != nil {
return err
}
}
}
return nil
}
// constructPointIter constructs a merging iterator and sets i.iter to it.
func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *iterAlloc) {
// Merging levels and levels from iterAlloc.
mlevels := buf.mlevels[:0]
levels := buf.levels[:0]
// We compute the number of levels needed ahead of time and reallocate a slice if
// the array from the iterAlloc isn't large enough. Doing this allocation once
// should improve the performance.
numMergingLevels := len(memtables)
numLevelIters := 0
current := i.readState.current
numMergingLevels += len(current.L0SublevelFiles)
numLevelIters += len(current.L0SublevelFiles)
for level := 1; level < len(current.Levels); level++ {
if current.Levels[level].Empty() {
continue
}
numMergingLevels++
numLevelIters++
}
if numMergingLevels > cap(mlevels) {
mlevels = make([]mergingIterLevel, 0, numMergingLevels)
}
if numLevelIters > cap(levels) {
levels = make([]levelIter, 0, numLevelIters)
}
// TODO(bilal): Push these into the iterAlloc buf.
var rangeDelMiter keyspan.MergingIter
rangeDelIters := make([]keyspan.FragmentIterator, 0, numMergingLevels)
rangeDelLevels := make([]keyspan.LevelIter, 0, numLevelIters)
// Next are the memtables.
for j := len(memtables) - 1; j >= 0; j-- {
mem := memtables[j]
mlevels = append(mlevels, mergingIterLevel{
iter: mem.newIter(&i.opts),
})
if rdi := mem.newRangeDelIter(&i.opts); rdi != nil {
rangeDelIters = append(rangeDelIters, rdi)
}
}
// Next are the file levels: L0 sub-levels followed by lower levels.
mlevelsIndex := len(mlevels)
levelsIndex := len(levels)
mlevels = mlevels[:numMergingLevels]
levels = levels[:numLevelIters]
rangeDelLevels = rangeDelLevels[:numLevelIters]
addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) {
li := &levels[levelsIndex]
rli := &rangeDelLevels[levelsIndex]
li.init(
context.Background(), i.opts, i.comparer.Compare, i.comparer.Split, i.newIters, files, level,
internalIterOpts{})
li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext)
mlevels[mlevelsIndex].iter = li
rli.Init(keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters},
i.comparer.Compare, tableNewRangeDelIter(context.Background(), i.newIters), files, level,
manifest.KeyTypePoint)
rangeDelIters = append(rangeDelIters, rli)
levelsIndex++
mlevelsIndex++
}
// Add level iterators for the L0 sublevels, iterating from newest to
// oldest.
for i := len(current.L0SublevelFiles) - 1; i >= 0; i-- {
addLevelIterForFiles(current.L0SublevelFiles[i].Iter(), manifest.L0Sublevel(i))
}
// Add level iterators for the non-empty non-L0 levels.
for level := 1; level < numLevels; level++ {
if current.Levels[level].Empty() {
continue
}
addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level))
}
buf.merging.init(&i.opts, &InternalIteratorStats{}, i.comparer.Compare, i.comparer.Split, mlevels...)
buf.merging.snapshot = i.seqNum
rangeDelMiter.Init(i.comparer.Compare, keyspan.VisibleTransform(i.seqNum), new(keyspan.MergingBuffers), rangeDelIters...)
i.pointKeyIter.Init(i.comparer, &buf.merging, &rangeDelMiter, nil /* mask */, i.opts.LowerBound, i.opts.UpperBound)
i.iter = &i.pointKeyIter
}
// constructRangeKeyIter constructs the range-key iterator stack, populating
// i.rangeKey.rangeKeyIter with the resulting iterator. This is similar to
// Iterator.constructRangeKeyIter, except it doesn't handle batches and ensures
// iterConfig does *not* elide unsets/deletes.
func (i *scanInternalIterator) constructRangeKeyIter() {
// We want the bounded iter from iterConfig, but not the collapsing of
// RangeKeyUnsets and RangeKeyDels.
i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
nil /* hasPrefix */, nil /* prefix */, false, /* onlySets */
&i.rangeKey.rangeKeyBuffers.internal)
// Next are the flushables: memtables and large batches.
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
}
}
current := i.readState.current
// Next are the file levels: L0 sub-levels followed by lower levels.
//
// Add file-specific iterators for L0 files containing range keys. This is less
// efficient than using levelIters for sublevels of L0 files containing
// range keys, but range keys are expected to be sparse anyway, reducing the
// cost benefit of maintaining a separate L0Sublevels instance for range key
// files and then using it here.
//
// NB: We iterate L0's files in reverse order. They're sorted by
// LargestSeqNum ascending, and we need to add them to the merging iterator
// in LargestSeqNum descending to preserve the merging iterator's invariants
// around Key Trailer order.
iter := current.RangeKeyLevels[0].Iter()
for f := iter.Last(); f != nil; f = iter.Prev() {
spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
spanIter, err := i.newIterRangeKey(f, spanIterOpts)
if err != nil {
i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
continue
}
i.rangeKey.iterConfig.AddLevel(spanIter)
}
// Add level iterators for the non-empty non-L0 levels.
for level := 1; level < len(current.RangeKeyLevels); level++ {
if current.RangeKeyLevels[level].Empty() {
continue
}
li := i.rangeKey.iterConfig.NewLevelIter()
spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
manifest.Level(level), manifest.KeyTypeRange)
i.rangeKey.iterConfig.AddLevel(li)
}
}
// seekGE seeks this iterator to the first key that's greater than or equal
// to the specified user key.
func (i *scanInternalIterator) seekGE(key []byte) bool {
i.iterKey, i.iterValue = i.iter.SeekGE(key, base.SeekGEFlagsNone)
return i.iterKey != nil
}
// unsafeKey returns the unsafe InternalKey at the current position. The value
// is nil if the iterator is invalid or exhausted.
func (i *scanInternalIterator) unsafeKey() *InternalKey {
return i.iterKey
}
// lazyValue returns a value pointer to the value at the current iterator
// position. Behaviour undefined if unsafeKey() returns a Range key or Rangedel
// kind key.
func (i *scanInternalIterator) lazyValue() LazyValue {
return i.iterValue
}
// unsafeRangeDel returns a range key span. Behaviour undefined if UnsafeKey returns
// a non-rangedel kind.
func (i *scanInternalIterator) unsafeRangeDel() *keyspan.Span {
return i.pointKeyIter.Span()
}
// unsafeSpan returns a range key span. Behaviour undefined if UnsafeKey returns
// a non-rangekey type.
func (i *scanInternalIterator) unsafeSpan() *keyspan.Span {
return i.rangeKey.iiter.Span()
}
// next advances the iterator in the forward direction, and returns the
// iterator's new validity state.
func (i *scanInternalIterator) next() bool {
i.iterKey, i.iterValue = i.iter.Next()
return i.iterKey != nil
}
// error returns an error from the internal iterator, if there's any.
func (i *scanInternalIterator) error() error {
return i.iter.Error()
}
// close closes this iterator, and releases any pooled objects.
func (i *scanInternalIterator) close() error {
if err := i.iter.Close(); err != nil {
return err
}
i.readState.unref()
if i.rangeKey != nil {
i.rangeKey.PrepareForReuse()
*i.rangeKey = iteratorRangeKeyState{
rangeKeyBuffers: i.rangeKey.rangeKeyBuffers,
}
iterRangeKeyStateAllocPool.Put(i.rangeKey)
i.rangeKey = nil
}
if alloc := i.alloc; alloc != nil {
for j := range i.boundsBuf {
if cap(i.boundsBuf[j]) >= maxKeyBufCacheSize {
alloc.boundsBuf[j] = nil
} else {
alloc.boundsBuf[j] = i.boundsBuf[j]
}
}
*alloc = iterAlloc{
keyBuf: alloc.keyBuf[:0],
boundsBuf: alloc.boundsBuf,
prefixOrFullSeekKey: alloc.prefixOrFullSeekKey[:0],
}
iterAllocPool.Put(alloc)
i.alloc = nil
}
return nil
}
func (i *scanInternalIterator) initializeBoundBufs(lower, upper []byte) {
buf := i.boundsBuf[i.boundsBufIdx][:0]
if lower != nil {
buf = append(buf, lower...)
i.opts.LowerBound = buf
} else {
i.opts.LowerBound = nil
}
if upper != nil {
buf = append(buf, upper...)
i.opts.UpperBound = buf[len(buf)-len(upper):]
} else {
i.opts.UpperBound = nil
}
i.boundsBuf[i.boundsBufIdx] = buf
i.boundsBufIdx = 1 - i.boundsBufIdx
}

369
vendor/github.com/cockroachdb/pebble/iterator.go generated vendored
View File

@@ -9,7 +9,6 @@ import (
"context"
"io"
"sync"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
@@ -188,7 +187,9 @@ type Iterator struct {
comparer base.Comparer
iter internalIterator
pointIter internalIterator
// Either readState or version is set, but not both.
readState *readState
version *version
// rangeKey holds iteration state specific to iteration over range keys.
// The range key field may be nil if the Iterator has never been configured
// to iterate over range keys. Its non-nilness cannot be used to determine
@@ -446,6 +447,11 @@ const (
seekPrefixGELastPositioningOp
seekGELastPositioningOp
seekLTLastPositioningOp
// internalNextOp is a special internal iterator positioning operation used
// by CanDeterministicallySingleDelete. It exists for enforcing requirements
// around calling CanDeterministicallySingleDelete at most once per external
// iterator position.
internalNextOp
// invalidatedLastPositionOp is similar to unknownLastPositionOp and the
// only reason to distinguish this is for the wider set of SeekGE
// optimizations we permit for the external iterator Iterator.forwardOnly
@@ -549,6 +555,21 @@ func (i *Iterator) findNextEntry(limit []byte) {
return
}
// If the user has configured a SkipPoint function, invoke it to see
// whether we should skip over the current user key.
if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(i.iterKey.UserKey) {
// NB: We could call nextUserKey, but in some cases the SkipPoint
// predicate function might be cheaper than nextUserKey's key copy
// and key comparison. This should be the case for MVCC suffix
// comparisons, for example. In the future, we could expand the
// SkipPoint interface to give the implementor more control over
// whether we skip over just the internal key, the user key, or even
// the key prefix.
i.stats.ForwardStepCount[InternalIterCall]++
i.iterKey, i.iterValue = i.iter.Next()
continue
}
switch key.Kind() {
case InternalKeyKindRangeKeySet:
// Save the current key.
@@ -569,7 +590,10 @@ func (i *Iterator) findNextEntry(limit []byte) {
i.iterValidityState = IterValid
return
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
// only simpler, but is also necessary for correctness due to
// InternalKeyKindSSTableInternalObsoleteBit.
i.nextUserKey()
continue
@@ -615,6 +639,13 @@ func (i *Iterator) findNextEntry(limit []byte) {
}
func (i *Iterator) nextPointCurrentUserKey() bool {
// If the user has configured a SkipPoint function and the current user key
// would be skipped by it, there's no need to step forward looking for a
// point key. If we were to find one, it should be skipped anyways.
if i.opts.SkipPoint != nil && i.opts.SkipPoint(i.key) {
return false
}
i.pos = iterPosCurForward
i.iterKey, i.iterValue = i.iter.Next()
@@ -632,7 +663,10 @@ func (i *Iterator) nextPointCurrentUserKey() bool {
i.err = base.CorruptionErrorf("pebble: unexpected range key set mid-user key")
return false
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
// only simpler, but is also necessary for correctness due to
// InternalKeyKindSSTableInternalObsoleteBit.
return false
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -773,52 +807,54 @@ func (i *Iterator) maybeSampleRead() {
func (i *Iterator) sampleRead() {
var topFile *manifest.FileMetadata
topLevel, numOverlappingLevels := numLevels, 0
if mi, ok := i.iter.(*mergingIter); ok {
if len(mi.levels) > 1 {
mi.ForEachLevelIter(func(li *levelIter) bool {
l := manifest.LevelToInt(li.level)
if f := li.iterFile; f != nil {
var containsKey bool
if i.pos == iterPosNext || i.pos == iterPosCurForward ||
i.pos == iterPosCurForwardPaused {
containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
i.pos == iterPosCurReversePaused {
containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
}
// Do nothing if the current key is not contained in f's
// bounds. We could seek the LevelIterator at this level
// to find the right file, but the performance impacts of
// doing that are significant enough to negate the benefits
// of read sampling in the first place. See the discussion
// at:
// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
if containsKey {
numOverlappingLevels++
if numOverlappingLevels >= 2 {
// Terminate the loop early if at least 2 overlapping levels are found.
return true
}
topLevel = l
topFile = f
}
mi := i.merging
if mi == nil {
return
}
if len(mi.levels) > 1 {
mi.ForEachLevelIter(func(li *levelIter) bool {
l := manifest.LevelToInt(li.level)
if f := li.iterFile; f != nil {
var containsKey bool
if i.pos == iterPosNext || i.pos == iterPosCurForward ||
i.pos == iterPosCurForwardPaused {
containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
i.pos == iterPosCurReversePaused {
containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
}
return false
})
}
// Do nothing if the current key is not contained in f's
// bounds. We could seek the LevelIterator at this level
// to find the right file, but the performance impacts of
// doing that are significant enough to negate the benefits
// of read sampling in the first place. See the discussion
// at:
// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
if containsKey {
numOverlappingLevels++
if numOverlappingLevels >= 2 {
// Terminate the loop early if at least 2 overlapping levels are found.
return true
}
topLevel = l
topFile = f
}
}
return false
})
}
if topFile == nil || topLevel >= numLevels {
return
}
if numOverlappingLevels >= 2 {
allowedSeeks := atomic.AddInt64(&topFile.Atomic.AllowedSeeks, -1)
allowedSeeks := topFile.AllowedSeeks.Add(-1)
if allowedSeeks == 0 {
// Since the compaction queue can handle duplicates, we can keep
// adding to the queue even once allowedSeeks hits 0.
// In fact, we NEED to keep adding to the queue, because the queue
// is small and evicts older and possibly useful compactions.
atomic.AddInt64(&topFile.Atomic.AllowedSeeks, topFile.InitAllowedSeeks)
topFile.AllowedSeeks.Add(topFile.InitAllowedSeeks)
read := readCompaction{
start: topFile.SmallestPointKey.UserKey,
@@ -902,6 +938,26 @@ func (i *Iterator) findPrevEntry(limit []byte) {
}
}
// If the user has configured a SkipPoint function, invoke it to see
// whether we should skip over the current user key.
if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(key.UserKey) {
// NB: We could call prevUserKey, but in some cases the SkipPoint
// predicate function might be cheaper than prevUserKey's key copy
// and key comparison. This should be the case for MVCC suffix
// comparisons, for example. In the future, we could expand the
// SkipPoint interface to give the implementor more control over
// whether we skip over just the internal key, the user key, or even
// the key prefix.
i.stats.ReverseStepCount[InternalIterCall]++
i.iterKey, i.iterValue = i.iter.Prev()
if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
i.iterValidityState = IterAtLimit
i.pos = iterPosCurReversePaused
return
}
continue
}
switch key.Kind() {
case InternalKeyKindRangeKeySet:
// Range key start boundary markers are interleaved with the maximum
@@ -930,7 +986,7 @@ func (i *Iterator) findPrevEntry(limit []byte) {
// return the key even if the MERGE point key is deleted.
rangeKeyBoundary = true
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
i.value = LazyValue{}
i.iterValidityState = IterExhausted
valueMerger = nil
@@ -939,12 +995,12 @@ func (i *Iterator) findPrevEntry(limit []byte) {
// Compare with the limit. We could optimize by only checking when
// we step to the previous user key, but detecting that requires a
// comparison too. Note that this position may already passed a
// number of versions of this user key, but they are all deleted,
// so the fact that a subsequent Prev*() call will not see them is
// number of versions of this user key, but they are all deleted, so
// the fact that a subsequent Prev*() call will not see them is
// harmless. Also note that this is the only place in the loop,
// other than the firstLoopIter case above, where we could step
// to a different user key and start processing it for returning
// to the caller.
// other than the firstLoopIter and SkipPoint cases above, where we
// could step to a different user key and start processing it for
// returning to the caller.
if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
i.iterValidityState = IterAtLimit
i.pos = iterPosCurReversePaused
@@ -1093,9 +1149,13 @@ func (i *Iterator) mergeNext(key InternalKey, valueMerger ValueMerger) {
return
}
switch key.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
// We've hit a deletion tombstone. Return everything up to this
// point.
//
// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
// only simpler, but is also necessary for correctness due to
// InternalKeyKindSSTableInternalObsoleteBit.
return
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -1751,7 +1811,8 @@ func (i *Iterator) internalNextPrefix(currKeyPrefixLen int) {
i.iterKey, i.iterValue = i.iter.NextPrefix(i.prefixOrFullSeekKey)
if invariants.Enabled && i.iterKey != nil {
if iterKeyPrefixLen := i.split(i.iterKey.UserKey); i.cmp(i.iterKey.UserKey[:iterKeyPrefixLen], i.prefixOrFullSeekKey) < 0 {
panic("pebble: iter.NextPrefix did not advance beyond the current prefix")
panic(errors.AssertionFailedf("pebble: iter.NextPrefix did not advance beyond the current prefix: now at %q; expected to be geq %q",
i.iterKey, i.prefixOrFullSeekKey))
}
}
}
@@ -2120,6 +2181,9 @@ func (i *Iterator) Value() []byte {
// ValueAndErr returns the value, and any error encountered in extracting the value.
// REQUIRES: i.Error()==nil and HasPointAndRange() returns true for hasPoint.
//
// The caller should not modify the contents of the returned slice, and its
// contents may change on the next call to Next.
func (i *Iterator) ValueAndErr() ([]byte, error) {
val, callerOwned, err := i.value.Value(i.lazyValueBuf)
if err != nil {
@@ -2150,7 +2214,13 @@ func (i *Iterator) RangeKeys() []RangeKeyData {
// Valid returns true if the iterator is positioned at a valid key/value pair
// and false otherwise.
func (i *Iterator) Valid() bool {
return i.iterValidityState == IterValid && !i.requiresReposition
valid := i.iterValidityState == IterValid && !i.requiresReposition
if invariants.Enabled {
if err := i.Error(); valid && err != nil {
panic(errors.WithSecondaryError(errors.AssertionFailedf("pebble: iterator is valid with non-nil Error"), err))
}
}
return valid
}
// Error returns any accumulated error.
@@ -2215,6 +2285,10 @@ func (i *Iterator) Close() error {
i.readState = nil
}
if i.version != nil {
i.version.Unref()
}
for _, readers := range i.externalReaders {
for _, r := range readers {
err = firstError(err, r.Close())
@@ -2407,7 +2481,8 @@ func (i *Iterator) SetOptions(o *IterOptions) {
// If either options specify block property filters for an iterator stack,
// reconstruct it.
if i.pointIter != nil && (closeBoth || len(o.PointKeyFilters) > 0 || len(i.opts.PointKeyFilters) > 0 ||
o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil) {
o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil || o.SkipPoint != nil ||
i.opts.SkipPoint != nil) {
i.err = firstError(i.err, i.pointIter.Close())
i.pointIter = nil
}
@@ -2646,11 +2721,22 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
}
readState := i.readState
if readState == nil {
vers := i.version
if readState == nil && vers == nil {
return nil, errors.Errorf("cannot Clone a closed Iterator")
}
// i is already holding a ref, so there is no race with unref here.
readState.ref()
//
// TODO(bilal): If the underlying iterator was created on a snapshot, we could
// grab a reference to the current readState instead of reffing the original
// readState. This allows us to release references to some zombie sstables
// and memtables.
if readState != nil {
readState.ref()
}
if vers != nil {
vers.Ref()
}
// Bundle various structures under a single umbrella in order to allocate
// them together.
buf := iterAllocPool.Get().(*iterAlloc)
@@ -2662,6 +2748,7 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
merge: i.merge,
comparer: i.comparer,
readState: readState,
version: vers,
keyBuf: buf.keyBuf,
prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
boundsBuf: buf.boundsBuf,
@@ -2716,19 +2803,19 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
s.SafeString(",\n(internal-stats: ")
s.Printf("(block-bytes: (total %s, cached %s, read-time %s)), "+
"(points: (count %s, key-bytes %s, value-bytes %s, tombstoned %s))",
humanize.IEC.Uint64(stats.InternalStats.BlockBytes),
humanize.IEC.Uint64(stats.InternalStats.BlockBytesInCache),
humanize.Bytes.Uint64(stats.InternalStats.BlockBytes),
humanize.Bytes.Uint64(stats.InternalStats.BlockBytesInCache),
humanize.FormattedString(stats.InternalStats.BlockReadDuration.String()),
humanize.SI.Uint64(stats.InternalStats.PointCount),
humanize.SI.Uint64(stats.InternalStats.KeyBytes),
humanize.SI.Uint64(stats.InternalStats.ValueBytes),
humanize.SI.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones),
humanize.Count.Uint64(stats.InternalStats.PointCount),
humanize.Bytes.Uint64(stats.InternalStats.KeyBytes),
humanize.Bytes.Uint64(stats.InternalStats.ValueBytes),
humanize.Count.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones),
)
if stats.InternalStats.SeparatedPointValue.Count != 0 {
s.Printf(", (separated: (count %s, bytes %s, fetched %s)))",
humanize.SI.Uint64(stats.InternalStats.SeparatedPointValue.Count),
humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes),
humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched))
humanize.Count.Uint64(stats.InternalStats.SeparatedPointValue.Count),
humanize.Bytes.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes),
humanize.Bytes.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched))
} else {
s.Printf(")")
}
@@ -2741,3 +2828,169 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
stats.RangeKeyStats.SkippedPoints)
}
}
// CanDeterministicallySingleDelete takes a valid iterator and examines internal
// state to determine if a SingleDelete deleting Iterator.Key() would
// deterministically delete the key. CanDeterministicallySingleDelete requires
// the iterator to be oriented in the forward direction (eg, the last
// positioning operation must've been a First, a Seek[Prefix]GE, or a
// Next[Prefix][WithLimit]).
//
// This function does not change the external position of the iterator, and all
// positioning methods should behave the same as if it was never called. This
// function will only return a meaningful result the first time it's invoked at
// an iterator position. This function invalidates the iterator Value's memory,
// and the caller must not rely on the memory safety of the previous Iterator
// position.
//
// If CanDeterministicallySingleDelete returns true AND the key at the iterator
// position is not modified between the creation of the Iterator and the commit
// of a batch containing a SingleDelete over the key, then the caller can be
// assured that SingleDelete is equivalent to Delete on the local engine, but it
// may not be true on another engine that received the same writes and with
// logically equivalent state since this engine may have collapsed multiple SETs
// into one.
func CanDeterministicallySingleDelete(it *Iterator) (bool, error) {
// This function may only be called once per external iterator position. We
// can validate this by checking the last positioning operation.
if it.lastPositioningOp == internalNextOp {
return false, errors.New("pebble: CanDeterministicallySingleDelete called twice")
}
validity, kind := it.internalNext()
var shadowedBySingleDelete bool
for validity == internalNextValid {
switch kind {
case InternalKeyKindDelete, InternalKeyKindDeleteSized:
// A DEL or DELSIZED tombstone is okay. An internal key
// sequence like SINGLEDEL; SET; DEL; SET can be handled
// deterministically. If there are SETs further down, we
// don't care about them.
return true, nil
case InternalKeyKindSingleDelete:
// A SingleDelete is okay as long as when that SingleDelete was
// written, it was written deterministically (eg, with its own
// CanDeterministicallySingleDelete check). Validate that it was
// written deterministically. We'll allow one set to appear after
// the SingleDelete.
shadowedBySingleDelete = true
validity, kind = it.internalNext()
continue
case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
// If we observed a single delete, it's allowed to delete 1 key.
// We'll keep looping to validate that the internal keys beneath the
// already-written single delete are copacetic.
if shadowedBySingleDelete {
shadowedBySingleDelete = false
validity, kind = it.internalNext()
continue
}
// We encountered a shadowed SET, SETWITHDEL, MERGE. A SINGLEDEL
// that deleted the KV at the original iterator position could
// result in this key becoming visible.
return false, nil
case InternalKeyKindRangeDelete:
// RangeDeletes are handled by the merging iterator and should never
// be observed by the top-level Iterator.
panic(errors.AssertionFailedf("pebble: unexpected range delete"))
case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
// Range keys are interleaved at the maximal sequence number and
// should never be observed within a user key.
panic(errors.AssertionFailedf("pebble: unexpected range key"))
default:
panic(errors.AssertionFailedf("pebble: unexpected key kind: %s", errors.Safe(kind)))
}
}
if validity == internalNextError {
return false, it.Error()
}
return true, nil
}
// internalNextValidity enumerates the potential outcomes of a call to
// internalNext.
type internalNextValidity int8
const (
// internalNextError is returned by internalNext when an error occurred and
// the caller is responsible for checking iter.Error().
internalNextError internalNextValidity = iota
// internalNextExhausted is returned by internalNext when the next internal
// key is an internal key with a different user key than Iterator.Key().
internalNextExhausted
// internalNextValid is returned by internalNext when the internal next
// found a shadowed internal key with a user key equal to Iterator.Key().
internalNextValid
)
// internalNext advances internal Iterator state forward to expose the
// InternalKeyKind of the next internal key with a user key equal to Key().
//
// internalNext is a highly specialized operation and is unlikely to be
// generally useful. See Iterator.Next for how to reposition the iterator to the
// next key. internalNext requires the Iterator to be at a valid position in the
// forward direction (the last positioning operation must've been a First, a
// Seek[Prefix]GE, or a Next[Prefix][WithLimit] and Valid() must return true).
//
// internalNext, unlike all other Iterator methods, exposes internal LSM state.
// internalNext advances the Iterator's internal iterator to the next shadowed
// key with a user key equal to Key(). When a key is overwritten or deleted, its
// removal from the LSM occurs lazily as a part of compactions. internalNext
// allows the caller to see whether an obsolete internal key exists with the
// current Key(), and what it's key kind is. Note that the existence of an
// internal key is nondeterministic and dependent on internal LSM state. These
// semantics are unlikely to be applicable to almost all use cases.
//
// If internalNext finds a key that shares the same user key as Key(), it
// returns internalNextValid and the internal key's kind. If internalNext
// encounters an error, it returns internalNextError and the caller is expected
// to call Iterator.Error() to retrieve it. In all other circumstances,
// internalNext returns internalNextExhausted, indicating that there are no more
// additional internal keys with the user key Key().
//
// internalNext does not change the external position of the iterator, and a
// Next operation should behave the same as if internalNext was never called.
// internalNext does invalidate the iterator Value's memory, and the caller must
// not rely on the memory safety of the previous Iterator position.
func (i *Iterator) internalNext() (internalNextValidity, base.InternalKeyKind) {
i.stats.ForwardStepCount[InterfaceCall]++
if i.err != nil {
return internalNextError, base.InternalKeyKindInvalid
} else if i.iterValidityState != IterValid {
return internalNextExhausted, base.InternalKeyKindInvalid
}
i.lastPositioningOp = internalNextOp
switch i.pos {
case iterPosCurForward:
i.iterKey, i.iterValue = i.iter.Next()
if i.iterKey == nil {
// We check i.iter.Error() here and return an internalNextError enum
// variant so that the caller does not need to check i.iter.Error()
// in the common case that the next internal key has a new user key.
if i.err = i.iter.Error(); i.err != nil {
return internalNextError, base.InternalKeyKindInvalid
}
i.pos = iterPosNext
return internalNextExhausted, base.InternalKeyKindInvalid
} else if i.comparer.Equal(i.iterKey.UserKey, i.key) {
return internalNextValid, i.iterKey.Kind()
}
i.pos = iterPosNext
return internalNextExhausted, base.InternalKeyKindInvalid
case iterPosCurReverse, iterPosCurReversePaused, iterPosPrev:
i.err = errors.New("switching from reverse to forward via internalNext is prohibited")
i.iterValidityState = IterExhausted
return internalNextError, base.InternalKeyKindInvalid
case iterPosNext, iterPosCurForwardPaused:
// The previous method already moved onto the next user key. This is
// only possible if
// - the last positioning method was a call to internalNext, and we
// advanced to a new user key.
// - the previous non-internalNext iterator operation encountered a
// range key or merge, forcing an internal Next that found a new
// user key that's not equal to i.Iterator.Key().
return internalNextExhausted, base.InternalKeyKindInvalid
default:
panic("unreachable")
}
}

18
vendor/github.com/cockroachdb/pebble/level_checker.go generated vendored
View File

@@ -9,7 +9,6 @@ import (
"fmt"
"io"
"sort"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
@@ -166,7 +165,7 @@ func (m *simpleMergingIter) step() bool {
if m.valueMerger != nil {
// Ongoing series of MERGE records.
switch item.key.Kind() {
case InternalKeyKindSingleDelete, InternalKeyKindDelete:
case InternalKeyKindSingleDelete, InternalKeyKindDelete, InternalKeyKindDeleteSized:
var closer io.Closer
_, closer, m.err = m.valueMerger.Finish(true /* includesBase */)
if m.err == nil && closer != nil {
@@ -346,7 +345,7 @@ func iterateAndCheckTombstones(
type checkConfig struct {
logger Logger
cmp Compare
comparer *Comparer
readState *readState
newIters tableNewIters
seqNum uint64
@@ -355,6 +354,9 @@ type checkConfig struct {
formatKey base.FormatKey
}
// cmp is shorthand for comparer.Compare.
func (c *checkConfig) cmp(a, b []byte) int { return c.comparer.Compare(a, b) }
func checkRangeTombstones(c *checkConfig) error {
var level int
var tombstones []tombstoneWithLevel
@@ -380,7 +382,7 @@ func checkRangeTombstones(c *checkConfig) error {
atomicUnit, _ := expandToAtomicUnit(c.cmp, lf.Slice(), true /* disableIsCompacting */)
lower, upper := manifest.KeyRange(c.cmp, atomicUnit.Iter())
iterToClose, iter, err := c.newIters(
context.Background(), lf.FileMetadata, nil, internalIterOpts{})
context.Background(), lf.FileMetadata, &IterOptions{level: manifest.Level(lsmLevel)}, internalIterOpts{})
if err != nil {
return err
}
@@ -568,11 +570,11 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
// Determine the seqnum to read at after grabbing the read state (current and
// memtables) above.
seqNum := atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum)
seqNum := d.mu.versions.visibleSeqNum.Load()
checkConfig := &checkConfig{
logger: d.opts.Logger,
cmp: d.cmp,
comparer: d.opts.Comparer,
readState: readState,
newIters: d.newIters,
seqNum: seqNum,
@@ -640,7 +642,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
manifestIter := current.L0SublevelFiles[sublevel].Iter()
iterOpts := IterOptions{logger: c.logger}
li := &levelIter{}
li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
li.init(context.Background(), iterOpts, c.comparer, c.newIters, manifestIter,
manifest.L0Sublevel(sublevel), internalIterOpts{})
li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)
@@ -654,7 +656,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
iterOpts := IterOptions{logger: c.logger}
li := &levelIter{}
li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters,
li.init(context.Background(), iterOpts, c.comparer, c.newIters,
current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{})
li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)

62
vendor/github.com/cockroachdb/pebble/level_iter.go generated vendored
View File

@@ -36,9 +36,8 @@ type tableNewIters func(
// tableNewRangeDelIter takes a tableNewIters and returns a TableNewSpanIter
// for the rangedel iterator returned by tableNewIters.
func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter {
return func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
iter, rangeDelIter, err := newIters(
ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters}, internalIterOpts{})
return func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
iter, rangeDelIter, err := newIters(ctx, file, nil, internalIterOpts{})
if iter != nil {
_ = iter.Close()
}
@@ -51,6 +50,7 @@ func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.T
type internalIterOpts struct {
bytesIterated *uint64
bufferPool *sstable.BufferPool
stats *base.InternalIteratorStats
boundLimitedFilter sstable.BoundLimitedBlockPropertyFilter
}
@@ -82,10 +82,11 @@ type levelIter struct {
// short-lived (since they pin sstables), (b) plumbing a context into every
// method is very painful, (c) they do not (yet) respect context
// cancellation and are only used for tracing.
ctx context.Context
logger Logger
cmp Compare
split Split
ctx context.Context
logger Logger
comparer *Comparer
cmp Compare
split Split
// The lower/upper bounds for iteration as specified at creation or the most
// recent call to SetBounds.
lower []byte
@@ -200,6 +201,11 @@ type levelIter struct {
// cache when constructing new table iterators.
internalOpts internalIterOpts
// Scratch space for the obsolete keys filter, when there are no other block
// property filters specified. See the performance note where
// IterOptions.PointKeyFilters is declared.
filtersBuf [1]BlockPropertyFilter
// Disable invariant checks even if they are otherwise enabled. Used by tests
// which construct "impossible" situations (e.g. seeking to a key before the
// lower bound).
@@ -236,24 +242,22 @@ var _ base.InternalIterator = (*levelIter)(nil)
// parameter if the caller is never going to call SeekPrefixGE.
func newLevelIter(
opts IterOptions,
cmp Compare,
split Split,
comparer *Comparer,
newIters tableNewIters,
files manifest.LevelIterator,
level manifest.Level,
bytesIterated *uint64,
internalOpts internalIterOpts,
) *levelIter {
l := &levelIter{}
l.init(context.Background(), opts, cmp, split, newIters, files, level,
internalIterOpts{bytesIterated: bytesIterated})
l.init(context.Background(), opts, comparer, newIters, files, level,
internalOpts)
return l
}
func (l *levelIter) init(
ctx context.Context,
opts IterOptions,
cmp Compare,
split Split,
comparer *Comparer,
newIters tableNewIters,
files manifest.LevelIterator,
level manifest.Level,
@@ -267,10 +271,15 @@ func (l *levelIter) init(
l.upper = opts.UpperBound
l.tableOpts.TableFilter = opts.TableFilter
l.tableOpts.PointKeyFilters = opts.PointKeyFilters
if len(opts.PointKeyFilters) == 0 {
l.tableOpts.PointKeyFilters = l.filtersBuf[:0:1]
}
l.tableOpts.UseL6Filters = opts.UseL6Filters
l.tableOpts.level = l.level
l.cmp = cmp
l.split = split
l.tableOpts.snapshotForHideObsoletePoints = opts.snapshotForHideObsoletePoints
l.comparer = comparer
l.cmp = comparer.Compare
l.split = comparer.Split
l.iterFile = nil
l.newIters = newIters
l.files = files
@@ -788,7 +797,7 @@ func (l *levelIter) SeekPrefixGE(
}
return l.verify(l.largestBoundary, base.LazyValue{})
}
// It is possible that we are here because bloom filter matching failed. In
// It is possible that we are here because bloom filter matching failed. In
// that case it is likely that all keys matching the prefix are wholly
// within the current file and cannot be in the subsequent file. In that
// case we don't want to go to the next file, since loading and seeking in
@@ -796,7 +805,16 @@ func (l *levelIter) SeekPrefixGE(
// next file will defeat the optimization for the next SeekPrefixGE that is
// called with flags.TrySeekUsingNext(), since for sparse key spaces it is
// likely that the next key will also be contained in the current file.
if n := l.split(l.iterFile.LargestPointKey.UserKey); l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
var n int
if l.split != nil {
// If the split function is specified, calculate the prefix length accordingly.
n = l.split(l.iterFile.LargestPointKey.UserKey)
} else {
// If the split function is not specified, the entire key is used as the
// prefix. This case can occur when getIter uses SeekPrefixGE.
n = len(l.iterFile.LargestPointKey.UserKey)
}
if l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
return nil, base.LazyValue{}
}
return l.verify(l.skipEmptyFileForward())
@@ -1066,7 +1084,9 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, base.LazyValue) {
if *l.rangeDelIterPtr != nil && l.filteredIter != nil &&
l.filteredIter.MaybeFilteredKeys() {
l.largestBoundary = &l.iterFile.Largest
l.boundaryContext.isIgnorableBoundaryKey = true
if l.boundaryContext != nil {
l.boundaryContext.isIgnorableBoundaryKey = true
}
return l.largestBoundary, base.LazyValue{}
}
}
@@ -1155,7 +1175,9 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, base.LazyValue) {
// the next file.
if *l.rangeDelIterPtr != nil && l.filteredIter != nil && l.filteredIter.MaybeFilteredKeys() {
l.smallestBoundary = &l.iterFile.Smallest
l.boundaryContext.isIgnorableBoundaryKey = true
if l.boundaryContext != nil {
l.boundaryContext.isIgnorableBoundaryKey = true
}
return l.smallestBoundary, base.LazyValue{}
}
}

10
vendor/github.com/cockroachdb/pebble/log_recycler.go generated vendored
View File

@@ -32,14 +32,14 @@ type logRecycler struct {
// the log file should not be deleted (i.e. the log is being recycled), and
// false otherwise.
func (r *logRecycler) add(logInfo fileInfo) bool {
if logInfo.fileNum < r.minRecycleLogNum {
if logInfo.fileNum.FileNum() < r.minRecycleLogNum {
return false
}
r.mu.Lock()
defer r.mu.Unlock()
if logInfo.fileNum <= r.mu.maxLogNum {
if logInfo.fileNum.FileNum() <= r.mu.maxLogNum {
// The log file number was already considered for recycling. Don't consider
// it again. This avoids a race between adding the same log file for
// recycling multiple times, and removing the log file for actual
@@ -49,7 +49,7 @@ func (r *logRecycler) add(logInfo fileInfo) bool {
// shouldn't be deleted.
return true
}
r.mu.maxLogNum = logInfo.fileNum
r.mu.maxLogNum = logInfo.fileNum.FileNum()
if len(r.mu.logs) >= r.limit {
return false
}
@@ -89,7 +89,7 @@ func (r *logRecycler) pop(logNum FileNum) error {
if len(r.mu.logs) == 0 {
return errors.New("pebble: log recycler empty")
}
if r.mu.logs[0].fileNum != logNum {
if r.mu.logs[0].fileNum.FileNum() != logNum {
return errors.Errorf("pebble: log recycler invalid %d vs %d", errors.Safe(logNum), errors.Safe(fileInfoNums(r.mu.logs)))
}
r.mu.logs = r.mu.logs[1:]
@@ -102,7 +102,7 @@ func fileInfoNums(finfos []fileInfo) []FileNum {
}
nums := make([]FileNum, len(finfos))
for i := range finfos {
nums[i] = finfos[i].fileNum
nums[i] = finfos[i].fileNum.FileNum()
}
return nums
}

104
vendor/github.com/cockroachdb/pebble/mem_table.go generated vendored
View File

@@ -10,12 +10,12 @@ import (
"os"
"sync"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/arenaskl"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manual"
"github.com/cockroachdb/pebble/internal/rangedel"
"github.com/cockroachdb/pebble/internal/rangekey"
)
@@ -80,12 +80,21 @@ type memTable struct {
// inflight mutations that have reserved space in the memtable but not yet
// applied. The memtable cannot be flushed to disk until the writer refs
// drops to zero.
writerRefs int32
writerRefs atomic.Int32
tombstones keySpanCache
rangeKeys keySpanCache
// The current logSeqNum at the time the memtable was created. This is
// guaranteed to be less than or equal to any seqnum stored in the memtable.
logSeqNum uint64
logSeqNum uint64
releaseAccountingReservation func()
}
func (m *memTable) free() {
if m != nil {
m.releaseAccountingReservation()
manual.Free(m.arenaBuf)
m.arenaBuf = nil
}
}
// memTableOptions holds configuration used when creating a memTable. All of
@@ -93,9 +102,10 @@ type memTable struct {
// which is used by tests.
type memTableOptions struct {
*Options
arenaBuf []byte
size int
logSeqNum uint64
arenaBuf []byte
size int
logSeqNum uint64
releaseAccountingReservation func()
}
func checkMemTable(obj interface{}) {
@@ -110,18 +120,24 @@ func checkMemTable(obj interface{}) {
// Options.MemTableSize is used instead.
func newMemTable(opts memTableOptions) *memTable {
opts.Options = opts.Options.EnsureDefaults()
if opts.size == 0 {
opts.size = opts.MemTableSize
}
m := new(memTable)
m.init(opts)
return m
}
m := &memTable{
cmp: opts.Comparer.Compare,
formatKey: opts.Comparer.FormatKey,
equal: opts.Comparer.Equal,
arenaBuf: opts.arenaBuf,
writerRefs: 1,
logSeqNum: opts.logSeqNum,
func (m *memTable) init(opts memTableOptions) {
if opts.size == 0 {
opts.size = int(opts.MemTableSize)
}
*m = memTable{
cmp: opts.Comparer.Compare,
formatKey: opts.Comparer.FormatKey,
equal: opts.Comparer.Equal,
arenaBuf: opts.arenaBuf,
logSeqNum: opts.logSeqNum,
releaseAccountingReservation: opts.releaseAccountingReservation,
}
m.writerRefs.Store(1)
m.tombstones = keySpanCache{
cmp: m.cmp,
formatKey: m.formatKey,
@@ -144,18 +160,18 @@ func newMemTable(opts memTableOptions) *memTable {
m.rangeDelSkl.Reset(arena, m.cmp)
m.rangeKeySkl.Reset(arena, m.cmp)
m.reserved = arena.Size()
return m
}
func (m *memTable) writerRef() {
switch v := atomic.AddInt32(&m.writerRefs, 1); {
switch v := m.writerRefs.Add(1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
}
func (m *memTable) writerUnref() bool {
switch v := atomic.AddInt32(&m.writerRefs, -1); {
// writerUnref drops a ref on the memtable. Returns true if this was the last ref.
func (m *memTable) writerUnref() (wasLastRef bool) {
switch v := m.writerRefs.Add(-1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
@@ -165,8 +181,9 @@ func (m *memTable) writerUnref() bool {
}
}
// readyForFlush is part of the flushable interface.
func (m *memTable) readyForFlush() bool {
return atomic.LoadInt32(&m.writerRefs) == 0
return m.writerRefs.Load() == 0
}
// Prepare reserves space for the batch in the memtable and references the
@@ -194,11 +211,13 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
var tombstoneCount, rangeKeyCount uint32
startSeqNum := seqNum
for r := batch.Reader(); ; seqNum++ {
kind, ukey, value, ok := r.Next()
kind, ukey, value, ok, err := r.Next()
if !ok {
if err != nil {
return err
}
break
}
var err error
ikey := base.MakeInternalKey(ukey, seqNum, kind)
switch kind {
case InternalKeyKindRangeDelete:
@@ -233,17 +252,19 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
return nil
}
// newIter returns an iterator that is unpositioned (Iterator.Valid() will
// return false). The iterator can be positioned via a call to SeekGE,
// SeekLT, First or Last.
// newIter is part of the flushable interface. It returns an iterator that is
// unpositioned (Iterator.Valid() will return false). The iterator can be
// positioned via a call to SeekGE, SeekLT, First or Last.
func (m *memTable) newIter(o *IterOptions) internalIterator {
return m.skl.NewIter(o.GetLowerBound(), o.GetUpperBound())
}
// newFlushIter is part of the flushable interface.
func (m *memTable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
return m.skl.NewFlushIter(bytesFlushed)
}
// newRangeDelIter is part of the flushable interface.
func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
tombstones := m.tombstones.get()
if tombstones == nil {
@@ -252,6 +273,7 @@ func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
return keyspan.NewIter(m.cmp, tombstones)
}
// newRangeKeyIter is part of the flushable interface.
func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
rangeKeys := m.rangeKeys.get()
if rangeKeys == nil {
@@ -260,13 +282,14 @@ func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
return keyspan.NewIter(m.cmp, rangeKeys)
}
// containsRangeKeys is part of the flushable interface.
func (m *memTable) containsRangeKeys() bool {
return atomic.LoadUint32(&m.rangeKeys.atomicCount) > 0
return m.rangeKeys.count.Load() > 0
}
func (m *memTable) availBytes() uint32 {
a := m.skl.Arena()
if atomic.LoadInt32(&m.writerRefs) == 1 {
if m.writerRefs.Load() == 1 {
// If there are no other concurrent apply operations, we can update the
// reserved bytes setting to accurately reflect how many bytes of been
// allocated vs the over-estimation present in memTableEntrySize.
@@ -275,10 +298,12 @@ func (m *memTable) availBytes() uint32 {
return a.Capacity() - m.reserved
}
// inuseBytes is part of the flushable interface.
func (m *memTable) inuseBytes() uint64 {
return uint64(m.skl.Size() - memTableEmptySize)
}
// totalBytes is part of the flushable interface.
func (m *memTable) totalBytes() uint64 {
return uint64(m.skl.Arena().Capacity())
}
@@ -355,8 +380,8 @@ func (f *keySpanFrags) get(
// invalidated whenever a key of the same kind is added to a memTable, and
// populated when empty when a span iterator of that key kind is created.
type keySpanCache struct {
atomicCount uint32
frags unsafe.Pointer
count atomic.Uint32
frags atomic.Pointer[keySpanFrags]
cmp Compare
formatKey base.FormatKey
constructSpan constructSpan
@@ -366,23 +391,20 @@ type keySpanCache struct {
// Invalidate the current set of cached spans, indicating the number of
// spans that were added.
func (c *keySpanCache) invalidate(count uint32) {
newCount := atomic.AddUint32(&c.atomicCount, count)
newCount := c.count.Add(count)
var frags *keySpanFrags
for {
oldPtr := atomic.LoadPointer(&c.frags)
if oldPtr != nil {
oldFrags := (*keySpanFrags)(oldPtr)
if oldFrags.count >= newCount {
// Someone else invalidated the cache before us and their invalidation
// subsumes ours.
break
}
oldFrags := c.frags.Load()
if oldFrags != nil && oldFrags.count >= newCount {
// Someone else invalidated the cache before us and their invalidation
// subsumes ours.
break
}
if frags == nil {
frags = &keySpanFrags{count: newCount}
}
if atomic.CompareAndSwapPointer(&c.frags, oldPtr, unsafe.Pointer(frags)) {
if c.frags.CompareAndSwap(oldFrags, frags) {
// We successfully invalidated the cache.
break
}
@@ -391,7 +413,7 @@ func (c *keySpanCache) invalidate(count uint32) {
}
func (c *keySpanCache) get() []keyspan.Span {
frags := (*keySpanFrags)(atomic.LoadPointer(&c.frags))
frags := c.frags.Load()
if frags == nil {
return nil
}

10
vendor/github.com/cockroachdb/pebble/merging_iter.go generated vendored
View File

@@ -27,6 +27,10 @@ type mergingIterLevel struct {
// iterKey and iterValue cache the current key and value iter are pointed at.
iterKey *InternalKey
iterValue base.LazyValue
// levelIter is non-nil if this level's iter is ultimately backed by a
// *levelIter. The handle in iter may have wrapped the levelIter with
// intermediary internalIterator implementations.
levelIter *levelIter
// levelIterBoundaryContext's fields are set when using levelIter, in order
// to surface sstable boundary keys and file-level context. See levelIter
@@ -1371,9 +1375,9 @@ func (m *mergingIter) DebugString() string {
}
func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) {
for _, iter := range m.levels {
if li, ok := iter.iter.(*levelIter); ok {
if done := fn(li); done {
for _, ml := range m.levels {
if ml.levelIter != nil {
if done := fn(ml.levelIter); done {
break
}
}

413
vendor/github.com/cockroachdb/pebble/metrics.go generated vendored
View File

@@ -6,12 +6,13 @@ package pebble
import (
"fmt"
"strings"
"math"
"time"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/redact"
@@ -28,13 +29,10 @@ type FilterMetrics = sstable.FilterMetrics
// comment in base.
type ThroughputMetric = base.ThroughputMetric
func formatCacheMetrics(w redact.SafePrinter, m *CacheMetrics, name redact.SafeString) {
w.Printf("%7s %9s %7s %6.1f%% (score == hit-rate)\n",
name,
humanize.SI.Int64(m.Count),
humanize.IEC.Int64(m.Size),
redact.Safe(hitRate(m.Hits, m.Misses)))
}
// SecondaryCacheMetrics holds metrics for the persistent secondary cache
// that caches commonly accessed blocks from blob storage on a local
// file system.
type SecondaryCacheMetrics = sharedcache.Metrics
// LevelMetrics holds per-level metrics such as the number of files and total
// size of the files, and compaction related metrics.
@@ -46,9 +44,14 @@ type LevelMetrics struct {
Sublevels int32
// The total number of files in the level.
NumFiles int64
// The total number of virtual sstables in the level.
NumVirtualFiles uint64
// The total size in bytes of the files in the level.
Size int64
// The level's compaction score.
// The total size of the virtual sstables in the level.
VirtualSize uint64
// The level's compaction score. This is the compensatedScoreRatio in the
// candidateLevelInfo.
Score float64
// The number of incoming bytes from other levels read during
// compactions. This excludes bytes moved and bytes ingested. For L0 this is
@@ -79,6 +82,18 @@ type LevelMetrics struct {
TablesIngested uint64
// The number of sstables moved to this level by a "move" compaction.
TablesMoved uint64
MultiLevel struct {
// BytesInTop are the total bytes in a multilevel compaction coming from the top level.
BytesInTop uint64
// BytesIn, exclusively for multiLevel compactions.
BytesIn uint64
// BytesRead, exclusively for multilevel compactions.
BytesRead uint64
}
// Additional contains misc additional metrics that are not always printed.
Additional struct {
// The sum of Properties.ValueBlocksSize for all the sstables in this
@@ -96,6 +111,8 @@ type LevelMetrics struct {
// Add updates the counter metrics for the level.
func (m *LevelMetrics) Add(u *LevelMetrics) {
m.NumFiles += u.NumFiles
m.NumVirtualFiles += u.NumVirtualFiles
m.VirtualSize += u.VirtualSize
m.Size += u.Size
m.BytesIn += u.BytesIn
m.BytesIngested += u.BytesIngested
@@ -107,6 +124,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) {
m.TablesFlushed += u.TablesFlushed
m.TablesIngested += u.TablesIngested
m.TablesMoved += u.TablesMoved
m.MultiLevel.BytesInTop += u.MultiLevel.BytesInTop
m.MultiLevel.BytesRead += u.MultiLevel.BytesRead
m.MultiLevel.BytesIn += u.MultiLevel.BytesIn
m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks
m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks
m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize
@@ -121,32 +141,6 @@ func (m *LevelMetrics) WriteAmp() float64 {
return float64(m.BytesFlushed+m.BytesCompacted) / float64(m.BytesIn)
}
// format generates a string of the receiver's metrics, formatting it into the
// supplied buffer.
func (m *LevelMetrics) format(
w redact.SafePrinter, score redact.SafeValue, includeValueBlocksSize bool,
) {
w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f",
redact.Safe(m.NumFiles),
humanize.IEC.Int64(m.Size),
score,
humanize.IEC.Uint64(m.BytesIn),
humanize.IEC.Uint64(m.BytesIngested),
humanize.SI.Uint64(m.TablesIngested),
humanize.IEC.Uint64(m.BytesMoved),
humanize.SI.Uint64(m.TablesMoved),
humanize.IEC.Uint64(m.BytesFlushed+m.BytesCompacted),
humanize.SI.Uint64(m.TablesFlushed+m.TablesCompacted),
humanize.IEC.Uint64(m.BytesRead),
redact.Safe(m.Sublevels),
redact.Safe(m.WriteAmp()))
if includeValueBlocksSize {
w.Printf(" %7s\n", humanize.IEC.Uint64(m.Additional.ValueBlocksSize))
} else {
w.SafeString("\n")
}
}
// Metrics holds metrics for various subsystems of the DB such as the Cache,
// Compactions, WAL, and per-Level metrics.
//
@@ -158,14 +152,15 @@ type Metrics struct {
Compact struct {
// The total number of compactions, and per-compaction type counts.
Count int64
DefaultCount int64
DeleteOnlyCount int64
ElisionOnlyCount int64
MoveCount int64
ReadCount int64
RewriteCount int64
MultiLevelCount int64
Count int64
DefaultCount int64
DeleteOnlyCount int64
ElisionOnlyCount int64
MoveCount int64
ReadCount int64
RewriteCount int64
MultiLevelCount int64
CounterLevelCount int64
// An estimate of the number of bytes that need to be compacted for the LSM
// to reach a stable state.
EstimatedDebt uint64
@@ -179,6 +174,14 @@ type Metrics struct {
// compaction. Such files are compacted in a rewrite compaction
// when no other compactions are picked.
MarkedFiles int
// Duration records the cumulative duration of all compactions since the
// database was opened.
Duration time.Duration
}
Ingest struct {
// The total number of ingestions
Count uint64
}
Flush struct {
@@ -210,7 +213,10 @@ type Metrics struct {
// The count of memtables.
Count int64
// The number of bytes present in zombie memtables which are no longer
// referenced by the current DB state but are still in use by an iterator.
// referenced by the current DB state. An unbounded number of memtables
// may be zombie if they're still in use by an iterator. One additional
// memtable may be zombie if it's no longer in use and waiting to be
// recycled.
ZombieSize uint64
// The count of zombie memtables.
ZombieCount int64
@@ -222,6 +228,9 @@ type Metrics struct {
// The approximate count of internal tombstones (DEL, SINGLEDEL and
// RANGEDEL key kinds) within the database.
TombstoneCount uint64
// A cumulative total number of missized DELSIZED keys encountered by
// compactions since the database was opened.
MissizedTombstonesCount uint64
}
Snapshots struct {
@@ -250,12 +259,18 @@ type Metrics struct {
ZombieSize uint64
// The count of zombie tables.
ZombieCount int64
// The count of the backing sstables.
BackingTableCount uint64
// The sum of the sizes of the all of the backing sstables.
BackingTableSize uint64
}
TableCache CacheMetrics
// Count of the number of open sstable iterators.
TableIters int64
// Uptime is the total time since this DB was opened.
Uptime time.Duration
WAL struct {
// Number of live WAL files.
@@ -281,6 +296,8 @@ type Metrics struct {
record.LogWriterMetrics
}
SecondaryCacheMetrics SecondaryCacheMetrics
private struct {
optionsFileSize uint64
manifestFileSize uint64
@@ -294,6 +311,13 @@ var (
prometheus.LinearBuckets(0.0, float64(time.Microsecond*100), 50),
prometheus.ExponentialBucketsRange(float64(time.Millisecond*5), float64(10*time.Second), 50)...,
)
// SecondaryCacheIOBuckets exported to enable exporting from package pebble to
// enable exporting metrics with below buckets in CRDB.
SecondaryCacheIOBuckets = sharedcache.IOBuckets
// SecondaryCacheChannelWriteBuckets exported to enable exporting from package
// pebble to enable exporting metrics with below buckets in CRDB.
SecondaryCacheChannelWriteBuckets = sharedcache.ChannelWriteBuckets
)
// DiskSpaceUsage returns the total disk space used by the database in bytes,
@@ -313,12 +337,25 @@ func (m *Metrics) DiskSpaceUsage() uint64 {
return usageBytes
}
func (m *Metrics) levelSizes() [numLevels]int64 {
var sizes [numLevels]int64
for i := 0; i < len(sizes); i++ {
sizes[i] = m.Levels[i].Size
// NumVirtual is the number of virtual sstables in the latest version
// summed over every level in the lsm.
func (m *Metrics) NumVirtual() uint64 {
var n uint64
for _, level := range m.Levels {
n += level.NumVirtualFiles
}
return sizes
return n
}
// VirtualSize is the sum of the sizes of the virtual sstables in the
// latest version. BackingTableSize - VirtualSize gives an estimate for
// the space amplification caused by not compacting virtual sstables.
func (m *Metrics) VirtualSize() uint64 {
var size uint64
for _, level := range m.Levels {
size += level.VirtualSize
}
return size
}
// ReadAmp returns the current read amplification of the database.
@@ -349,59 +386,34 @@ func (m *Metrics) Total() LevelMetrics {
return total
}
const notApplicable = redact.SafeString("-")
func (m *Metrics) formatWAL(w redact.SafePrinter) {
var writeAmp float64
if m.WAL.BytesIn > 0 {
writeAmp = float64(m.WAL.BytesWritten) / float64(m.WAL.BytesIn)
}
w.Printf(" WAL %9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7.1f\n",
redact.Safe(m.WAL.Files),
humanize.Uint64(m.WAL.Size),
notApplicable,
humanize.Uint64(m.WAL.BytesIn),
notApplicable,
notApplicable,
notApplicable,
notApplicable,
humanize.Uint64(m.WAL.BytesWritten),
notApplicable,
notApplicable,
notApplicable,
redact.Safe(writeAmp))
}
// String pretty-prints the metrics, showing a line for the WAL, a line per-level, and
// a total:
// String pretty-prints the metrics as below:
//
// __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp
// WAL 1 28 B - 17 B - - - - 56 B - - - 3.3
// 0 1 770 B 0.25 28 B 0 B 0 0 B 0 770 B 1 0 B 1 27.5
// 1 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 2 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 3 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 4 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 5 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// 6 0 0 B - 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0
// total 1 770 B - 56 B 0 B 0 0 B 0 826 B 1 0 B 1 14.8
// flush 1 0 B 0 0 (ingest = ingested-as-flushable, move = tables-ingested)
// compact 0 0 B 0 B 0 (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)
// ctype 0 0 0 0 0 0 0 (default, delete, elision, move, read, rewrite, multi-level)
// memtbl 1 256 K
// zmemtbl 1 256 K
// ztbl 0 0 B
// bcache 4 697 B 0.0% (score == hit-rate)
// tcache 1 696 B 0.0% (score == hit-rate)
// snaps 0 - 0 (score == earliest seq num)
// titers 1
// filter - - 0.0% (score == utility)
//
// The WAL "in" metric is the size of the batches written to the WAL. The WAL
// "write" metric is the size of the physical data written to the WAL which
// includes record fragment overhead. Write amplification is computed as
// bytes-written / bytes-in, except for the total row where bytes-in is
// replaced with WAL-bytes-written + bytes-ingested.
// | | | | ingested | moved | written | | amp
// level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w
// ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------
// 0 | 101 102B 0B 0 | 103.0 | 104B | 112 104B | 113 106B | 221 217B | 107B | 1 2.1
// 1 | 201 202B 0B 0 | 203.0 | 204B | 212 204B | 213 206B | 421 417B | 207B | 2 2.0
// 2 | 301 302B 0B 0 | 303.0 | 304B | 312 304B | 313 306B | 621 617B | 307B | 3 2.0
// 3 | 401 402B 0B 0 | 403.0 | 404B | 412 404B | 413 406B | 821 817B | 407B | 4 2.0
// 4 | 501 502B 0B 0 | 503.0 | 504B | 512 504B | 513 506B | 1.0K 1017B | 507B | 5 2.0
// 5 | 601 602B 0B 0 | 603.0 | 604B | 612 604B | 613 606B | 1.2K 1.2KB | 607B | 6 2.0
// 6 | 701 702B 0B 0 | - | 704B | 712 704B | 713 706B | 1.4K 1.4KB | 707B | 7 2.0
// total | 2.8K 2.7KB 0B 0 | - | 2.8KB | 2.9K 2.8KB | 2.9K 2.8KB | 5.7K 8.4KB | 2.8KB | 28 3.0
// -------------------------------------------------------------------------------------------------------------------
// WAL: 22 files (24B) in: 25B written: 26B (4% overhead)
// Flushes: 8
// Compactions: 5 estimated debt: 6B in progress: 2 (7B)
// default: 27 delete: 28 elision: 29 move: 30 read: 31 rewrite: 32 multi-level: 33
// MemTables: 12 (11B) zombie: 14 (13B)
// Zombie tables: 16 (15B)
// Backing tables: 0 (0B)
// Block cache: 2 entries (1B) hit rate: 42.9%
// Table cache: 18 entries (17B) hit rate: 48.7%
// Secondary cache: 40 entries (40B) hit rate: 49.9%
// Snapshots: 4 earliest seq num: 1024
// Table iters: 21
// Filter utility: 47.4%
// Ingestions: 27 as flushable: 36 (34B in 35 tables)
func (m *Metrics) String() string {
return redact.StringWithoutMarkers(m)
}
@@ -422,29 +434,84 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
// width specifiers. When the issue is fixed, we can convert these to
// RedactableStrings. https://github.com/cockroachdb/redact/issues/17
haveValueBlocks := false
var valueBlocksHeading redact.SafeString
for level := 0; level < numLevels; level++ {
if m.Levels[level].Additional.ValueBlocksSize > 0 {
haveValueBlocks = true
valueBlocksHeading = "__val-bl"
break
multiExists := m.Compact.MultiLevelCount > 0
appendIfMulti := func(line redact.SafeString) {
if multiExists {
w.SafeString(line)
}
}
newline := func() {
w.SafeString("\n")
}
w.SafeString(" | | | | ingested | moved | written | | amp")
appendIfMulti(" | multilevel")
newline()
w.SafeString("level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w")
appendIfMulti(" | top in read")
newline()
w.SafeString("------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------")
appendIfMulti("-+------------------")
newline()
// formatRow prints out a row of the table.
formatRow := func(m *LevelMetrics, score float64) {
scoreStr := "-"
if !math.IsNaN(score) {
// Try to keep the string no longer than 5 characters.
switch {
case score < 99.995:
scoreStr = fmt.Sprintf("%.2f", score)
case score < 999.95:
scoreStr = fmt.Sprintf("%.1f", score)
default:
scoreStr = fmt.Sprintf("%.0f", score)
}
}
var wampStr string
if wamp := m.WriteAmp(); wamp > 99.5 {
wampStr = fmt.Sprintf("%.0f", wamp)
} else {
wampStr = fmt.Sprintf("%.1f", wamp)
}
w.Printf("| %5s %6s %6s %7s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s",
humanize.Count.Int64(m.NumFiles),
humanize.Bytes.Int64(m.Size),
humanize.Bytes.Uint64(m.Additional.ValueBlocksSize),
humanize.Count.Uint64(m.NumVirtualFiles),
redact.Safe(scoreStr),
humanize.Bytes.Uint64(m.BytesIn),
humanize.Count.Uint64(m.TablesIngested),
humanize.Bytes.Uint64(m.BytesIngested),
humanize.Count.Uint64(m.TablesMoved),
humanize.Bytes.Uint64(m.BytesMoved),
humanize.Count.Uint64(m.TablesFlushed+m.TablesCompacted),
humanize.Bytes.Uint64(m.BytesFlushed+m.BytesCompacted),
humanize.Bytes.Uint64(m.BytesRead),
redact.Safe(m.Sublevels),
redact.Safe(wampStr))
if multiExists {
w.Printf(" | %5s %5s %5s",
humanize.Bytes.Uint64(m.MultiLevel.BytesInTop),
humanize.Bytes.Uint64(m.MultiLevel.BytesIn),
humanize.Bytes.Uint64(m.MultiLevel.BytesRead))
}
newline()
}
var total LevelMetrics
w.Printf("__level_____count____size___score______in__ingest(sz_cnt)"+
"____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp%s\n", valueBlocksHeading)
m.formatWAL(w)
for level := 0; level < numLevels; level++ {
l := &m.Levels[level]
w.Printf("%7d ", redact.Safe(level))
w.Printf("%5d ", redact.Safe(level))
// Format the score.
var score redact.SafeValue = notApplicable
score := math.NaN()
if level < numLevels-1 {
score = redact.Safe(fmt.Sprintf("%0.2f", l.Score))
score = l.Score
}
l.format(w, score, haveValueBlocks)
formatRow(l, score)
total.Add(l)
total.Sublevels += l.Sublevels
}
@@ -454,57 +521,105 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
// the bytes written to the log and bytes written externally and then
// ingested.
total.BytesFlushed += total.BytesIn
w.SafeString(" total ")
total.format(w, notApplicable, haveValueBlocks)
w.SafeString("total ")
formatRow(&total, math.NaN())
w.Printf(" flush %9d %31s %7d %7d %s\n",
redact.Safe(m.Flush.Count),
humanize.IEC.Uint64(m.Flush.AsIngestBytes),
redact.Safe(m.Flush.AsIngestTableCount),
redact.Safe(m.Flush.AsIngestCount),
redact.SafeString(`(ingest = tables-ingested, move = ingested-as-flushable)`))
w.Printf("compact %9d %7s %7s %7d %s %s\n",
w.SafeString("-------------------------------------------------------------------------------------------------------------------")
appendIfMulti("--------------------")
newline()
w.Printf("WAL: %d files (%s) in: %s written: %s (%.0f%% overhead)\n",
redact.Safe(m.WAL.Files),
humanize.Bytes.Uint64(m.WAL.Size),
humanize.Bytes.Uint64(m.WAL.BytesIn),
humanize.Bytes.Uint64(m.WAL.BytesWritten),
redact.Safe(percent(int64(m.WAL.BytesWritten)-int64(m.WAL.BytesIn), int64(m.WAL.BytesIn))))
w.Printf("Flushes: %d\n", redact.Safe(m.Flush.Count))
w.Printf("Compactions: %d estimated debt: %s in progress: %d (%s)\n",
redact.Safe(m.Compact.Count),
humanize.IEC.Uint64(m.Compact.EstimatedDebt),
humanize.IEC.Int64(m.Compact.InProgressBytes),
humanize.Bytes.Uint64(m.Compact.EstimatedDebt),
redact.Safe(m.Compact.NumInProgress),
redact.SafeString(strings.Repeat(" ", 24)),
redact.SafeString(`(size == estimated-debt, score = in-progress-bytes, in = num-in-progress)`))
w.Printf(" ctype %9d %7d %7d %7d %7d %7d %7d %s\n",
humanize.Bytes.Int64(m.Compact.InProgressBytes))
w.Printf(" default: %d delete: %d elision: %d move: %d read: %d rewrite: %d multi-level: %d\n",
redact.Safe(m.Compact.DefaultCount),
redact.Safe(m.Compact.DeleteOnlyCount),
redact.Safe(m.Compact.ElisionOnlyCount),
redact.Safe(m.Compact.MoveCount),
redact.Safe(m.Compact.ReadCount),
redact.Safe(m.Compact.RewriteCount),
redact.Safe(m.Compact.MultiLevelCount),
redact.SafeString(`(default, delete, elision, move, read, rewrite, multi-level)`))
w.Printf(" memtbl %9d %7s\n",
redact.Safe(m.Compact.MultiLevelCount))
w.Printf("MemTables: %d (%s) zombie: %d (%s)\n",
redact.Safe(m.MemTable.Count),
humanize.IEC.Uint64(m.MemTable.Size))
w.Printf("zmemtbl %9d %7s\n",
humanize.Bytes.Uint64(m.MemTable.Size),
redact.Safe(m.MemTable.ZombieCount),
humanize.IEC.Uint64(m.MemTable.ZombieSize))
w.Printf(" ztbl %9d %7s\n",
humanize.Bytes.Uint64(m.MemTable.ZombieSize))
w.Printf("Zombie tables: %d (%s)\n",
redact.Safe(m.Table.ZombieCount),
humanize.IEC.Uint64(m.Table.ZombieSize))
formatCacheMetrics(w, &m.BlockCache, "bcache")
formatCacheMetrics(w, &m.TableCache, "tcache")
w.Printf(" snaps %9d %7s %7d (score == earliest seq num)\n",
humanize.Bytes.Uint64(m.Table.ZombieSize))
w.Printf("Backing tables: %d (%s)\n",
redact.Safe(m.Table.BackingTableCount),
humanize.Bytes.Uint64(m.Table.BackingTableSize))
w.Printf("Virtual tables: %d (%s)\n",
redact.Safe(m.NumVirtual()),
humanize.Bytes.Uint64(m.VirtualSize()))
formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
w.Printf("%s: %s entries (%s) hit rate: %.1f%%\n",
name,
humanize.Count.Int64(m.Count),
humanize.Bytes.Int64(m.Size),
redact.Safe(hitRate(m.Hits, m.Misses)))
}
formatCacheMetrics(&m.BlockCache, "Block cache")
formatCacheMetrics(&m.TableCache, "Table cache")
formatSharedCacheMetrics := func(w redact.SafePrinter, m *SecondaryCacheMetrics, name redact.SafeString) {
w.Printf("%s: %s entries (%s) hit rate: %.1f%%\n",
name,
humanize.Count.Int64(m.Count),
humanize.Bytes.Int64(m.Size),
redact.Safe(hitRate(m.ReadsWithFullHit, m.ReadsWithPartialHit+m.ReadsWithNoHit)))
}
formatSharedCacheMetrics(w, &m.SecondaryCacheMetrics, "Secondary cache")
w.Printf("Snapshots: %d earliest seq num: %d\n",
redact.Safe(m.Snapshots.Count),
notApplicable,
redact.Safe(m.Snapshots.EarliestSeqNum))
w.Printf(" titers %9d\n", redact.Safe(m.TableIters))
w.Printf(" filter %9s %7s %6.1f%% (score == utility)\n",
notApplicable,
notApplicable,
redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
w.Printf("Table iters: %d\n", redact.Safe(m.TableIters))
w.Printf("Filter utility: %.1f%%\n", redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
w.Printf("Ingestions: %d as flushable: %d (%s in %d tables)\n",
redact.Safe(m.Ingest.Count),
redact.Safe(m.Flush.AsIngestCount),
humanize.Bytes.Uint64(m.Flush.AsIngestBytes),
redact.Safe(m.Flush.AsIngestTableCount))
}
func hitRate(hits, misses int64) float64 {
sum := hits + misses
if sum == 0 {
return percent(hits, hits+misses)
}
func percent(numerator, denominator int64) float64 {
if denominator == 0 {
return 0
}
return 100 * float64(hits) / float64(sum)
return 100 * float64(numerator) / float64(denominator)
}
// StringForTests is identical to m.String() on 64-bit platforms. It is used to
// provide a platform-independent result for tests.
func (m *Metrics) StringForTests() string {
mCopy := *m
if math.MaxInt == math.MaxInt32 {
// This is the difference in Sizeof(sstable.Reader{})) between 64 and 32 bit
// platforms.
const tableCacheSizeAdjustment = 212
mCopy.TableCache.Size += mCopy.TableCache.Count * tableCacheSizeAdjustment
}
return redact.StringWithoutMarkers(&mCopy)
}

19
vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go generated vendored
View File

@@ -4,26 +4,31 @@
package objstorage
import "io"
import "context"
// NoopReadHandle can be used by Readable implementations that don't
// support read-ahead.
type NoopReadHandle struct {
io.ReaderAt
readable Readable
}
// MakeNoopReadHandle initializes a NoopReadHandle.
func MakeNoopReadHandle(r io.ReaderAt) NoopReadHandle {
return NoopReadHandle{ReaderAt: r}
func MakeNoopReadHandle(r Readable) NoopReadHandle {
return NoopReadHandle{readable: r}
}
var _ ReadHandle = (*NoopReadHandle)(nil)
// ReadAt is part of the ReadHandle interface.
func (h *NoopReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error {
return h.readable.ReadAt(ctx, p, off)
}
// Close is part of the ReadHandle interface.
func (*NoopReadHandle) Close() error { return nil }
// MaxReadahead is part of the ReadHandle interface.
func (*NoopReadHandle) MaxReadahead() {}
// SetupForCompaction is part of the ReadHandle interface.
func (*NoopReadHandle) SetupForCompaction() {}
// RecordCacheHit is part of the ReadHandle interface.
func (*NoopReadHandle) RecordCacheHit(offset, size int64) {}
func (*NoopReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {}

324
vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go generated vendored Normal file
View File

@@ -0,0 +1,324 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import (
"context"
"fmt"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
)
// Readable is the handle for an object that is open for reading.
type Readable interface {
// ReadAt reads len(p) bytes into p starting at offset off.
//
// Does not return partial results; if off + len(p) is past the end of the
// object, an error is returned.
//
// Clients of ReadAt can execute parallel ReadAt calls on the
// same Readable.
ReadAt(ctx context.Context, p []byte, off int64) error
Close() error
// Size returns the size of the object.
Size() int64
// NewReadHandle creates a read handle for ReadAt requests that are related
// and can benefit from optimizations like read-ahead.
//
// The ReadHandle must be closed before the Readable is closed.
//
// Multiple separate ReadHandles can be used.
NewReadHandle(ctx context.Context) ReadHandle
}
// ReadHandle is used to perform reads that are related and might benefit from
// optimizations like read-ahead.
type ReadHandle interface {
// ReadAt reads len(p) bytes into p starting at offset off.
//
// Does not return partial results; if off + len(p) is past the end of the
// object, an error is returned.
//
// Parallel ReadAt calls on the same ReadHandle are not allowed.
ReadAt(ctx context.Context, p []byte, off int64) error
Close() error
// SetupForCompaction informs the implementation that the read handle will
// be used to read data blocks for a compaction. The implementation can expect
// sequential reads, and can decide to not retain data in any caches.
SetupForCompaction()
// RecordCacheHit informs the implementation that we were able to retrieve a
// block from cache. This is useful for example when the implementation is
// trying to detect a sequential reading pattern.
RecordCacheHit(ctx context.Context, offset, size int64)
}
// Writable is the handle for an object that is open for writing.
// Either Finish or Abort must be called.
type Writable interface {
// Write writes len(p) bytes from p to the underlying object. The data is not
// guaranteed to be durable until Finish is called.
//
// Note that Write *is* allowed to modify the slice passed in, whether
// temporarily or permanently. Callers of Write need to take this into
// account.
Write(p []byte) error
// Finish completes the object and makes the data durable.
// No further calls are allowed after calling Finish.
Finish() error
// Abort gives up on finishing the object. There is no guarantee about whether
// the object exists after calling Abort.
// No further calls are allowed after calling Abort.
Abort()
}
// ObjectMetadata contains the metadata required to be able to access an object.
type ObjectMetadata struct {
DiskFileNum base.DiskFileNum
FileType base.FileType
// The fields below are only set if the object is on remote storage.
Remote struct {
// CreatorID identifies the DB instance that originally created the object.
//
// Only used when CustomObjectName is not set.
CreatorID CreatorID
// CreatorFileNum is the identifier for the object within the context of the
// DB instance that originally created the object.
//
// Only used when CustomObjectName is not set.
CreatorFileNum base.DiskFileNum
// CustomObjectName (if it is set) overrides the object name that is normally
// derived from the CreatorID and CreatorFileNum.
CustomObjectName string
// CleanupMethod indicates the method for cleaning up unused shared objects.
CleanupMethod SharedCleanupMethod
// Locator identifies the remote.Storage implementation for this object.
Locator remote.Locator
// Storage is the remote.Storage object corresponding to the Locator. Used
// to avoid lookups in hot paths.
Storage remote.Storage
}
}
// IsRemote returns true if the object is on remote storage.
func (meta *ObjectMetadata) IsRemote() bool {
return meta.IsShared() || meta.IsExternal()
}
// IsExternal returns true if the object is on remote storage but is not owned
// by any Pebble instances in the cluster.
func (meta *ObjectMetadata) IsExternal() bool {
return meta.Remote.CustomObjectName != ""
}
// IsShared returns true if the object is on remote storage and is owned by a
// Pebble instance in the cluster (potentially shared between multiple
// instances).
func (meta *ObjectMetadata) IsShared() bool {
return meta.Remote.CreatorID.IsSet()
}
// AssertValid checks that the metadata is sane.
func (meta *ObjectMetadata) AssertValid() {
if !meta.IsRemote() {
// Verify all Remote fields are empty.
if meta.Remote != (ObjectMetadata{}).Remote {
panic(errors.AssertionFailedf("meta.Remote not empty: %#v", meta.Remote))
}
} else {
if meta.Remote.CustomObjectName != "" {
if meta.Remote.CreatorID == 0 {
panic(errors.AssertionFailedf("CreatorID not set"))
}
if meta.Remote.CreatorFileNum == base.FileNum(0).DiskFileNum() {
panic(errors.AssertionFailedf("CreatorFileNum not set"))
}
}
if meta.Remote.CleanupMethod != SharedNoCleanup && meta.Remote.CleanupMethod != SharedRefTracking {
panic(errors.AssertionFailedf("invalid CleanupMethod %d", meta.Remote.CleanupMethod))
}
if meta.Remote.Storage == nil {
panic(errors.AssertionFailedf("Storage not set"))
}
}
}
// CreatorID identifies the DB instance that originally created a shared object.
// This ID is incorporated in backing object names.
// Must be non-zero.
type CreatorID uint64
// IsSet returns true if the CreatorID is not zero.
func (c CreatorID) IsSet() bool { return c != 0 }
func (c CreatorID) String() string { return fmt.Sprintf("%d", c) }
// SharedCleanupMethod indicates the method for cleaning up unused shared objects.
type SharedCleanupMethod uint8
const (
// SharedRefTracking is used for shared objects for which objstorage providers
// keep track of references via reference marker objects.
SharedRefTracking SharedCleanupMethod = iota
// SharedNoCleanup is used for remote objects that are managed externally; the
// objstorage provider never deletes such objects.
SharedNoCleanup
)
// OpenOptions contains optional arguments for OpenForReading.
type OpenOptions struct {
// MustExist triggers a fatal error if the file does not exist. The fatal
// error message contains extra information helpful for debugging.
MustExist bool
}
// CreateOptions contains optional arguments for Create.
type CreateOptions struct {
// PreferSharedStorage causes the object to be created on shared storage if
// the provider has shared storage configured.
PreferSharedStorage bool
// SharedCleanupMethod is used for the object when it is created on shared storage.
// The default (zero) value is SharedRefTracking.
SharedCleanupMethod SharedCleanupMethod
}
// Provider is a singleton object used to access and manage objects.
//
// An object is conceptually like a large immutable file. The main use of
// objects is for storing sstables; in the future it could also be used for blob
// storage.
//
// The Provider can only manage objects that it knows about - either objects
// created by the provider, or existing objects the Provider was informed about
// via AddObjects.
//
// Objects are currently backed by a vfs.File or a remote.Storage object.
type Provider interface {
// OpenForReading opens an existing object.
OpenForReading(
ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts OpenOptions,
) (Readable, error)
// Create creates a new object and opens it for writing.
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
Create(
ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts CreateOptions,
) (w Writable, meta ObjectMetadata, err error)
// Remove removes an object.
//
// The object is not guaranteed to be durably removed until Sync is called.
Remove(fileType base.FileType, FileNum base.DiskFileNum) error
// Sync flushes the metadata from creation or removal of objects since the last Sync.
// This includes objects that have been Created but for which
// Writable.Finish() has not yet been called.
Sync() error
// LinkOrCopyFromLocal creates a new object that is either a copy of a given
// local file or a hard link (if the new object is created on the same FS, and
// if the FS supports it).
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
LinkOrCopyFromLocal(
ctx context.Context,
srcFS vfs.FS,
srcFilePath string,
dstFileType base.FileType,
dstFileNum base.DiskFileNum,
opts CreateOptions,
) (ObjectMetadata, error)
// Lookup returns the metadata of an object that is already known to the Provider.
// Does not perform any I/O.
Lookup(fileType base.FileType, FileNum base.DiskFileNum) (ObjectMetadata, error)
// Path returns an internal, implementation-dependent path for the object. It is
// meant to be used for informational purposes (like logging).
Path(meta ObjectMetadata) string
// Size returns the size of the object.
Size(meta ObjectMetadata) (int64, error)
// List returns the objects currently known to the provider. Does not perform any I/O.
List() []ObjectMetadata
// SetCreatorID sets the CreatorID which is needed in order to use shared
// objects. Remote object usage is disabled until this method is called the
// first time. Once set, the Creator ID is persisted and cannot change.
//
// Cannot be called if shared storage is not configured for the provider.
SetCreatorID(creatorID CreatorID) error
// IsSharedForeign returns whether this object is owned by a different node.
IsSharedForeign(meta ObjectMetadata) bool
// RemoteObjectBacking encodes the remote object metadata for the given object.
RemoteObjectBacking(meta *ObjectMetadata) (RemoteObjectBackingHandle, error)
// CreateExternalObjectBacking creates a backing for an existing object with a
// custom object name. The object is considered to be managed outside of
// Pebble and will never be removed by Pebble.
CreateExternalObjectBacking(locator remote.Locator, objName string) (RemoteObjectBacking, error)
// AttachRemoteObjects registers existing remote objects with this provider.
AttachRemoteObjects(objs []RemoteObjectToAttach) ([]ObjectMetadata, error)
Close() error
// IsNotExistError indicates whether the error is known to report that a file or
// directory does not exist.
IsNotExistError(err error) bool
// Metrics returns metrics about objstorage. Currently, it only returns metrics
// about the shared cache.
Metrics() sharedcache.Metrics
}
// RemoteObjectBacking encodes the metadata necessary to incorporate a shared
// object into a different Pebble instance. The encoding is specific to a given
// Provider implementation.
type RemoteObjectBacking []byte
// RemoteObjectBackingHandle is a container for a RemoteObjectBacking which
// ensures that the backing stays valid. A backing can otherwise become invalid
// if this provider unrefs the shared object. The RemoteObjectBackingHandle
// delays any unref until Close.
type RemoteObjectBackingHandle interface {
// Get returns the backing. The backing is only guaranteed to be valid until
// Close is called (or until the Provider is closed). If Close was already
// called, returns an error.
Get() (RemoteObjectBacking, error)
Close()
}
// RemoteObjectToAttach contains the arguments needed to attach an existing remote object.
type RemoteObjectToAttach struct {
// FileNum is the file number that will be used to refer to this object (in
// the context of this instance).
FileNum base.DiskFileNum
FileType base.FileType
// Backing contains the metadata for the remote object backing (normally
// generated from a different instance, but using the same Provider
// implementation).
Backing RemoteObjectBacking
}

68
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,68 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objiotracing
import "github.com/cockroachdb/pebble/internal/base"
// OpType indicates the type of operation.
type OpType uint8
// OpType values.
const (
ReadOp OpType = iota
WriteOp
// RecordCacheHitOp happens when a read is satisfied from the block cache. See
// objstorage.ReadHandle.RecordCacheHit().
RecordCacheHitOp
// SetupForCompactionOp is a "meta operation" that configures a read handle
// for large sequential reads. See objstorage.ReadHandle.SetupForCompaction().
SetupForCompactionOp
)
// Reason indicates the higher-level context of the operation.
type Reason uint8
// Reason values.
const (
UnknownReason Reason = iota
ForFlush
ForCompaction
ForIngestion
// TODO(radu): add ForUserFacing.
)
// BlockType indicates the type of data block relevant to an operation.
type BlockType uint8
// BlockType values.
const (
UnknownBlock BlockType = iota
DataBlock
ValueBlock
FilterBlock
MetadataBlock
)
// Event is the on-disk format of a tracing event. It is exported here so that
// trace processing tools can use it by importing this package.
type Event struct {
// Event start time as a Unix time (see time.Time.StartUnixNano()).
// Note that recorded events are not necessarily ordered by time - this is
// because separate event "streams" use local buffers (for performance).
StartUnixNano int64
Op OpType
Reason Reason
BlockType BlockType
// LSM level plus one (with 0 indicating unknown level).
LevelPlusOne uint8
// Hardcoded padding so that struct layout doesn't depend on architecture.
_ uint32
FileNum base.FileNum
// HandleID is a unique identifier corresponding to an objstorage.ReadHandle;
// only set for read operations performed through a ReadHandle.
HandleID uint64
Offset int64
Size int64
}

59
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go generated vendored Normal file
View File

@@ -0,0 +1,59 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !pebble_obj_io_tracing
// +build !pebble_obj_io_tracing
package objiotracing
import (
"context"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// Enabled is used to short circuit tracing-related code in regular builds.
const Enabled = false
// Tracer manages the writing of object IO traces to files.
type Tracer struct{}
// Open creates a Tracer which generates trace files in the given directory.
// Each trace file contains a series of Events (as they are in memory).
func Open(fs vfs.FS, fsDir string) *Tracer {
return nil
}
// Close the tracer, flushing any remaining events.
func (*Tracer) Close() {}
// WrapReadable wraps an objstorage.Readable with one that generates tracing
// events.
func (*Tracer) WrapReadable(
ctx context.Context, r objstorage.Readable, fileNum base.DiskFileNum,
) objstorage.Readable {
return r
}
// WrapWritable wraps an objstorage.Writable with one that generates tracing
// events.
func (t *Tracer) WrapWritable(
ctx context.Context, w objstorage.Writable, fileNum base.DiskFileNum,
) objstorage.Writable {
return w
}
// WithReason creates a context that has an associated Reason (which ends up in
// traces created under that context).
func WithReason(ctx context.Context, reason Reason) context.Context { return ctx }
// WithBlockType creates a context that has an associated BlockType (which ends up in
// traces created under that context).
func WithBlockType(ctx context.Context, blockType BlockType) context.Context { return ctx }
// WithLevel creates a context that has an associated level (which ends up in
// traces created under that context).
func WithLevel(ctx context.Context, level int) context.Context { return ctx }

410
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go generated vendored Normal file
View File

@@ -0,0 +1,410 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build pebble_obj_io_tracing
// +build pebble_obj_io_tracing
package objiotracing
import (
"bufio"
"context"
"fmt"
"math/rand"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// Enabled is used to short circuit tracing-related code in regular builds.
const Enabled = true
// Tracer manages the writing of object IO traces to files.
//
// The tracer runs a background worker goroutine which receives trace event
// buffers over a channel and dumps them to IOTRACES- files. Wrapper
// implementations for Readable, ReadHandle, Writable are producers of traces;
// they maintain internal buffers of events which get flushed to the buffered
// channel when they get full. This allows for minimal synchronization per IO
// (as for most of these structures, an instance only allows a single IO at a
// time).
type Tracer struct {
fs vfs.FS
fsDir string
handleID atomic.Uint64
workerStopCh chan struct{}
workerDataCh chan eventBuf
workerWait sync.WaitGroup
}
// Open creates a Tracer which generates trace files in the given directory.
// Each trace file contains a series of Events (as they are in memory).
func Open(fs vfs.FS, fsDir string) *Tracer {
t := &Tracer{
fs: fs,
fsDir: fsDir,
workerStopCh: make(chan struct{}),
workerDataCh: make(chan eventBuf, channelBufSize),
}
t.handleID.Store(uint64(rand.NewSource(time.Now().UnixNano()).Int63()))
t.workerWait.Add(1)
go t.workerLoop()
return t
}
// Close the tracer, flushing any remaining events.
func (t *Tracer) Close() {
if t.workerStopCh == nil {
return
}
// Tell the worker to stop and wait for it to finish up.
close(t.workerStopCh)
t.workerWait.Wait()
t.workerStopCh = nil
}
// WrapWritable wraps an objstorage.Writable with one that generates tracing
// events.
func (t *Tracer) WrapWritable(
ctx context.Context, w objstorage.Writable, fileNum base.FileNum,
) objstorage.Writable {
return &writable{
w: w,
fileNum: fileNum,
g: makeEventGenerator(ctx, t),
}
}
type writable struct {
w objstorage.Writable
fileNum base.FileNum
curOffset int64
g eventGenerator
}
var _ objstorage.Writable = (*writable)(nil)
// Write is part of the objstorage.Writable interface.
func (w *writable) Write(p []byte) error {
w.g.add(context.Background(), Event{
Op: WriteOp,
FileNum: w.fileNum,
Offset: w.curOffset,
Size: int64(len(p)),
})
// If w.w.Write(p) returns an error, a new writable
// will be used, so even tho all of p may not have
// been written to the underlying "file", it is okay
// to add len(p) to curOffset.
w.curOffset += int64(len(p))
return w.w.Write(p)
}
// Finish is part of the objstorage.Writable interface.
func (w *writable) Finish() error {
w.g.flush()
return w.w.Finish()
}
// Abort is part of the objstorage.Writable interface.
func (w *writable) Abort() {
w.g.flush()
w.w.Abort()
}
// WrapReadable wraps an objstorage.Readable with one that generates tracing
// events.
func (t *Tracer) WrapReadable(
ctx context.Context, r objstorage.Readable, fileNum base.FileNum,
) objstorage.Readable {
res := &readable{
r: r,
fileNum: fileNum,
}
res.mu.g = makeEventGenerator(ctx, t)
return res
}
type readable struct {
r objstorage.Readable
fileNum base.FileNum
mu struct {
sync.Mutex
g eventGenerator
}
}
var _ objstorage.Readable = (*readable)(nil)
// ReadAt is part of the objstorage.Readable interface.
func (r *readable) ReadAt(ctx context.Context, v []byte, off int64) (n int, err error) {
r.mu.Lock()
r.mu.g.add(ctx, Event{
Op: ReadOp,
FileNum: r.fileNum,
Offset: off,
Size: int64(len(v)),
})
r.mu.Unlock()
return r.r.ReadAt(ctx, v, off)
}
// Close is part of the objstorage.Readable interface.
func (r *readable) Close() error {
r.mu.g.flush()
return r.r.Close()
}
// Size is part of the objstorage.Readable interface.
func (r *readable) Size() int64 {
return r.r.Size()
}
// NewReadHandle is part of the objstorage.Readable interface.
func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle {
// It's safe to get the tracer from the generator without the mutex since it never changes.
t := r.mu.g.t
return &readHandle{
rh: r.r.NewReadHandle(ctx),
fileNum: r.fileNum,
handleID: t.handleID.Add(1),
g: makeEventGenerator(ctx, t),
}
}
type readHandle struct {
rh objstorage.ReadHandle
fileNum base.FileNum
handleID uint64
g eventGenerator
}
var _ objstorage.ReadHandle = (*readHandle)(nil)
// ReadAt is part of the objstorage.ReadHandle interface.
func (rh *readHandle) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) {
rh.g.add(ctx, Event{
Op: ReadOp,
FileNum: rh.fileNum,
HandleID: rh.handleID,
Offset: off,
Size: int64(len(p)),
})
return rh.rh.ReadAt(ctx, p, off)
}
// Close is part of the objstorage.ReadHandle interface.
func (rh *readHandle) Close() error {
rh.g.flush()
return rh.rh.Close()
}
// SetupForCompaction is part of the objstorage.ReadHandle interface.
func (rh *readHandle) SetupForCompaction() {
rh.g.add(context.Background(), Event{
Op: SetupForCompactionOp,
FileNum: rh.fileNum,
HandleID: rh.handleID,
})
rh.rh.SetupForCompaction()
}
// RecordCacheHit is part of the objstorage.ReadHandle interface.
func (rh *readHandle) RecordCacheHit(ctx context.Context, offset, size int64) {
rh.g.add(ctx, Event{
Op: RecordCacheHitOp,
FileNum: rh.fileNum,
HandleID: rh.handleID,
Offset: offset,
Size: size,
})
rh.rh.RecordCacheHit(ctx, offset, size)
}
type ctxInfo struct {
reason Reason
blockType BlockType
levelPlusOne uint8
}
func mergeCtxInfo(base, other ctxInfo) ctxInfo {
res := other
if res.reason == 0 {
res.reason = base.reason
}
if res.blockType == 0 {
res.blockType = base.blockType
}
if res.levelPlusOne == 0 {
res.levelPlusOne = base.levelPlusOne
}
return res
}
type ctxInfoKey struct{}
func withInfo(ctx context.Context, info ctxInfo) context.Context {
return context.WithValue(ctx, ctxInfoKey{}, info)
}
func infoFromCtx(ctx context.Context) ctxInfo {
res := ctx.Value(ctxInfoKey{})
if res == nil {
return ctxInfo{}
}
return res.(ctxInfo)
}
// WithReason creates a context that has an associated Reason (which ends up in
// traces created under that context).
func WithReason(ctx context.Context, reason Reason) context.Context {
info := infoFromCtx(ctx)
info.reason = reason
return withInfo(ctx, info)
}
// WithBlockType creates a context that has an associated BlockType (which ends up in
// traces created under that context).
func WithBlockType(ctx context.Context, blockType BlockType) context.Context {
info := infoFromCtx(ctx)
info.blockType = blockType
return withInfo(ctx, info)
}
// WithLevel creates a context that has an associated level (which ends up in
// traces created under that context).
func WithLevel(ctx context.Context, level int) context.Context {
info := infoFromCtx(ctx)
info.levelPlusOne = uint8(level) + 1
return withInfo(ctx, info)
}
const (
eventSize = int(unsafe.Sizeof(Event{}))
targetEntriesPerFile = 256 * 1024 * 1024 / eventSize // 256MB files
eventsPerBuf = 16
channelBufSize = 512 * 1024 / eventsPerBuf // 512K events.
bytesPerFileSync = 128 * 1024
)
type eventBuf struct {
events [eventsPerBuf]Event
num int
}
type eventGenerator struct {
t *Tracer
baseCtxInfo ctxInfo
buf eventBuf
}
func makeEventGenerator(ctx context.Context, t *Tracer) eventGenerator {
return eventGenerator{
t: t,
baseCtxInfo: infoFromCtx(ctx),
}
}
func (g *eventGenerator) flush() {
if g.buf.num > 0 {
g.t.workerDataCh <- g.buf
g.buf.num = 0
}
}
func (g *eventGenerator) add(ctx context.Context, e Event) {
e.StartUnixNano = time.Now().UnixNano()
info := infoFromCtx(ctx)
info = mergeCtxInfo(g.baseCtxInfo, info)
e.Reason = info.reason
e.BlockType = info.blockType
e.LevelPlusOne = info.levelPlusOne
if g.buf.num == eventsPerBuf {
g.flush()
}
g.buf.events[g.buf.num] = e
g.buf.num++
}
type workerState struct {
curFile vfs.File
curBW *bufio.Writer
numEntriesInFile int
}
func (t *Tracer) workerLoop() {
defer t.workerWait.Done()
stopCh := t.workerStopCh
dataCh := t.workerDataCh
var state workerState
t.workerNewFile(&state)
for {
select {
case <-stopCh:
close(dataCh)
// Flush any remaining traces.
for data := range dataCh {
t.workerWriteTraces(&state, data)
}
t.workerCloseFile(&state)
return
case data := <-dataCh:
t.workerWriteTraces(&state, data)
}
}
}
func (t *Tracer) workerWriteTraces(state *workerState, data eventBuf) {
if state.numEntriesInFile >= targetEntriesPerFile {
t.workerCloseFile(state)
t.workerNewFile(state)
}
state.numEntriesInFile += data.num
p := unsafe.Pointer(&data.events[0])
b := unsafe.Slice((*byte)(p), eventSize*data.num)
if _, err := state.curBW.Write(b); err != nil {
panic(err)
}
}
func (t *Tracer) workerNewFile(state *workerState) {
filename := fmt.Sprintf("IOTRACES-%s", time.Now().UTC().Format(time.RFC3339Nano))
file, err := t.fs.Create(t.fs.PathJoin(t.fsDir, filename))
if err != nil {
panic(err)
}
file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
BytesPerSync: bytesPerFileSync,
})
state.curFile = file
state.curBW = bufio.NewWriter(file)
state.numEntriesInFile = 0
}
func (t *Tracer) workerCloseFile(state *workerState) {
if state.curFile != nil {
if err := state.curBW.Flush(); err != nil {
panic(err)
}
if err := state.curFile.Sync(); err != nil {
panic(err)
}
if err := state.curFile.Close(); err != nil {
panic(err)
}
state.curFile = nil
state.curBW = nil
}
}

522
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go generated vendored Normal file
View File

@@ -0,0 +1,522 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"io"
"os"
"sort"
"sync"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
)
// provider is the implementation of objstorage.Provider.
type provider struct {
st Settings
fsDir vfs.File
tracer *objiotracing.Tracer
remote remoteSubsystem
mu struct {
sync.RWMutex
remote struct {
// catalogBatch accumulates remote object creations and deletions until
// Sync is called.
catalogBatch remoteobjcat.Batch
storageObjects map[remote.Locator]remote.Storage
}
// localObjectsChanged is set if non-remote objects were created or deleted
// but Sync was not yet called.
localObjectsChanged bool
// knownObjects maintains information about objects that are known to the provider.
// It is initialized with the list of files in the manifest when we open a DB.
knownObjects map[base.DiskFileNum]objstorage.ObjectMetadata
// protectedObjects are objects that cannot be unreferenced because they
// have outstanding SharedObjectBackingHandles. The value is a count of outstanding handles
protectedObjects map[base.DiskFileNum]int
}
}
var _ objstorage.Provider = (*provider)(nil)
// Settings that must be specified when creating the provider.
type Settings struct {
Logger base.Logger
// Local filesystem configuration.
FS vfs.FS
FSDirName string
// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
//
// This is an optional optimization to avoid double listing on Open when the
// higher layer already has a listing. When nil, we obtain the listing on
// Open.
FSDirInitialListing []string
// Cleaner cleans obsolete files from the local filesystem.
//
// The default cleaner uses the DeleteCleaner.
FSCleaner base.Cleaner
// NoSyncOnClose decides whether the implementation will enforce a
// close-time synchronization (e.g., fdatasync() or sync_file_range())
// on files it writes to. Setting this to true removes the guarantee for a
// sync on close. Some implementations can still issue a non-blocking sync.
NoSyncOnClose bool
// BytesPerSync enables periodic syncing of files in order to smooth out
// writes to disk. This option does not provide any persistence guarantee, but
// is used to avoid latency spikes if the OS automatically decides to write
// out a large chunk of dirty filesystem buffers.
BytesPerSync int
// Fields here are set only if the provider is to support remote objects
// (experimental).
Remote struct {
StorageFactory remote.StorageFactory
// If CreateOnShared is non-zero, sstables are created on remote storage using
// the CreateOnSharedLocator (when the PreferSharedStorage create option is
// true).
CreateOnShared remote.CreateOnSharedStrategy
CreateOnSharedLocator remote.Locator
// CacheSizeBytes is the size of the on-disk block cache for objects
// on remote storage. If it is 0, no cache is used.
CacheSizeBytes int64
// CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used.
CacheBlockSize int
// ShardingBlockSize is the size of a shard block. The cache is split into contiguous
// ShardingBlockSize units. The units are distributed across multiple independent shards
// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
// policies operate at the level of shard, not whole cache. This is done to reduce lock
// contention.
//
// If ShardingBlockSize is 0, the default of 1 MB is used.
ShardingBlockSize int64
// The number of independent shards the cache leverages. Each shard is the same size,
// and a hash of filenum & offset map a read to a certain shard. If set to 0,
// 2*runtime.GOMAXPROCS is used as the shard count.
CacheShardCount int
// TODO(radu): allow the cache to live on another FS/location (e.g. to use
// instance-local SSD).
}
}
// DefaultSettings initializes default settings (with no remote storage),
// suitable for tests and tools.
func DefaultSettings(fs vfs.FS, dirName string) Settings {
return Settings{
Logger: base.DefaultLogger,
FS: fs,
FSDirName: dirName,
FSCleaner: base.DeleteCleaner{},
NoSyncOnClose: false,
BytesPerSync: 512 * 1024, // 512KB
}
}
// Open creates the provider.
func Open(settings Settings) (objstorage.Provider, error) {
// Note: we can't just `return open(settings)` because in an error case we
// would return (*provider)(nil) which is not objstorage.Provider(nil).
p, err := open(settings)
if err != nil {
return nil, err
}
return p, nil
}
func open(settings Settings) (p *provider, _ error) {
fsDir, err := settings.FS.OpenDir(settings.FSDirName)
if err != nil {
return nil, err
}
defer func() {
if p == nil {
fsDir.Close()
}
}()
p = &provider{
st: settings,
fsDir: fsDir,
}
p.mu.knownObjects = make(map[base.DiskFileNum]objstorage.ObjectMetadata)
p.mu.protectedObjects = make(map[base.DiskFileNum]int)
if objiotracing.Enabled {
p.tracer = objiotracing.Open(settings.FS, settings.FSDirName)
}
// Add local FS objects.
if err := p.vfsInit(); err != nil {
return nil, err
}
// Initialize remote subsystem (if configured) and add remote objects.
if err := p.remoteInit(); err != nil {
return nil, err
}
return p, nil
}
// Close is part of the objstorage.Provider interface.
func (p *provider) Close() error {
err := p.sharedClose()
if p.fsDir != nil {
err = firstError(err, p.fsDir.Close())
p.fsDir = nil
}
if objiotracing.Enabled {
if p.tracer != nil {
p.tracer.Close()
p.tracer = nil
}
}
return err
}
// OpenForReading opens an existing object.
func (p *provider) OpenForReading(
ctx context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
meta, err := p.Lookup(fileType, fileNum)
if err != nil {
if opts.MustExist {
p.st.Logger.Fatalf("%v", err)
}
return nil, err
}
var r objstorage.Readable
if !meta.IsRemote() {
r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts)
} else {
r, err = p.remoteOpenForReading(ctx, meta, opts)
if err != nil && p.isNotExistError(meta, err) {
// Wrap the error so that IsNotExistError functions properly.
err = errors.Mark(err, os.ErrNotExist)
}
}
if err != nil {
return nil, err
}
if objiotracing.Enabled {
r = p.tracer.WrapReadable(ctx, r, fileNum)
}
return r, nil
}
// Create creates a new object and opens it for writing.
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
func (p *provider) Create(
ctx context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
if opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone {
w, meta, err = p.sharedCreate(ctx, fileType, fileNum, p.st.Remote.CreateOnSharedLocator, opts)
} else {
w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
}
if err != nil {
err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum))
return nil, objstorage.ObjectMetadata{}, err
}
p.addMetadata(meta)
if objiotracing.Enabled {
w = p.tracer.WrapWritable(ctx, w, fileNum)
}
return w, meta, nil
}
// Remove removes an object.
//
// Note that if the object is remote, the object is only (conceptually) removed
// from this provider. If other providers have references on the remote object,
// it will not be removed.
//
// The object is not guaranteed to be durably removed until Sync is called.
func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) error {
meta, err := p.Lookup(fileType, fileNum)
if err != nil {
return err
}
if !meta.IsRemote() {
err = p.vfsRemove(fileType, fileNum)
} else {
// TODO(radu): implement remote object removal (i.e. deref).
err = p.sharedUnref(meta)
if err != nil && p.isNotExistError(meta, err) {
// Wrap the error so that IsNotExistError functions properly.
err = errors.Mark(err, os.ErrNotExist)
}
}
if err != nil && !p.IsNotExistError(err) {
// We want to be able to retry a Remove, so we keep the object in our list.
// TODO(radu): we should mark the object as "zombie" and not allow any other
// operations.
return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum))
}
p.removeMetadata(fileNum)
return err
}
func (p *provider) isNotExistError(meta objstorage.ObjectMetadata, err error) bool {
if meta.Remote.Storage != nil {
return meta.Remote.Storage.IsNotExistError(err)
}
return oserror.IsNotExist(err)
}
// IsNotExistError is part of the objstorage.Provider interface.
func (p *provider) IsNotExistError(err error) bool {
// We use errors.Mark(err, os.ErrNotExist) for not-exist errors coming from
// remote.Storage.
return oserror.IsNotExist(err)
}
// Sync flushes the metadata from creation or removal of objects since the last Sync.
func (p *provider) Sync() error {
if err := p.vfsSync(); err != nil {
return err
}
if err := p.sharedSync(); err != nil {
return err
}
return nil
}
// LinkOrCopyFromLocal creates a new object that is either a copy of a given
// local file or a hard link (if the new object is created on the same FS, and
// if the FS supports it).
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
func (p *provider) LinkOrCopyFromLocal(
ctx context.Context,
srcFS vfs.FS,
srcFilePath string,
dstFileType base.FileType,
dstFileNum base.DiskFileNum,
opts objstorage.CreateOptions,
) (objstorage.ObjectMetadata, error) {
shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone
if !shared && srcFS == p.st.FS {
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{
NoSyncOnClose: p.st.NoSyncOnClose,
BytesPerSync: p.st.BytesPerSync,
})
dstPath := p.vfsPath(dstFileType, dstFileNum)
if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
return objstorage.ObjectMetadata{}, err
}
meta := objstorage.ObjectMetadata{
DiskFileNum: dstFileNum,
FileType: dstFileType,
}
p.addMetadata(meta)
return meta, nil
}
// Create the object and copy the data.
w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts)
if err != nil {
return objstorage.ObjectMetadata{}, err
}
f, err := srcFS.Open(srcFilePath, vfs.SequentialReadsOption)
if err != nil {
return objstorage.ObjectMetadata{}, err
}
defer f.Close()
buf := make([]byte, 64*1024)
for {
n, readErr := f.Read(buf)
if readErr != nil && readErr != io.EOF {
w.Abort()
return objstorage.ObjectMetadata{}, readErr
}
if n > 0 {
if err := w.Write(buf[:n]); err != nil {
w.Abort()
return objstorage.ObjectMetadata{}, err
}
}
if readErr == io.EOF {
break
}
}
if err := w.Finish(); err != nil {
return objstorage.ObjectMetadata{}, err
}
return meta, nil
}
// Lookup is part of the objstorage.Provider interface.
func (p *provider) Lookup(
fileType base.FileType, fileNum base.DiskFileNum,
) (objstorage.ObjectMetadata, error) {
p.mu.RLock()
defer p.mu.RUnlock()
meta, ok := p.mu.knownObjects[fileNum]
if !ok {
return objstorage.ObjectMetadata{}, errors.Wrapf(
os.ErrNotExist,
"file %s (type %d) unknown to the objstorage provider",
errors.Safe(fileNum), errors.Safe(fileType),
)
}
if meta.FileType != fileType {
return objstorage.ObjectMetadata{}, errors.AssertionFailedf(
"file %s type mismatch (known type %d, expected type %d)",
errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType),
)
}
return meta, nil
}
// Path is part of the objstorage.Provider interface.
func (p *provider) Path(meta objstorage.ObjectMetadata) string {
if !meta.IsRemote() {
return p.vfsPath(meta.FileType, meta.DiskFileNum)
}
return p.remotePath(meta)
}
// Size returns the size of the object.
func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
if !meta.IsRemote() {
return p.vfsSize(meta.FileType, meta.DiskFileNum)
}
return p.remoteSize(meta)
}
// List is part of the objstorage.Provider interface.
func (p *provider) List() []objstorage.ObjectMetadata {
p.mu.RLock()
defer p.mu.RUnlock()
res := make([]objstorage.ObjectMetadata, 0, len(p.mu.knownObjects))
for _, meta := range p.mu.knownObjects {
res = append(res, meta)
}
sort.Slice(res, func(i, j int) bool {
return res[i].DiskFileNum.FileNum() < res[j].DiskFileNum.FileNum()
})
return res
}
// Metrics is part of the objstorage.Provider interface.
func (p *provider) Metrics() sharedcache.Metrics {
if p.remote.cache != nil {
return p.remote.cache.Metrics()
}
return sharedcache.Metrics{}
}
func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
if invariants.Enabled {
meta.AssertValid()
}
p.mu.Lock()
defer p.mu.Unlock()
p.mu.knownObjects[meta.DiskFileNum] = meta
if meta.IsRemote() {
p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: meta.DiskFileNum,
FileType: meta.FileType,
CreatorID: meta.Remote.CreatorID,
CreatorFileNum: meta.Remote.CreatorFileNum,
Locator: meta.Remote.Locator,
CleanupMethod: meta.Remote.CleanupMethod,
})
} else {
p.mu.localObjectsChanged = true
}
}
func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
p.mu.Lock()
defer p.mu.Unlock()
meta, ok := p.mu.knownObjects[fileNum]
if !ok {
return
}
delete(p.mu.knownObjects, fileNum)
if meta.IsRemote() {
p.mu.remote.catalogBatch.DeleteObject(fileNum)
} else {
p.mu.localObjectsChanged = true
}
}
// protectObject prevents the unreferencing of a remote object until
// unprotectObject is called.
func (p *provider) protectObject(fileNum base.DiskFileNum) {
p.mu.Lock()
defer p.mu.Unlock()
p.mu.protectedObjects[fileNum] = p.mu.protectedObjects[fileNum] + 1
}
func (p *provider) unprotectObject(fileNum base.DiskFileNum) {
p.mu.Lock()
defer p.mu.Unlock()
v := p.mu.protectedObjects[fileNum]
if invariants.Enabled && v == 0 {
panic("invalid protection count")
}
if v > 1 {
p.mu.protectedObjects[fileNum] = v - 1
} else {
delete(p.mu.protectedObjects, fileNum)
// TODO(radu): check if the object is still in knownObject; if not, unref it
// now.
}
}
func (p *provider) isProtected(fileNum base.DiskFileNum) bool {
p.mu.Lock()
defer p.mu.Unlock()
return p.mu.protectedObjects[fileNum] > 0
}

49
vendor/github.com/cockroachdb/pebble/objstorage/readahead.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go generated vendored
View File

@@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
package objstorageprovider
const (
// Constants for dynamic readahead of data blocks. Note that the size values
@@ -11,15 +11,15 @@ const (
minFileReadsForReadahead = 2
// TODO(bilal): Have the initial size value be a factor of the block size,
// as opposed to a hardcoded value.
initialReadaheadSize = 64 << 10 /* 64KB */
maxReadaheadSize = 256 << 10 /* 256KB */
initialReadaheadSize = 64 << 10 /* 64KB */
)
// readaheadState contains state variables related to readahead. Updated on
// file reads.
type readaheadState struct {
// Number of sequential reads.
numReads int64
numReads int64
maxReadaheadSize int64
// Size issued to the next call to Prefetch. Starts at or above
// initialReadaheadSize and grows exponentially until maxReadaheadSize.
size int64
@@ -32,22 +32,23 @@ type readaheadState struct {
limit int64
}
func makeReadaheadState() readaheadState {
func makeReadaheadState(maxReadaheadSize int64) readaheadState {
return readaheadState{
size: initialReadaheadSize,
size: initialReadaheadSize,
maxReadaheadSize: maxReadaheadSize,
}
}
func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
currentReadEnd := offset + blockLength
if rs.numReads >= minFileReadsForReadahead {
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
// This is a read that would have resulted in a readahead, had it
// not been a cache hit.
rs.limit = currentReadEnd
return
}
if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
// We read too far away from rs.limit to benefit from readahead in
// any scenario. Reset all variables.
rs.numReads = 1
@@ -61,7 +62,7 @@ func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
// readahead.
return
}
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
// Blocks are being read sequentially and would benefit from readahead
// down the line.
rs.numReads++
@@ -89,28 +90,28 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
// as well as the interval where a read would benefit from read ahead:
// [rs.limit, rs.limit + rs.size]
// We increase the latter interval to
// [rs.limit, rs.limit + maxReadaheadSize] to account for cases where
// [rs.limit, rs.limit + rs.maxReadaheadSize] to account for cases where
// readahead may not be beneficial with a small readahead size, but over
// time the readahead size would increase exponentially to make it
// beneficial.
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
// We are doing a read in the interval ahead of
// the last readahead range. In the diagrams below, ++++ is the last
// readahead range, ==== is the range represented by
// [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range
// [rs.limit, rs.limit + rs.maxReadaheadSize], and ---- is the range
// being read.
//
// rs.limit rs.limit + maxReadaheadSize
// rs.limit rs.limit + rs.maxReadaheadSize
// ++++++++++|===========================|
//
// |-------------|
// offset currentReadEnd
//
// This case is also possible, as are all cases with an overlap
// between [rs.limit, rs.limit + maxReadaheadSize] and [offset,
// between [rs.limit, rs.limit + rs.maxReadaheadSize] and [offset,
// currentReadEnd]:
//
// rs.limit rs.limit + maxReadaheadSize
// rs.limit rs.limit + rs.maxReadaheadSize
// ++++++++++|===========================|
//
// |-------------|
@@ -122,19 +123,19 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
rs.prevSize = rs.size
// Increase rs.size for the next read.
rs.size *= 2
if rs.size > maxReadaheadSize {
rs.size = maxReadaheadSize
if rs.size > rs.maxReadaheadSize {
rs.size = rs.maxReadaheadSize
}
return rs.prevSize
}
if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
// The above conditional has rs.limit > rs.prevSize to confirm that
// rs.limit - rs.prevSize would not underflow.
// We read too far away from rs.limit to benefit from readahead in
// any scenario. Reset all variables.
// The case where we read too far ahead:
//
// (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize)
// (rs.limit - rs.prevSize) (rs.limit) (rs.limit + rs.maxReadaheadSize)
// |+++++++++++++|=============|
//
// |-------------|
@@ -142,7 +143,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
//
// Or too far behind:
//
// (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize)
// (rs.limit - rs.prevSize) (rs.limit) (rs.limit + rs.maxReadaheadSize)
// |+++++++++++++|=============|
//
// |-------------|
@@ -160,7 +161,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
// readahead, but there's no reason to issue a readahead call at the
// moment.
//
// (rs.limit - rs.prevSize) (rs.limit + maxReadaheadSize)
// (rs.limit - rs.prevSize) (rs.limit + rs.maxReadaheadSize)
// |+++++++++++++|===============|
// (rs.limit)
//
@@ -170,11 +171,11 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
rs.numReads++
return 0
}
if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
// Blocks are being read sequentially and would benefit from readahead
// down the line.
//
// (rs.limit) (rs.limit + maxReadaheadSize)
// (rs.limit) (rs.limit + rs.maxReadaheadSize)
// |=============|
//
// |-------|
@@ -186,7 +187,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
// We read too far ahead of the last read, or before it. This indicates
// a random read, where readahead is not desirable. Reset all variables.
//
// (rs.limit - maxReadaheadSize) (rs.limit) (rs.limit + maxReadaheadSize)
// (rs.limit - rs.maxReadaheadSize) (rs.limit) (rs.limit + rs.maxReadaheadSize)
// |+++++++++++++|=============|
//
// |-------|

376
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go generated vendored Normal file
View File

@@ -0,0 +1,376 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"fmt"
"runtime"
"sync"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/remote"
)
// remoteSubsystem contains the provider fields related to remote storage.
// All fields remain unset if remote storage is not configured.
type remoteSubsystem struct {
catalog *remoteobjcat.Catalog
// catalogSyncMutex is used to correctly serialize two sharedSync operations.
// It must be acquired before the provider mutex.
catalogSyncMutex sync.Mutex
cache *sharedcache.Cache
// shared contains the fields relevant to shared objects, i.e. objects that
// are created by Pebble and potentially shared between Pebble instances.
shared struct {
// initialized guards access to the creatorID field.
initialized atomic.Bool
creatorID objstorage.CreatorID
initOnce sync.Once
// checkRefsOnOpen controls whether we check the ref marker file when opening
// an object. Normally this is true when invariants are enabled (but the provider
// test tweaks this field).
checkRefsOnOpen bool
}
}
// remoteInit initializes the remote object subsystem (if configured) and finds
// any remote objects.
func (p *provider) remoteInit() error {
if p.st.Remote.StorageFactory == nil {
return nil
}
catalog, contents, err := remoteobjcat.Open(p.st.FS, p.st.FSDirName)
if err != nil {
return errors.Wrapf(err, "pebble: could not open remote object catalog")
}
p.remote.catalog = catalog
p.remote.shared.checkRefsOnOpen = invariants.Enabled
// The creator ID may or may not be initialized yet.
if contents.CreatorID.IsSet() {
p.remote.initShared(contents.CreatorID)
p.st.Logger.Infof("remote storage configured; creatorID = %s", contents.CreatorID)
} else {
p.st.Logger.Infof("remote storage configured; no creatorID yet")
}
if p.st.Remote.CacheSizeBytes > 0 {
const defaultBlockSize = 32 * 1024
blockSize := p.st.Remote.CacheBlockSize
if blockSize == 0 {
blockSize = defaultBlockSize
}
const defaultShardingBlockSize = 1024 * 1024
shardingBlockSize := p.st.Remote.ShardingBlockSize
if shardingBlockSize == 0 {
shardingBlockSize = defaultShardingBlockSize
}
numShards := p.st.Remote.CacheShardCount
if numShards == 0 {
numShards = 2 * runtime.GOMAXPROCS(0)
}
p.remote.cache, err = sharedcache.Open(
p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, shardingBlockSize, p.st.Remote.CacheSizeBytes, numShards)
if err != nil {
return errors.Wrapf(err, "pebble: could not open remote object cache")
}
}
for _, meta := range contents.Objects {
o := objstorage.ObjectMetadata{
DiskFileNum: meta.FileNum,
FileType: meta.FileType,
}
o.Remote.CreatorID = meta.CreatorID
o.Remote.CreatorFileNum = meta.CreatorFileNum
o.Remote.CleanupMethod = meta.CleanupMethod
o.Remote.Locator = meta.Locator
o.Remote.CustomObjectName = meta.CustomObjectName
o.Remote.Storage, err = p.ensureStorageLocked(o.Remote.Locator)
if err != nil {
return errors.Wrapf(err, "creating remote.Storage object for locator '%s'", o.Remote.Locator)
}
if invariants.Enabled {
o.AssertValid()
}
p.mu.knownObjects[o.DiskFileNum] = o
}
return nil
}
// initShared initializes the creator ID, allowing use of shared objects.
func (ss *remoteSubsystem) initShared(creatorID objstorage.CreatorID) {
ss.shared.initOnce.Do(func() {
ss.shared.creatorID = creatorID
ss.shared.initialized.Store(true)
})
}
func (p *provider) sharedClose() error {
if p.st.Remote.StorageFactory == nil {
return nil
}
var err error
if p.remote.cache != nil {
err = p.remote.cache.Close()
p.remote.cache = nil
}
if p.remote.catalog != nil {
err = firstError(err, p.remote.catalog.Close())
p.remote.catalog = nil
}
return err
}
// SetCreatorID is part of the objstorage.Provider interface.
func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
if p.st.Remote.StorageFactory == nil {
return errors.AssertionFailedf("attempt to set CreatorID but remote storage not enabled")
}
// Note: this call is a cheap no-op if the creator ID was already set. This
// call also checks if we are trying to change the ID.
if err := p.remote.catalog.SetCreatorID(creatorID); err != nil {
return err
}
if !p.remote.shared.initialized.Load() {
p.st.Logger.Infof("remote storage creatorID set to %s", creatorID)
p.remote.initShared(creatorID)
}
return nil
}
// IsSharedForeign is part of the objstorage.Provider interface.
func (p *provider) IsSharedForeign(meta objstorage.ObjectMetadata) bool {
if !p.remote.shared.initialized.Load() {
return false
}
return meta.IsShared() && (meta.Remote.CreatorID != p.remote.shared.creatorID)
}
func (p *provider) remoteCheckInitialized() error {
if p.st.Remote.StorageFactory == nil {
return errors.Errorf("remote object support not configured")
}
return nil
}
func (p *provider) sharedCheckInitialized() error {
if err := p.remoteCheckInitialized(); err != nil {
return err
}
if !p.remote.shared.initialized.Load() {
return errors.Errorf("remote object support not available: remote creator ID not yet set")
}
return nil
}
func (p *provider) sharedSync() error {
// Serialize parallel sync operations. Note that ApplyBatch is already
// serialized internally, but we want to make sure they get called with
// batches in the right order.
p.remote.catalogSyncMutex.Lock()
defer p.remote.catalogSyncMutex.Unlock()
batch := func() remoteobjcat.Batch {
p.mu.Lock()
defer p.mu.Unlock()
res := p.mu.remote.catalogBatch.Copy()
p.mu.remote.catalogBatch.Reset()
return res
}()
if batch.IsEmpty() {
return nil
}
if err := p.remote.catalog.ApplyBatch(batch); err != nil {
// Put back the batch (for the next Sync), appending any operations that
// happened in the meantime.
p.mu.Lock()
defer p.mu.Unlock()
batch.Append(p.mu.remote.catalogBatch)
p.mu.remote.catalogBatch = batch
return err
}
return nil
}
func (p *provider) remotePath(meta objstorage.ObjectMetadata) string {
if meta.Remote.Locator != "" {
return fmt.Sprintf("remote-%s://%s", meta.Remote.Locator, remoteObjectName(meta))
}
return "remote://" + remoteObjectName(meta)
}
// sharedCreateRef creates a reference marker object.
func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
if err := p.sharedCheckInitialized(); err != nil {
return err
}
if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
return nil
}
refName := p.sharedObjectRefName(meta)
writer, err := meta.Remote.Storage.CreateObject(refName)
if err == nil {
// The object is empty, just close the writer.
err = writer.Close()
}
if err != nil {
return errors.Wrapf(err, "creating marker object %q", refName)
}
return nil
}
func (p *provider) sharedCreate(
_ context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
locator remote.Locator,
opts objstorage.CreateOptions,
) (objstorage.Writable, objstorage.ObjectMetadata, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, objstorage.ObjectMetadata{}, err
}
storage, err := p.ensureStorage(locator)
if err != nil {
return nil, objstorage.ObjectMetadata{}, err
}
meta := objstorage.ObjectMetadata{
DiskFileNum: fileNum,
FileType: fileType,
}
meta.Remote.CreatorID = p.remote.shared.creatorID
meta.Remote.CreatorFileNum = fileNum
meta.Remote.CleanupMethod = opts.SharedCleanupMethod
meta.Remote.Locator = locator
meta.Remote.Storage = storage
objName := remoteObjectName(meta)
writer, err := storage.CreateObject(objName)
if err != nil {
return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
}
return &sharedWritable{
p: p,
meta: meta,
storageWriter: writer,
}, meta, nil
}
func (p *provider) remoteOpenForReading(
ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
if err := p.remoteCheckInitialized(); err != nil {
return nil, err
}
// Verify we have a reference on this object; for performance reasons, we only
// do this in testing scenarios.
if p.remote.shared.checkRefsOnOpen && meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
if err := p.sharedCheckInitialized(); err != nil {
return nil, err
}
refName := p.sharedObjectRefName(meta)
if _, err := meta.Remote.Storage.Size(refName); err != nil {
if meta.Remote.Storage.IsNotExistError(err) {
if opts.MustExist {
p.st.Logger.Fatalf("marker object %q does not exist", refName)
// TODO(radu): maybe list references for the object.
}
return nil, errors.Errorf("marker object %q does not exist", refName)
}
return nil, errors.Wrapf(err, "checking marker object %q", refName)
}
}
objName := remoteObjectName(meta)
reader, size, err := meta.Remote.Storage.ReadObject(ctx, objName)
if err != nil {
if opts.MustExist && meta.Remote.Storage.IsNotExistError(err) {
p.st.Logger.Fatalf("object %q does not exist", objName)
// TODO(radu): maybe list references for the object.
}
return nil, err
}
return p.newRemoteReadable(reader, size, meta.DiskFileNum), nil
}
func (p *provider) remoteSize(meta objstorage.ObjectMetadata) (int64, error) {
if err := p.remoteCheckInitialized(); err != nil {
return 0, err
}
objName := remoteObjectName(meta)
return meta.Remote.Storage.Size(objName)
}
// sharedUnref implements object "removal" with the remote backend. The ref
// marker object is removed and the backing object is removed only if there are
// no other ref markers.
func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup {
// Never delete objects in this mode.
return nil
}
if p.isProtected(meta.DiskFileNum) {
// TODO(radu): we need a mechanism to unref the object when it becomes
// unprotected.
return nil
}
refName := p.sharedObjectRefName(meta)
// Tolerate a not-exists error.
if err := meta.Remote.Storage.Delete(refName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
return err
}
otherRefs, err := meta.Remote.Storage.List(sharedObjectRefPrefix(meta), "" /* delimiter */)
if err != nil {
return err
}
if len(otherRefs) == 0 {
objName := remoteObjectName(meta)
if err := meta.Remote.Storage.Delete(objName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
return err
}
}
return nil
}
// ensureStorageLocked populates the remote.Storage object for the given
// locator, if necessary. p.mu must be held.
func (p *provider) ensureStorageLocked(locator remote.Locator) (remote.Storage, error) {
if p.mu.remote.storageObjects == nil {
p.mu.remote.storageObjects = make(map[remote.Locator]remote.Storage)
}
if res, ok := p.mu.remote.storageObjects[locator]; ok {
return res, nil
}
res, err := p.st.Remote.StorageFactory.CreateStorage(locator)
if err != nil {
return nil, err
}
p.mu.remote.storageObjects[locator] = res
return res, nil
}
// ensureStorage populates the remote.Storage object for the given locator, if necessary.
func (p *provider) ensureStorage(locator remote.Locator) (remote.Storage, error) {
p.mu.Lock()
defer p.mu.Unlock()
return p.ensureStorageLocked(locator)
}

303
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go generated vendored Normal file
View File

@@ -0,0 +1,303 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"bytes"
"encoding/binary"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
"github.com/cockroachdb/pebble/objstorage/remote"
)
const (
tagCreatorID = 1
tagCreatorFileNum = 2
tagCleanupMethod = 3
// tagRefCheckID encodes the information for a ref marker that needs to be
// checked when attaching this object to another provider. This is set to the
// creator ID and FileNum for the provider that encodes the backing, and
// allows the "target" provider to check that the "source" provider kept its
// reference on the object alive.
tagRefCheckID = 4
// tagLocator encodes the remote.Locator; if absent the locator is "". It is
// followed by the locator string length and the locator string.
tagLocator = 5
// tagLocator encodes a custom object name (if present). It is followed by the
// custom name string length and the string.
tagCustomObjectName = 6
// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
// followed by the length of the data (so they can be skipped).
// Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if
// they are encountered by earlier code that doesn't know the tag.
tagNotSafeToIgnoreMask = 64
)
func (p *provider) encodeRemoteObjectBacking(
meta *objstorage.ObjectMetadata,
) (objstorage.RemoteObjectBacking, error) {
if !meta.IsRemote() {
return nil, errors.AssertionFailedf("object %s not on remote storage", meta.DiskFileNum)
}
buf := make([]byte, 0, binary.MaxVarintLen64*4)
buf = binary.AppendUvarint(buf, tagCreatorID)
buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorID))
// TODO(radu): encode file type as well?
buf = binary.AppendUvarint(buf, tagCreatorFileNum)
buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, tagCleanupMethod)
buf = binary.AppendUvarint(buf, uint64(meta.Remote.CleanupMethod))
if meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
buf = binary.AppendUvarint(buf, tagRefCheckID)
buf = binary.AppendUvarint(buf, uint64(p.remote.shared.creatorID))
buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum()))
}
if meta.Remote.Locator != "" {
buf = binary.AppendUvarint(buf, tagLocator)
buf = encodeString(buf, string(meta.Remote.Locator))
}
if meta.Remote.CustomObjectName != "" {
buf = binary.AppendUvarint(buf, tagCustomObjectName)
buf = encodeString(buf, meta.Remote.CustomObjectName)
}
return buf, nil
}
type remoteObjectBackingHandle struct {
backing objstorage.RemoteObjectBacking
fileNum base.DiskFileNum
p *provider
}
func (s *remoteObjectBackingHandle) Get() (objstorage.RemoteObjectBacking, error) {
if s.backing == nil {
return nil, errors.Errorf("RemoteObjectBackingHandle.Get() called after Close()")
}
return s.backing, nil
}
func (s *remoteObjectBackingHandle) Close() {
if s.backing != nil {
s.backing = nil
s.p.unprotectObject(s.fileNum)
}
}
var _ objstorage.RemoteObjectBackingHandle = (*remoteObjectBackingHandle)(nil)
// RemoteObjectBacking is part of the objstorage.Provider interface.
func (p *provider) RemoteObjectBacking(
meta *objstorage.ObjectMetadata,
) (objstorage.RemoteObjectBackingHandle, error) {
backing, err := p.encodeRemoteObjectBacking(meta)
if err != nil {
return nil, err
}
p.protectObject(meta.DiskFileNum)
return &remoteObjectBackingHandle{
backing: backing,
fileNum: meta.DiskFileNum,
p: p,
}, nil
}
// CreateExternalObjectBacking is part of the objstorage.Provider interface.
func (p *provider) CreateExternalObjectBacking(
locator remote.Locator, objName string,
) (objstorage.RemoteObjectBacking, error) {
var meta objstorage.ObjectMetadata
meta.Remote.Locator = locator
meta.Remote.CustomObjectName = objName
meta.Remote.CleanupMethod = objstorage.SharedNoCleanup
return p.encodeRemoteObjectBacking(&meta)
}
type decodedBacking struct {
meta objstorage.ObjectMetadata
// refToCheck is set only when meta.Remote.CleanupMethod is RefTracking
refToCheck struct {
creatorID objstorage.CreatorID
fileNum base.DiskFileNum
}
}
// decodeRemoteObjectBacking decodes the remote object metadata.
//
// Note that the meta.Remote.Storage field is not set.
func decodeRemoteObjectBacking(
fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.RemoteObjectBacking,
) (decodedBacking, error) {
var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64
var locator, customObjName string
br := bytes.NewReader(buf)
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return decodedBacking{}, err
}
switch tag {
case tagCreatorID:
creatorID, err = binary.ReadUvarint(br)
case tagCreatorFileNum:
creatorFileNum, err = binary.ReadUvarint(br)
case tagCleanupMethod:
cleanupMethod, err = binary.ReadUvarint(br)
case tagRefCheckID:
refCheckCreatorID, err = binary.ReadUvarint(br)
if err == nil {
refCheckFileNum, err = binary.ReadUvarint(br)
}
case tagLocator:
locator, err = decodeString(br)
case tagCustomObjectName:
customObjName, err = decodeString(br)
default:
// Ignore unknown tags, unless they're not safe to ignore.
if tag&tagNotSafeToIgnoreMask != 0 {
return decodedBacking{}, errors.Newf("unknown tag %d", tag)
}
var dataLen uint64
dataLen, err = binary.ReadUvarint(br)
if err == nil {
_, err = br.Seek(int64(dataLen), io.SeekCurrent)
}
}
if err != nil {
return decodedBacking{}, err
}
}
if customObjName == "" {
if creatorID == 0 {
return decodedBacking{}, errors.Newf("remote object backing missing creator ID")
}
if creatorFileNum == 0 {
return decodedBacking{}, errors.Newf("remote object backing missing creator file num")
}
}
var res decodedBacking
res.meta.DiskFileNum = fileNum
res.meta.FileType = fileType
res.meta.Remote.CreatorID = objstorage.CreatorID(creatorID)
res.meta.Remote.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
res.meta.Remote.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
if res.meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
if refCheckCreatorID == 0 || refCheckFileNum == 0 {
return decodedBacking{}, errors.Newf("remote object backing missing ref to check")
}
res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID)
res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum()
}
res.meta.Remote.Locator = remote.Locator(locator)
res.meta.Remote.CustomObjectName = customObjName
return res, nil
}
func encodeString(buf []byte, s string) []byte {
buf = binary.AppendUvarint(buf, uint64(len(s)))
buf = append(buf, []byte(s)...)
return buf
}
func decodeString(br io.ByteReader) (string, error) {
length, err := binary.ReadUvarint(br)
if err != nil || length == 0 {
return "", err
}
buf := make([]byte, length)
for i := range buf {
buf[i], err = br.ReadByte()
if err != nil {
return "", err
}
}
return string(buf), nil
}
// AttachRemoteObjects is part of the objstorage.Provider interface.
func (p *provider) AttachRemoteObjects(
objs []objstorage.RemoteObjectToAttach,
) ([]objstorage.ObjectMetadata, error) {
decoded := make([]decodedBacking, len(objs))
for i, o := range objs {
var err error
decoded[i], err = decodeRemoteObjectBacking(o.FileType, o.FileNum, o.Backing)
if err != nil {
return nil, err
}
decoded[i].meta.Remote.Storage, err = p.ensureStorage(decoded[i].meta.Remote.Locator)
if err != nil {
return nil, err
}
}
// Create the reference marker objects.
// TODO(radu): parallelize this.
for _, d := range decoded {
if d.meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
continue
}
if err := p.sharedCreateRef(d.meta); err != nil {
// TODO(radu): clean up references previously created in this loop.
return nil, err
}
// Check the "origin"'s reference.
refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum)
if _, err := d.meta.Remote.Storage.Size(refName); err != nil {
_ = p.sharedUnref(d.meta)
// TODO(radu): clean up references previously created in this loop.
if d.meta.Remote.Storage.IsNotExistError(err) {
return nil, errors.Errorf("origin marker object %q does not exist;"+
" object probably removed from the provider which created the backing", refName)
}
return nil, errors.Wrapf(err, "checking origin's marker object %s", refName)
}
}
func() {
p.mu.Lock()
defer p.mu.Unlock()
for _, d := range decoded {
p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
FileNum: d.meta.DiskFileNum,
FileType: d.meta.FileType,
CreatorID: d.meta.Remote.CreatorID,
CreatorFileNum: d.meta.Remote.CreatorFileNum,
CleanupMethod: d.meta.Remote.CleanupMethod,
Locator: d.meta.Remote.Locator,
})
}
}()
if err := p.sharedSync(); err != nil {
return nil, err
}
metas := make([]objstorage.ObjectMetadata, len(decoded))
for i, d := range decoded {
metas[i] = d.meta
}
p.mu.Lock()
defer p.mu.Unlock()
for _, meta := range metas {
p.mu.knownObjects[meta.DiskFileNum] = meta
}
return metas, nil
}

91
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go generated vendored Normal file
View File

@@ -0,0 +1,91 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
)
// remoteObjectName returns the name of an object on remote storage.
//
// For sstables, the format is: <hash>-<creator-id>-<file-num>.sst
// For example: 1a3f-2-000001.sst
func remoteObjectName(meta objstorage.ObjectMetadata) string {
if meta.Remote.CustomObjectName != "" {
return meta.Remote.CustomObjectName
}
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst",
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
)
}
panic("unknown FileType")
}
// sharedObjectRefName returns the name of the object's ref marker associated
// with a given referencing provider. This name is the object's name concatenated with
// ".ref.<ref-creator-id>.<local-file-num>".
//
// For example: 1a3f-2-000001.sst.ref.5.000008
func sharedObjectRefName(
meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum,
) string {
if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
panic("ref object used when ref tracking disabled")
}
if meta.Remote.CustomObjectName != "" {
return fmt.Sprintf(
"%s.ref.%d.%06d", meta.Remote.CustomObjectName, refCreatorID, refFileNum.FileNum(),
)
}
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst.ref.%d.%06d",
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
)
}
panic("unknown FileType")
}
func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
if meta.Remote.CustomObjectName != "" {
return meta.Remote.CustomObjectName + ".ref."
}
switch meta.FileType {
case base.FileTypeTable:
return fmt.Sprintf(
"%04x-%d-%06d.sst.ref.",
objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
)
}
panic("unknown FileType")
}
// sharedObjectRefName returns the name of the object's ref marker associated
// with this provider. This name is the object's name concatenated with
// ".ref.<creator-id>.<local-file-num>".
//
// For example: 1a3f-2-000001.sst.ref.5.000008
func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
panic("ref object used when ref tracking disabled")
}
return sharedObjectRefName(meta, p.remote.shared.creatorID, meta.DiskFileNum)
}
// objHash returns a 16-bit hash value derived from the creator ID and creator
// file num. We prepend this value to object names to ensure balanced
// partitioning with AWS (and likely other blob storage providers).
func objHash(meta objstorage.ObjectMetadata) uint16 {
const prime1 = 7459
const prime2 = 17539
return uint16(uint64(meta.Remote.CreatorID)*prime1 + uint64(meta.Remote.CreatorFileNum.FileNum())*prime2)
}

162
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go generated vendored Normal file
View File

@@ -0,0 +1,162 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"context"
"io"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
"github.com/cockroachdb/pebble/objstorage/remote"
)
const remoteMaxReadaheadSize = 1024 * 1024 /* 1MB */
// remoteReadable is a very simple implementation of Readable on top of the
// ReadCloser returned by remote.Storage.CreateObject.
type remoteReadable struct {
objReader remote.ObjectReader
size int64
fileNum base.DiskFileNum
provider *provider
}
var _ objstorage.Readable = (*remoteReadable)(nil)
func (p *provider) newRemoteReadable(
objReader remote.ObjectReader, size int64, fileNum base.DiskFileNum,
) *remoteReadable {
return &remoteReadable{
objReader: objReader,
size: size,
fileNum: fileNum,
provider: p,
}
}
// ReadAt is part of the objstorage.Readable interface.
func (r *remoteReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
return r.readInternal(ctx, p, offset, false /* forCompaction */)
}
// readInternal performs a read for the object, using the cache when
// appropriate.
func (r *remoteReadable) readInternal(
ctx context.Context, p []byte, offset int64, forCompaction bool,
) error {
if cache := r.provider.remote.cache; cache != nil {
flags := sharedcache.ReadFlags{
// Don't add data to the cache if this read is for a compaction.
ReadOnly: forCompaction,
}
return r.provider.remote.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
}
return r.objReader.ReadAt(ctx, p, offset)
}
func (r *remoteReadable) Close() error {
defer func() { r.objReader = nil }()
return r.objReader.Close()
}
func (r *remoteReadable) Size() int64 {
return r.size
}
func (r *remoteReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
// TODO(radu): use a pool.
rh := &remoteReadHandle{readable: r}
rh.readahead.state = makeReadaheadState(remoteMaxReadaheadSize)
return rh
}
type remoteReadHandle struct {
readable *remoteReadable
readahead struct {
state readaheadState
data []byte
offset int64
}
forCompaction bool
}
var _ objstorage.ReadHandle = (*remoteReadHandle)(nil)
// ReadAt is part of the objstorage.ReadHandle interface.
func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
readaheadSize := r.maybeReadahead(offset, len(p))
// Check if we already have the data from a previous read-ahead.
if rhSize := int64(len(r.readahead.data)); rhSize > 0 {
if r.readahead.offset <= offset && r.readahead.offset+rhSize > offset {
n := copy(p, r.readahead.data[offset-r.readahead.offset:])
if n == len(p) {
// All data was available.
return nil
}
// Use the data that we had and do a shorter read.
offset += int64(n)
p = p[n:]
readaheadSize -= n
}
}
if readaheadSize > len(p) {
// Don't try to read past EOF.
if offset+int64(readaheadSize) > r.readable.size {
readaheadSize = int(r.readable.size - offset)
if readaheadSize <= 0 {
// This shouldn't happen in practice (Pebble should never try to read
// past EOF).
return io.EOF
}
}
r.readahead.offset = offset
// TODO(radu): we need to somehow account for this memory.
if cap(r.readahead.data) >= readaheadSize {
r.readahead.data = r.readahead.data[:readaheadSize]
} else {
r.readahead.data = make([]byte, readaheadSize)
}
if err := r.readable.readInternal(ctx, r.readahead.data, offset, r.forCompaction); err != nil {
// Make sure we don't treat the data as valid next time.
r.readahead.data = r.readahead.data[:0]
return err
}
copy(p, r.readahead.data)
return nil
}
return r.readable.readInternal(ctx, p, offset, r.forCompaction)
}
func (r *remoteReadHandle) maybeReadahead(offset int64, len int) int {
if r.forCompaction {
return remoteMaxReadaheadSize
}
return int(r.readahead.state.maybeReadahead(offset, int64(len)))
}
// Close is part of the objstorage.ReadHandle interface.
func (r *remoteReadHandle) Close() error {
r.readable = nil
r.readahead.data = nil
return nil
}
// SetupForCompaction is part of the objstorage.ReadHandle interface.
func (r *remoteReadHandle) SetupForCompaction() {
r.forCompaction = true
}
// RecordCacheHit is part of the objstorage.ReadHandle interface.
func (r *remoteReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
if !r.forCompaction {
r.readahead.state.recordCacheHit(offset, size)
}
}

137
vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/catalog.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go generated vendored
View File

@@ -2,7 +2,7 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sharedobjcat
package remoteobjcat
import (
"fmt"
@@ -12,23 +12,25 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
// Catalog is used to manage the on-disk shared object catalog.
// Catalog is used to manage the on-disk remote object catalog.
//
// The catalog file is a log of records, where each record is an encoded
// versionEdit.
// VersionEdit.
type Catalog struct {
fs vfs.FS
dirname string
mu struct {
sync.Mutex
creatorID CreatorID
objects map[base.FileNum]SharedObjectMetadata
creatorID objstorage.CreatorID
objects map[base.DiskFileNum]RemoteObjectMetadata
marker *atomicfs.Marker
@@ -43,44 +45,41 @@ type Catalog struct {
}
}
// CreatorID identifies the DB instance that originally created a shared object.
// This ID is incorporated in backing object names.
// Must be non-zero.
type CreatorID uint64
// IsSet returns true if the CreatorID is not zero.
func (c CreatorID) IsSet() bool { return c != 0 }
func (c CreatorID) String() string { return fmt.Sprintf("%020d", c) }
// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
type SharedObjectMetadata struct {
// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object.
type RemoteObjectMetadata struct {
// FileNum is the identifier for the object within the context of a single DB
// instance.
FileNum base.FileNum
FileNum base.DiskFileNum
// FileType is the type of the object. Only certain FileTypes are possible.
FileType base.FileType
// CreatorID identifies the DB instance that originally created the object.
CreatorID CreatorID
CreatorID objstorage.CreatorID
// CreatorFileNum is the identifier for the object within the context of the
// DB instance that originally created the object.
CreatorFileNum base.FileNum
CreatorFileNum base.DiskFileNum
// CleanupMethod indicates the method for cleaning up unused shared objects.
CleanupMethod objstorage.SharedCleanupMethod
// Locator identifies a remote.Storage implementation.
Locator remote.Locator
// CustomObjectName (if it is set) overrides the object name that is normally
// derived from the CreatorID and CreatorFileNum.
CustomObjectName string
}
const (
catalogFilenameBase = "SHARED-CATALOG"
catalogMarkerName = "shared-catalog"
catalogFilenameBase = "REMOTE-OBJ-CATALOG"
catalogMarkerName = "remote-obj-catalog"
// We create a new file when the size exceeds 1MB (and some other conditions
// hold; see record.RotationHelper).
rotateFileSize = 1024 * 1024 // 1MB
)
// CatalogContents contains the shared objects in the catalog.
// CatalogContents contains the remote objects in the catalog.
type CatalogContents struct {
// CreatorID, if it is set.
CreatorID CreatorID
Objects []SharedObjectMetadata
CreatorID objstorage.CreatorID
Objects []RemoteObjectMetadata
}
// Open creates a Catalog and loads any existing catalog file, returning the
@@ -90,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
fs: fs,
dirname: dirname,
}
c.mu.objects = make(map[base.FileNum]SharedObjectMetadata)
c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata)
var err error
c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
@@ -109,20 +108,20 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
}
res := CatalogContents{
CreatorID: c.mu.creatorID,
Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)),
Objects: make([]RemoteObjectMetadata, 0, len(c.mu.objects)),
}
for _, meta := range c.mu.objects {
res.Objects = append(res.Objects, meta)
}
// Sort the objects so the function is deterministic.
sort.Slice(res.Objects, func(i, j int) bool {
return res.Objects[i].FileNum < res.Objects[j].FileNum
return res.Objects[i].FileNum.FileNum() < res.Objects[j].FileNum.FileNum()
})
return c, res, nil
}
// SetCreatorID sets the creator ID. If it is already set, it must match.
func (c *Catalog) SetCreatorID(id CreatorID) error {
func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {
if !id.IsSet() {
return errors.AssertionFailedf("attempt to unset CreatorID")
}
@@ -137,9 +136,9 @@ func (c *Catalog) SetCreatorID(id CreatorID) error {
return nil
}
ve := versionEdit{CreatorID: id}
ve := VersionEdit{CreatorID: id}
if err := c.writeToCatalogFileLocked(&ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}
c.mu.creatorID = id
return nil
@@ -166,19 +165,19 @@ func (c *Catalog) closeCatalogFile() error {
// Batch is used to perform multiple object additions/deletions at once.
type Batch struct {
ve versionEdit
ve VersionEdit
}
// AddObject adds a new object to the batch.
//
// The given FileNum must be new - it must not match that of any object that was
// ever in the catalog.
func (b *Batch) AddObject(meta SharedObjectMetadata) {
func (b *Batch) AddObject(meta RemoteObjectMetadata) {
b.ve.NewObjects = append(b.ve.NewObjects, meta)
}
// DeleteObject adds an object removal to the batch.
func (b *Batch) DeleteObject(fileNum base.FileNum) {
func (b *Batch) DeleteObject(fileNum base.DiskFileNum) {
b.ve.DeletedObjects = append(b.ve.DeletedObjects, fileNum)
}
@@ -197,11 +196,11 @@ func (b *Batch) IsEmpty() bool {
func (b *Batch) Copy() Batch {
var res Batch
if len(b.ve.NewObjects) > 0 {
res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects))
copy(res.ve.NewObjects, b.ve.NewObjects)
}
if len(b.ve.DeletedObjects) > 0 {
res.ve.DeletedObjects = make([]base.FileNum, len(b.ve.DeletedObjects))
res.ve.DeletedObjects = make([]base.DiskFileNum, len(b.ve.DeletedObjects))
copy(res.ve.DeletedObjects, b.ve.DeletedObjects)
}
return res
@@ -219,29 +218,40 @@ func (c *Catalog) ApplyBatch(b Batch) error {
c.mu.Lock()
defer c.mu.Unlock()
for _, n := range b.ve.DeletedObjects {
if _, exists := c.mu.objects[n]; !exists {
return errors.AssertionFailedf("deleting non-existent object %s", n)
// Sanity checks.
toAdd := make(map[base.DiskFileNum]struct{}, len(b.ve.NewObjects))
exists := func(n base.DiskFileNum) bool {
_, ok := c.mu.objects[n]
if !ok {
_, ok = toAdd[n]
}
return ok
}
for _, meta := range b.ve.NewObjects {
if _, exists := c.mu.objects[meta.FileNum]; exists {
if exists(meta.FileNum) {
return errors.AssertionFailedf("adding existing object %s", meta.FileNum)
}
toAdd[meta.FileNum] = struct{}{}
}
for _, n := range b.ve.DeletedObjects {
if !exists(n) {
return errors.AssertionFailedf("deleting non-existent object %s", n)
}
}
if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
}
// Apply the batch to our current state.
for _, n := range b.ve.DeletedObjects {
delete(c.mu.objects, n)
}
// Add new objects before deleting any objects. This allows for cases where
// the same batch adds and deletes an object.
for _, meta := range b.ve.NewObjects {
c.mu.objects[meta.FileNum] = meta
}
b.Reset()
for _, n := range b.ve.DeletedObjects {
delete(c.mu.objects, n)
}
return nil
}
@@ -250,7 +260,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
f, err := c.fs.Open(catalogPath)
if err != nil {
return errors.Wrapf(
err, "pebble: could not open shared object catalog file %q for DB %q",
err, "pebble: could not open remote object catalog file %q for DB %q",
errors.Safe(filename), c.dirname,
)
}
@@ -262,32 +272,26 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
break
}
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
var ve versionEdit
err = ve.Decode(r)
if err != nil {
return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
var ve VersionEdit
if err := ve.Decode(r); err != nil {
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
// Apply the version edit to the current state.
if ve.CreatorID.IsSet() {
c.mu.creatorID = ve.CreatorID
}
for _, fileNum := range ve.DeletedObjects {
delete(c.mu.objects, fileNum)
}
for _, meta := range ve.NewObjects {
c.mu.objects[meta.FileNum] = meta
if err := ve.Apply(&c.mu.creatorID, c.mu.objects); err != nil {
return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
errors.Safe(filename))
}
}
return nil
}
// writeToCatalogFileLocked writes a versionEdit to the catalog file.
// writeToCatalogFileLocked writes a VersionEdit to the catalog file.
// Creates a new file if this is the first write.
func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error {
func (c *Catalog) writeToCatalogFileLocked(ve *VersionEdit) error {
c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects)))
snapshotSize := int64(len(c.mu.objects))
@@ -330,9 +334,10 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
}
recWriter := record.NewWriter(file)
err = func() error {
// Create a versionEdit that gets us from an empty catalog to the current state.
var ve versionEdit
ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
// Create a VersionEdit that gets us from an empty catalog to the current state.
var ve VersionEdit
ve.CreatorID = c.mu.creatorID
ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects))
for _, meta := range c.mu.objects {
ve.NewObjects = append(ve.NewObjects, meta)
}
@@ -367,7 +372,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
return nil
}
func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error {
func writeRecord(ve *VersionEdit, file vfs.File, recWriter *record.Writer) error {
w, err := recWriter.Next()
if err != nil {
return err

254
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go generated vendored Normal file
View File

@@ -0,0 +1,254 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package remoteobjcat
import (
"bufio"
"encoding/binary"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/remote"
)
// VersionEdit is a modification to the remote object state which can be encoded
// into a record.
//
// TODO(radu): consider adding creation and deletion time for debugging purposes.
type VersionEdit struct {
NewObjects []RemoteObjectMetadata
DeletedObjects []base.DiskFileNum
CreatorID objstorage.CreatorID
}
const (
// tagNewObject is followed by the FileNum, creator ID, creator FileNum,
// cleanup method, optional new object tags, and ending with a 0 byte.
tagNewObject = 1
// tagDeletedObject is followed by the FileNum.
tagDeletedObject = 2
// tagCreatorID is followed by the Creator ID for this store. This ID can
// never change.
tagCreatorID = 3
// tagNewObjectLocator is an optional tag inside the tagNewObject payload. It
// is followed by the encoded length of the locator string and the string.
tagNewObjectLocator = 4
// tagNewObjectCustomName is an optional tag inside the tagNewObject payload.
// It is followed by the encoded length of the custom object name string
// followed by the string.
tagNewObjectCustomName = 5
)
// Object type values. We don't want to encode FileType directly because it is
// more general (and we want freedom to change it in the future).
const (
objTypeTable = 1
)
func objTypeToFileType(objType uint64) (base.FileType, error) {
switch objType {
case objTypeTable:
return base.FileTypeTable, nil
default:
return 0, errors.Newf("unknown object type %d", objType)
}
}
func fileTypeToObjType(fileType base.FileType) (uint64, error) {
switch fileType {
case base.FileTypeTable:
return objTypeTable, nil
default:
return 0, errors.Newf("unknown object type for file type %d", fileType)
}
}
// Encode encodes an edit to the specified writer.
func (v *VersionEdit) Encode(w io.Writer) error {
buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*10+len(v.DeletedObjects)*2+2))
for _, meta := range v.NewObjects {
objType, err := fileTypeToObjType(meta.FileType)
if err != nil {
return err
}
buf = binary.AppendUvarint(buf, uint64(tagNewObject))
buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum()))
buf = binary.AppendUvarint(buf, objType)
buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
if meta.Locator != "" {
buf = binary.AppendUvarint(buf, uint64(tagNewObjectLocator))
buf = encodeString(buf, string(meta.Locator))
}
if meta.CustomObjectName != "" {
buf = binary.AppendUvarint(buf, uint64(tagNewObjectCustomName))
buf = encodeString(buf, meta.CustomObjectName)
}
// Append 0 as the terminator for optional new object tags.
buf = binary.AppendUvarint(buf, 0)
}
for _, dfn := range v.DeletedObjects {
buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
buf = binary.AppendUvarint(buf, uint64(dfn.FileNum()))
}
if v.CreatorID.IsSet() {
buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
}
_, err := w.Write(buf)
return err
}
// Decode decodes an edit from the specified reader.
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(io.ByteReader)
if !ok {
br = bufio.NewReader(r)
}
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return err
}
err = nil
switch tag {
case tagNewObject:
var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
var locator, customName string
var fileType base.FileType
fileNum, err = binary.ReadUvarint(br)
if err == nil {
var objType uint64
objType, err = binary.ReadUvarint(br)
if err == nil {
fileType, err = objTypeToFileType(objType)
}
}
if err == nil {
creatorID, err = binary.ReadUvarint(br)
}
if err == nil {
creatorFileNum, err = binary.ReadUvarint(br)
}
if err == nil {
cleanupMethod, err = binary.ReadUvarint(br)
}
for err == nil {
var optionalTag uint64
optionalTag, err = binary.ReadUvarint(br)
if err != nil || optionalTag == 0 {
break
}
switch optionalTag {
case tagNewObjectLocator:
locator, err = decodeString(br)
case tagNewObjectCustomName:
customName, err = decodeString(br)
default:
err = errors.Newf("unknown newObject tag %d", optionalTag)
}
}
if err == nil {
v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{
FileNum: base.FileNum(fileNum).DiskFileNum(),
FileType: fileType,
CreatorID: objstorage.CreatorID(creatorID),
CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(),
CleanupMethod: objstorage.SharedCleanupMethod(cleanupMethod),
Locator: remote.Locator(locator),
CustomObjectName: customName,
})
}
case tagDeletedObject:
var fileNum uint64
fileNum, err = binary.ReadUvarint(br)
if err == nil {
v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum())
}
case tagCreatorID:
var id uint64
id, err = binary.ReadUvarint(br)
if err == nil {
v.CreatorID = objstorage.CreatorID(id)
}
default:
err = errors.Newf("unknown tag %d", tag)
}
if err != nil {
if err == io.EOF {
return errCorruptCatalog
}
return err
}
}
return nil
}
func encodeString(buf []byte, s string) []byte {
buf = binary.AppendUvarint(buf, uint64(len(s)))
buf = append(buf, []byte(s)...)
return buf
}
func decodeString(br io.ByteReader) (string, error) {
length, err := binary.ReadUvarint(br)
if err != nil || length == 0 {
return "", err
}
buf := make([]byte, length)
for i := range buf {
buf[i], err = br.ReadByte()
if err != nil {
return "", err
}
}
return string(buf), nil
}
var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog")
// Apply the version edit to a creator ID and a map of objects.
func (v *VersionEdit) Apply(
creatorID *objstorage.CreatorID, objects map[base.DiskFileNum]RemoteObjectMetadata,
) error {
if v.CreatorID.IsSet() {
*creatorID = v.CreatorID
}
for _, meta := range v.NewObjects {
if invariants.Enabled {
if _, exists := objects[meta.FileNum]; exists {
return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
}
}
objects[meta.FileNum] = meta
}
for _, fileNum := range v.DeletedObjects {
if invariants.Enabled {
if _, exists := objects[fileNum]; !exists {
return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
}
}
delete(objects, fileNum)
}
return nil
}

65
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go generated vendored Normal file
View File

@@ -0,0 +1,65 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorageprovider
import (
"io"
"github.com/cockroachdb/pebble/objstorage"
)
// NewRemoteWritable creates an objstorage.Writable out of an io.WriteCloser.
func NewRemoteWritable(obj io.WriteCloser) objstorage.Writable {
return &sharedWritable{storageWriter: obj}
}
// sharedWritable is a very simple implementation of Writable on top of the
// WriteCloser returned by remote.Storage.CreateObject.
type sharedWritable struct {
// Either both p and meta must be unset / zero values, or both must be set.
// The case where both are unset is true only in tests.
p *provider
meta objstorage.ObjectMetadata
storageWriter io.WriteCloser
}
var _ objstorage.Writable = (*sharedWritable)(nil)
// Write is part of the Writable interface.
func (w *sharedWritable) Write(p []byte) error {
_, err := w.storageWriter.Write(p)
return err
}
// Finish is part of the Writable interface.
func (w *sharedWritable) Finish() error {
err := w.storageWriter.Close()
w.storageWriter = nil
if err != nil {
w.Abort()
return err
}
// Create the marker object.
if w.p != nil {
if err := w.p.sharedCreateRef(w.meta); err != nil {
w.Abort()
return err
}
}
return nil
}
// Abort is part of the Writable interface.
func (w *sharedWritable) Abort() {
if w.storageWriter != nil {
_ = w.storageWriter.Close()
w.storageWriter = nil
}
if w.p != nil {
w.p.removeMetadata(w.meta.DiskFileNum)
}
// TODO(radu): delete the object if it was created.
}

900
vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go generated vendored Normal file
View File

@@ -0,0 +1,900 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sharedcache
import (
"context"
"fmt"
"io"
"math/bits"
"sync"
"sync/atomic"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/vfs"
"github.com/prometheus/client_golang/prometheus"
)
// Exported to enable exporting from package pebble to enable
// exporting metrics with below buckets in CRDB.
var (
IOBuckets = prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50)
ChannelWriteBuckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50)
)
// Cache is a persistent cache backed by a local filesystem. It is intended
// to cache data that is in slower shared storage (e.g. S3), hence the
// package name 'sharedcache'.
type Cache struct {
shards []shard
writeWorkers writeWorkers
bm blockMath
shardingBlockSize int64
logger base.Logger
metrics internalMetrics
}
// Metrics is a struct containing metrics exported by the secondary cache.
// TODO(josh): Reconsider the set of metrics exported by the secondary cache
// before we release the secondary cache to users. We choose to export many metrics
// right now, so we learn a lot from the benchmarking we are doing over the 23.2
// cycle.
type Metrics struct {
// The number of sstable bytes stored in the cache.
Size int64
// The count of cache blocks in the cache (not sstable blocks).
Count int64
// The number of calls to ReadAt.
TotalReads int64
// The number of calls to ReadAt that require reading data from 2+ shards.
MultiShardReads int64
// The number of calls to ReadAt that require reading data from 2+ cache blocks.
MultiBlockReads int64
// The number of calls to ReadAt where all data returned was read from the cache.
ReadsWithFullHit int64
// The number of calls to ReadAt where some data returned was read from the cache.
ReadsWithPartialHit int64
// The number of calls to ReadAt where no data returned was read from the cache.
ReadsWithNoHit int64
// The number of times a cache block was evicted from the cache.
Evictions int64
// The number of times writing a cache block to the cache failed.
WriteBackFailures int64
// The latency of calls to get some data from the cache.
GetLatency prometheus.Histogram
// The latency of reads of a single cache block from disk.
DiskReadLatency prometheus.Histogram
// The latency of writing data to write back to the cache to a channel.
// Generally should be low, but if the channel is full, could be high.
QueuePutLatency prometheus.Histogram
// The latency of calls to put some data read from block storage into the cache.
PutLatency prometheus.Histogram
// The latency of writes of a single cache block to disk.
DiskWriteLatency prometheus.Histogram
}
// See docs at Metrics.
type internalMetrics struct {
count atomic.Int64
totalReads atomic.Int64
multiShardReads atomic.Int64
multiBlockReads atomic.Int64
readsWithFullHit atomic.Int64
readsWithPartialHit atomic.Int64
readsWithNoHit atomic.Int64
evictions atomic.Int64
writeBackFailures atomic.Int64
getLatency prometheus.Histogram
diskReadLatency prometheus.Histogram
queuePutLatency prometheus.Histogram
putLatency prometheus.Histogram
diskWriteLatency prometheus.Histogram
}
const (
// writeWorkersPerShard is used to establish the number of worker goroutines
// that perform writes to the cache.
writeWorkersPerShard = 4
// writeTaskPerWorker is used to establish how many tasks can be queued up
// until we have to block.
writeTasksPerWorker = 4
)
// Open opens a cache. If there is no existing cache at fsDir, a new one
// is created.
func Open(
fs vfs.FS,
logger base.Logger,
fsDir string,
blockSize int,
// shardingBlockSize is the size of a shard block. The cache is split into contiguous
// shardingBlockSize units. The units are distributed across multiple independent shards
// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
// policies operate at the level of shard, not whole cache. This is done to reduce lock
// contention.
shardingBlockSize int64,
sizeBytes int64,
numShards int,
) (*Cache, error) {
if minSize := shardingBlockSize * int64(numShards); sizeBytes < minSize {
// Up the size so that we have one block per shard. In practice, this should
// only happen in tests.
sizeBytes = minSize
}
c := &Cache{
logger: logger,
bm: makeBlockMath(blockSize),
shardingBlockSize: shardingBlockSize,
}
c.shards = make([]shard, numShards)
blocksPerShard := sizeBytes / int64(numShards) / int64(blockSize)
for i := range c.shards {
if err := c.shards[i].init(c, fs, fsDir, i, blocksPerShard, blockSize, shardingBlockSize); err != nil {
return nil, err
}
}
c.writeWorkers.Start(c, numShards*writeWorkersPerShard)
c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
// Measures a channel write, so lower min.
c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: ChannelWriteBuckets})
return c, nil
}
// Close closes the cache. Methods such as ReadAt should not be called after Close is
// called.
func (c *Cache) Close() error {
c.writeWorkers.Stop()
var retErr error
for i := range c.shards {
if err := c.shards[i].close(); err != nil && retErr == nil {
retErr = err
}
}
c.shards = nil
return retErr
}
// Metrics return metrics for the cache. Callers should not mutate
// the returned histograms, which are pointer types.
func (c *Cache) Metrics() Metrics {
return Metrics{
Count: c.metrics.count.Load(),
Size: c.metrics.count.Load() * int64(c.bm.BlockSize()),
TotalReads: c.metrics.totalReads.Load(),
MultiShardReads: c.metrics.multiShardReads.Load(),
MultiBlockReads: c.metrics.multiBlockReads.Load(),
ReadsWithFullHit: c.metrics.readsWithFullHit.Load(),
ReadsWithPartialHit: c.metrics.readsWithPartialHit.Load(),
ReadsWithNoHit: c.metrics.readsWithNoHit.Load(),
Evictions: c.metrics.evictions.Load(),
WriteBackFailures: c.metrics.writeBackFailures.Load(),
GetLatency: c.metrics.getLatency,
DiskReadLatency: c.metrics.diskReadLatency,
QueuePutLatency: c.metrics.queuePutLatency,
PutLatency: c.metrics.putLatency,
DiskWriteLatency: c.metrics.diskWriteLatency,
}
}
// ReadFlags contains options for Cache.ReadAt.
type ReadFlags struct {
// ReadOnly instructs ReadAt to not write any new data into the cache; it is
// used when the data is unlikely to be used again.
ReadOnly bool
}
// ReadAt performs a read form an object, attempting to use cached data when
// possible.
func (c *Cache) ReadAt(
ctx context.Context,
fileNum base.DiskFileNum,
p []byte,
ofs int64,
objReader remote.ObjectReader,
objSize int64,
flags ReadFlags,
) error {
c.metrics.totalReads.Add(1)
if ofs >= objSize {
if invariants.Enabled {
panic(fmt.Sprintf("invalid ReadAt offset %v %v", ofs, objSize))
}
return io.EOF
}
// TODO(radu): for compaction reads, we may not want to read from the cache at
// all.
{
start := time.Now()
n, err := c.get(fileNum, p, ofs)
c.metrics.getLatency.Observe(float64(time.Since(start)))
if err != nil {
return err
}
if n == len(p) {
// Everything was in cache!
c.metrics.readsWithFullHit.Add(1)
return nil
}
if n == 0 {
c.metrics.readsWithNoHit.Add(1)
} else {
c.metrics.readsWithPartialHit.Add(1)
}
// Note this. The below code does not need the original ofs, as with the earlier
// reading from the cache done, the relevant offset is ofs + int64(n). Same with p.
ofs += int64(n)
p = p[n:]
if invariants.Enabled {
if n != 0 && c.bm.Remainder(ofs) != 0 {
panic(fmt.Sprintf("after non-zero read from cache, ofs is not block-aligned: %v %v", ofs, n))
}
}
}
if flags.ReadOnly {
return objReader.ReadAt(ctx, p, ofs)
}
// We must do reads with offset & size that are multiples of the block size. Else
// later cache hits may return incorrect zeroed results from the cache.
firstBlockInd := c.bm.Block(ofs)
adjustedOfs := c.bm.BlockOffset(firstBlockInd)
// Take the length of what is left to read plus the length of the adjustment of
// the offset plus the size of a block minus one and divide by the size of a block
// to get the number of blocks to read from the object.
sizeOfOffAdjustment := int(ofs - adjustedOfs)
adjustedLen := int(c.bm.RoundUp(int64(len(p) + sizeOfOffAdjustment)))
adjustedP := make([]byte, adjustedLen)
// Read the rest from the object. We may need to cap the length to avoid past EOF reads.
eofCap := int64(adjustedLen)
if adjustedOfs+eofCap > objSize {
eofCap = objSize - adjustedOfs
}
if err := objReader.ReadAt(ctx, adjustedP[:eofCap], adjustedOfs); err != nil {
return err
}
copy(p, adjustedP[sizeOfOffAdjustment:])
start := time.Now()
c.writeWorkers.QueueWrite(fileNum, adjustedP, adjustedOfs)
c.metrics.queuePutLatency.Observe(float64(time.Since(start)))
return nil
}
// get attempts to read the requested data from the cache, if it is already
// there.
//
// If all data is available, returns n = len(p).
//
// If data is partially available, a prefix of the data is read; returns n < len(p)
// and no error. If no prefix is available, returns n = 0 and no error.
func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
// The data extent might cross shard boundaries, hence the loop. In the hot
// path, max two iterations of this loop will be executed, since reads are sized
// in units of sstable block size.
var multiShard bool
for {
shard := c.getShard(fileNum, ofs+int64(n))
cappedLen := len(p[n:])
if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
cappedLen = toBoundary
}
numRead, err := shard.get(fileNum, p[n:n+cappedLen], ofs+int64(n))
if err != nil {
return n, err
}
n += numRead
if numRead < cappedLen {
// We only read a prefix from this shard.
return n, nil
}
if n == len(p) {
// We are done.
return n, nil
}
// Data extent crosses shard boundary, continue with next shard.
if !multiShard {
c.metrics.multiShardReads.Add(1)
multiShard = true
}
}
}
// set attempts to write the requested data to the cache. Both ofs & len(p) must
// be multiples of the block size.
//
// If all of p is not written to the shard, set returns a non-nil error.
func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
if invariants.Enabled {
if c.bm.Remainder(ofs) != 0 || c.bm.Remainder(int64(len(p))) != 0 {
panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
}
}
// The data extent might cross shard boundaries, hence the loop. In the hot
// path, max two iterations of this loop will be executed, since reads are sized
// in units of sstable block size.
n := 0
for {
shard := c.getShard(fileNum, ofs+int64(n))
cappedLen := len(p[n:])
if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
cappedLen = toBoundary
}
err := shard.set(fileNum, p[n:n+cappedLen], ofs+int64(n))
if err != nil {
return err
}
// set returns an error if cappedLen bytes aren't written to the shard.
n += cappedLen
if n == len(p) {
// We are done.
return nil
}
// Data extent crosses shard boundary, continue with next shard.
}
}
func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard {
const prime64 = 1099511628211
hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs/c.shardingBlockSize)
// TODO(josh): Instance change ops are often run in production. Such an operation
// updates len(c.shards); see openSharedCache. As a result, the behavior of this
// function changes, and the cache empties out at restart time. We may want a better
// story here eventually.
return &c.shards[hash%uint64(len(c.shards))]
}
type shard struct {
cache *Cache
file vfs.File
sizeInBlocks int64
bm blockMath
shardingBlockSize int64
mu struct {
sync.Mutex
// TODO(josh): None of these datastructures are space-efficient.
// Focusing on correctness to start.
where whereMap
blocks []cacheBlockState
// Head of LRU list (doubly-linked circular).
lruHead cacheBlockIndex
// Head of free list (singly-linked chain).
freeHead cacheBlockIndex
}
}
type cacheBlockState struct {
lock lockState
logical logicalBlockID
// next is the next block in the LRU or free list (or invalidBlockIndex if it
// is the last block in the free list).
next cacheBlockIndex
// prev is the previous block in the LRU list. It is not used when the block
// is in the free list.
prev cacheBlockIndex
}
// Maps a logical block in an SST to an index of the cache block with the
// file contents (to the "cache block index").
type whereMap map[logicalBlockID]cacheBlockIndex
type logicalBlockID struct {
filenum base.DiskFileNum
cacheBlockIdx cacheBlockIndex
}
type lockState int64
const (
unlocked lockState = 0
// >0 lockState tracks the number of distinct readers of some cache block / logical block
// which is in the secondary cache. It is used to ensure that a cache block is not evicted
// and overwritten, while there are active readers.
readLockTakenInc = 1
// -1 lockState indicates that some cache block is currently being populated with data from
// blob storage. It is used to ensure that a cache block is not read or evicted again, while
// it is being populated.
writeLockTaken = -1
)
func (s *shard) init(
cache *Cache,
fs vfs.FS,
fsDir string,
shardIdx int,
sizeInBlocks int64,
blockSize int,
shardingBlockSize int64,
) error {
*s = shard{
cache: cache,
sizeInBlocks: sizeInBlocks,
}
if blockSize < 1024 || shardingBlockSize%int64(blockSize) != 0 {
return errors.Newf("invalid block size %d (must divide %d)", blockSize, shardingBlockSize)
}
s.bm = makeBlockMath(blockSize)
s.shardingBlockSize = shardingBlockSize
file, err := fs.OpenReadWrite(fs.PathJoin(fsDir, fmt.Sprintf("SHARED-CACHE-%03d", shardIdx)))
if err != nil {
return err
}
// TODO(radu): truncate file if necessary (especially important if we restart
// with more shards).
if err := file.Preallocate(0, int64(blockSize)*sizeInBlocks); err != nil {
return err
}
s.file = file
// TODO(josh): Right now, the secondary cache is not persistent. All existing
// cache contents will be over-written, since all metadata is only stored in
// memory.
s.mu.where = make(whereMap)
s.mu.blocks = make([]cacheBlockState, sizeInBlocks)
s.mu.lruHead = invalidBlockIndex
s.mu.freeHead = invalidBlockIndex
for i := range s.mu.blocks {
s.freePush(cacheBlockIndex(i))
}
return nil
}
func (s *shard) close() error {
defer func() {
s.file = nil
}()
return s.file.Close()
}
// freePush pushes a block to the front of the free list.
func (s *shard) freePush(index cacheBlockIndex) {
s.mu.blocks[index].next = s.mu.freeHead
s.mu.freeHead = index
}
// freePop removes the block from the front of the free list. Must not be called
// if the list is empty (i.e. freeHead = invalidBlockIndex).
func (s *shard) freePop() cacheBlockIndex {
index := s.mu.freeHead
s.mu.freeHead = s.mu.blocks[index].next
return index
}
// lruInsertFront inserts a block at the front of the LRU list.
func (s *shard) lruInsertFront(index cacheBlockIndex) {
b := &s.mu.blocks[index]
if s.mu.lruHead == invalidBlockIndex {
b.next = index
b.prev = index
} else {
b.next = s.mu.lruHead
h := &s.mu.blocks[s.mu.lruHead]
b.prev = h.prev
s.mu.blocks[h.prev].next = index
h.prev = index
}
s.mu.lruHead = index
}
func (s *shard) lruNext(index cacheBlockIndex) cacheBlockIndex {
return s.mu.blocks[index].next
}
func (s *shard) lruPrev(index cacheBlockIndex) cacheBlockIndex {
return s.mu.blocks[index].prev
}
// lruUnlink removes a block from the LRU list.
func (s *shard) lruUnlink(index cacheBlockIndex) {
b := &s.mu.blocks[index]
if b.next == index {
s.mu.lruHead = invalidBlockIndex
} else {
s.mu.blocks[b.prev].next = b.next
s.mu.blocks[b.next].prev = b.prev
if s.mu.lruHead == index {
s.mu.lruHead = b.next
}
}
b.next, b.prev = invalidBlockIndex, invalidBlockIndex
}
// get attempts to read the requested data from the shard. The data must not
// cross a shard boundary.
//
// If all data is available, returns n = len(p).
//
// If data is partially available, a prefix of the data is read; returns n < len(p)
// and no error. If no prefix is available, returns n = 0 and no error.
//
// TODO(josh): Today, if there are two cache blocks needed to satisfy a read, and the
// first block is not in the cache and the second one is, we will read both from
// blob storage. We should fix this. This is not an unlikely scenario if we are doing
// a reverse scan, since those iterate over sstable blocks in reverse order and due to
// cache block aligned reads will have read the suffix of the sstable block that will
// be needed next.
func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
if invariants.Enabled {
if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
panic(fmt.Sprintf("get crosses shard boundary: %v %v", ofs, len(p)))
}
s.assertShardStateIsConsistent()
}
// The data extent might cross cache block boundaries, hence the loop. In the hot
// path, max two iterations of this loop will be executed, since reads are sized
// in units of sstable block size.
var multiBlock bool
for {
k := logicalBlockID{
filenum: fileNum,
cacheBlockIdx: s.bm.Block(ofs + int64(n)),
}
s.mu.Lock()
cacheBlockIdx, ok := s.mu.where[k]
// TODO(josh): Multiple reads within the same few milliseconds (anything that is smaller
// than blob storage read latency) that miss on the same logical block ID will not necessarily
// be rare. We may want to do only one read, with the later readers blocking on the first read
// completing. This could be implemented either here or in the primary block cache. See
// https://github.com/cockroachdb/pebble/pull/2586 for additional discussion.
if !ok {
s.mu.Unlock()
return n, nil
}
if s.mu.blocks[cacheBlockIdx].lock == writeLockTaken {
// In practice, if we have two reads of the same SST block in close succession, we
// would expect the second to hit in the in-memory block cache. So it's not worth
// optimizing this case here.
s.mu.Unlock()
return n, nil
}
s.mu.blocks[cacheBlockIdx].lock += readLockTakenInc
// Move to front of the LRU list.
s.lruUnlink(cacheBlockIdx)
s.lruInsertFront(cacheBlockIdx)
s.mu.Unlock()
readAt := s.bm.BlockOffset(cacheBlockIdx)
readSize := s.bm.BlockSize()
if n == 0 { // if first read
rem := s.bm.Remainder(ofs)
readAt += rem
readSize -= int(rem)
}
if len(p[n:]) <= readSize {
start := time.Now()
numRead, err := s.file.ReadAt(p[n:], readAt)
s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
s.dropReadLock(cacheBlockIdx)
return n + numRead, err
}
start := time.Now()
numRead, err := s.file.ReadAt(p[n:n+readSize], readAt)
s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
s.dropReadLock(cacheBlockIdx)
if err != nil {
return 0, err
}
// Note that numRead == readSize, since we checked for an error above.
n += numRead
if !multiBlock {
s.cache.metrics.multiBlockReads.Add(1)
multiBlock = true
}
}
}
// set attempts to write the requested data to the shard. The data must not
// cross a shard boundary, and both ofs & len(p) must be multiples of the
// block size.
//
// If all of p is not written to the shard, set returns a non-nil error.
func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
if invariants.Enabled {
if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
panic(fmt.Sprintf("set crosses shard boundary: %v %v", ofs, len(p)))
}
if s.bm.Remainder(ofs) != 0 || s.bm.Remainder(int64(len(p))) != 0 {
panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
}
s.assertShardStateIsConsistent()
}
// The data extent might cross cache block boundaries, hence the loop. In the hot
// path, max two iterations of this loop will be executed, since reads are sized
// in units of sstable block size.
n := 0
for {
if n == len(p) {
return nil
}
if invariants.Enabled {
if n > len(p) {
panic(fmt.Sprintf("set with n greater than len(p): %v %v", n, len(p)))
}
}
// If the logical block is already in the cache, we should skip doing a set.
k := logicalBlockID{
filenum: fileNum,
cacheBlockIdx: s.bm.Block(ofs + int64(n)),
}
s.mu.Lock()
if _, ok := s.mu.where[k]; ok {
s.mu.Unlock()
n += s.bm.BlockSize()
continue
}
var cacheBlockIdx cacheBlockIndex
if s.mu.freeHead == invalidBlockIndex {
if invariants.Enabled && s.mu.lruHead == invalidBlockIndex {
panic("both LRU and free lists empty")
}
// Find the last element in the LRU list which is not locked.
for idx := s.lruPrev(s.mu.lruHead); ; idx = s.lruPrev(idx) {
if lock := s.mu.blocks[idx].lock; lock == unlocked {
cacheBlockIdx = idx
break
}
if idx == s.mu.lruHead {
// No unlocked block to evict.
//
// TODO(josh): We may want to block until a block frees up, instead of returning
// an error here. But I think we can do that later on, e.g. after running some production
// experiments.
s.mu.Unlock()
return errors.New("no block to evict so skipping write to cache")
}
}
s.cache.metrics.evictions.Add(1)
s.lruUnlink(cacheBlockIdx)
delete(s.mu.where, s.mu.blocks[cacheBlockIdx].logical)
} else {
s.cache.metrics.count.Add(1)
cacheBlockIdx = s.freePop()
}
s.lruInsertFront(cacheBlockIdx)
s.mu.where[k] = cacheBlockIdx
s.mu.blocks[cacheBlockIdx].logical = k
s.mu.blocks[cacheBlockIdx].lock = writeLockTaken
s.mu.Unlock()
writeAt := s.bm.BlockOffset(cacheBlockIdx)
writeSize := s.bm.BlockSize()
if len(p[n:]) <= writeSize {
writeSize = len(p[n:])
}
start := time.Now()
_, err := s.file.WriteAt(p[n:n+writeSize], writeAt)
s.cache.metrics.diskWriteLatency.Observe(float64(time.Since(start)))
if err != nil {
// Free the block.
s.mu.Lock()
defer s.mu.Unlock()
delete(s.mu.where, k)
s.lruUnlink(cacheBlockIdx)
s.freePush(cacheBlockIdx)
return err
}
s.dropWriteLock(cacheBlockIdx)
n += writeSize
}
}
// Doesn't inline currently. This might be okay, but something to keep in mind.
func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) {
s.mu.Lock()
s.mu.blocks[cacheBlockInd].lock -= readLockTakenInc
if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock < 0 {
panic(fmt.Sprintf("unexpected lock state %v in dropReadLock", s.mu.blocks[cacheBlockInd].lock))
}
s.mu.Unlock()
}
// Doesn't inline currently. This might be okay, but something to keep in mind.
func (s *shard) dropWriteLock(cacheBlockInd cacheBlockIndex) {
s.mu.Lock()
if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock != writeLockTaken {
panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock", s.mu.blocks[cacheBlockInd].lock))
}
s.mu.blocks[cacheBlockInd].lock = unlocked
s.mu.Unlock()
}
func (s *shard) assertShardStateIsConsistent() {
s.mu.Lock()
defer s.mu.Unlock()
lruLen := 0
if s.mu.lruHead != invalidBlockIndex {
for b := s.mu.lruHead; ; {
lruLen++
if idx, ok := s.mu.where[s.mu.blocks[b].logical]; !ok || idx != b {
panic("block in LRU list with no entry in where map")
}
b = s.lruNext(b)
if b == s.mu.lruHead {
break
}
}
}
if lruLen != len(s.mu.where) {
panic(fmt.Sprintf("lru list len is %d but where map has %d entries", lruLen, len(s.mu.where)))
}
freeLen := 0
for n := s.mu.freeHead; n != invalidBlockIndex; n = s.mu.blocks[n].next {
freeLen++
}
if lruLen+freeLen != int(s.sizeInBlocks) {
panic(fmt.Sprintf("%d lru blocks and %d free blocks don't add up to %d", lruLen, freeLen, s.sizeInBlocks))
}
for i := range s.mu.blocks {
if state := s.mu.blocks[i].lock; state < writeLockTaken {
panic(fmt.Sprintf("lock state %v is not allowed", state))
}
}
}
// cacheBlockIndex is the index of a blockSize-aligned cache block.
type cacheBlockIndex int64
// invalidBlockIndex is used for the head of a list when the list is empty.
const invalidBlockIndex cacheBlockIndex = -1
// blockMath is a helper type for performing conversions between offsets and
// block indexes.
type blockMath struct {
blockSizeBits int8
}
func makeBlockMath(blockSize int) blockMath {
bm := blockMath{
blockSizeBits: int8(bits.Len64(uint64(blockSize)) - 1),
}
if blockSize != (1 << bm.blockSizeBits) {
panic(fmt.Sprintf("blockSize %d is not a power of 2", blockSize))
}
return bm
}
func (bm blockMath) mask() int64 {
return (1 << bm.blockSizeBits) - 1
}
// BlockSize returns the block size.
func (bm blockMath) BlockSize() int {
return 1 << bm.blockSizeBits
}
// Block returns the block index containing the given offset.
func (bm blockMath) Block(offset int64) cacheBlockIndex {
return cacheBlockIndex(offset >> bm.blockSizeBits)
}
// Remainder returns the offset relative to the start of the cache block.
func (bm blockMath) Remainder(offset int64) int64 {
return offset & bm.mask()
}
// BlockOffset returns the object offset where the given block starts.
func (bm blockMath) BlockOffset(block cacheBlockIndex) int64 {
return int64(block) << bm.blockSizeBits
}
// RoundUp rounds up the given value to the closest multiple of block size.
func (bm blockMath) RoundUp(x int64) int64 {
return (x + bm.mask()) & ^(bm.mask())
}
type writeWorkers struct {
doneCh chan struct{}
doneWaitGroup sync.WaitGroup
numWorkers int
tasksCh chan writeTask
}
type writeTask struct {
fileNum base.DiskFileNum
p []byte
offset int64
}
// Start starts the worker goroutines.
func (w *writeWorkers) Start(c *Cache, numWorkers int) {
doneCh := make(chan struct{})
tasksCh := make(chan writeTask, numWorkers*writeTasksPerWorker)
w.numWorkers = numWorkers
w.doneCh = doneCh
w.tasksCh = tasksCh
w.doneWaitGroup.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
defer w.doneWaitGroup.Done()
for {
select {
case <-doneCh:
return
case task, ok := <-tasksCh:
if !ok {
// The tasks channel was closed; this is used in testing code to
// ensure all writes are completed.
return
}
// TODO(radu): set() can perform multiple writes; perhaps each one
// should be its own task.
start := time.Now()
err := c.set(task.fileNum, task.p, task.offset)
c.metrics.putLatency.Observe(float64(time.Since(start)))
if err != nil {
c.metrics.writeBackFailures.Add(1)
// TODO(radu): throttle logs.
c.logger.Infof("writing back to cache after miss failed: %v", err)
}
}
}
}()
}
}
// Stop waits for any in-progress writes to complete and stops the worker
// goroutines and waits for any in-pro. Any queued writes not yet started are
// discarded.
func (w *writeWorkers) Stop() {
close(w.doneCh)
w.doneCh = nil
w.tasksCh = nil
w.doneWaitGroup.Wait()
}
// QueueWrite adds a write task to the queue. Can block if the queue is full.
func (w *writeWorkers) QueueWrite(fileNum base.DiskFileNum, p []byte, offset int64) {
w.tasksCh <- writeTask{
fileNum: fileNum,
p: p,
offset: offset,
}
}

56
vendor/github.com/cockroachdb/pebble/objstorage/vfs.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go generated vendored
View File

@@ -2,63 +2,63 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
package objstorageprovider
import (
"context"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
func (p *Provider) vfsPath(fileType base.FileType, fileNum base.FileNum) string {
func (p *provider) vfsPath(fileType base.FileType, fileNum base.DiskFileNum) string {
return base.MakeFilepath(p.st.FS, p.st.FSDirName, fileType, fileNum)
}
func (p *Provider) vfsOpenForReading(
fileType base.FileType, fileNum base.FileNum, mustExist bool,
) (Readable, error) {
func (p *provider) vfsOpenForReading(
ctx context.Context,
fileType base.FileType,
fileNum base.DiskFileNum,
opts objstorage.OpenOptions,
) (objstorage.Readable, error) {
filename := p.vfsPath(fileType, fileNum)
file, err := p.st.FS.Open(filename, vfs.RandomReadsOption)
if err != nil {
if mustExist {
if opts.MustExist {
base.MustExist(p.st.FS, filename, p.st.Logger, err)
}
return nil, err
}
// TODO(radu): we use the existence of the file descriptor as an indication
// that the File might support Prefetch and SequentialReadsOption. We should
// replace this with a cleaner way to obtain the capabilities of the FS / File.
if fd := file.Fd(); fd != vfs.InvalidFd {
return newFileReadable(file, p.st.FS, filename)
}
return newGenericFileReadable(file)
return newFileReadable(file, p.st.FS, filename)
}
func (p *Provider) vfsCreate(
fileType base.FileType, fileNum base.FileNum,
) (Writable, ObjectMetadata, error) {
func (p *provider) vfsCreate(
_ context.Context, fileType base.FileType, fileNum base.DiskFileNum,
) (objstorage.Writable, objstorage.ObjectMetadata, error) {
filename := p.vfsPath(fileType, fileNum)
file, err := p.st.FS.Create(filename)
if err != nil {
return nil, ObjectMetadata{}, err
return nil, objstorage.ObjectMetadata{}, err
}
file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
NoSyncOnClose: p.st.NoSyncOnClose,
BytesPerSync: p.st.BytesPerSync,
})
meta := ObjectMetadata{
FileNum: fileNum,
FileType: fileType,
meta := objstorage.ObjectMetadata{
DiskFileNum: fileNum,
FileType: fileType,
}
return newFileBufferedWritable(file), meta, nil
}
func (p *Provider) vfsRemove(fileType base.FileType, fileNum base.FileNum) error {
func (p *provider) vfsRemove(fileType base.FileType, fileNum base.DiskFileNum) error {
return p.st.FSCleaner.Clean(p.st.FS, fileType, p.vfsPath(fileType, fileNum))
}
// vfsInit finds any local FS objects.
func (p *Provider) vfsInit() error {
func (p *provider) vfsInit() error {
listing := p.st.FSDirInitialListing
if listing == nil {
var err error
@@ -71,17 +71,17 @@ func (p *Provider) vfsInit() error {
for _, filename := range listing {
fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename)
if ok && fileType == base.FileTypeTable {
o := ObjectMetadata{
FileType: fileType,
FileNum: fileNum,
o := objstorage.ObjectMetadata{
FileType: fileType,
DiskFileNum: fileNum,
}
p.mu.knownObjects[o.FileNum] = o
p.mu.knownObjects[o.DiskFileNum] = o
}
}
return nil
}
func (p *Provider) vfsSync() error {
func (p *provider) vfsSync() error {
p.mu.Lock()
shouldSync := p.mu.localObjectsChanged
p.mu.localObjectsChanged = false
@@ -99,7 +99,7 @@ func (p *Provider) vfsSync() error {
return nil
}
func (p *Provider) vfsSize(fileType base.FileType, fileNum base.FileNum) (int64, error) {
func (p *provider) vfsSize(fileType base.FileType, fileNum base.DiskFileNum) (int64, error) {
filename := p.vfsPath(fileType, fileNum)
stat, err := p.st.FS.Stat(filename)
if err != nil {

126
vendor/github.com/cockroachdb/pebble/objstorage/vfs_readable.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go generated vendored
View File

@@ -2,19 +2,25 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
package objstorageprovider
import (
"context"
"fmt"
"os"
"sync"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// fileReadable implements objstorage.Readable on top of a vfs.File that
// supports Prefetch and/or on an FS that supports SequentialReadsOption.
const fileMaxReadaheadSize = 256 * 1024 /* 256KB */
// fileReadable implements objstorage.Readable on top of a vfs.File.
//
// The implementation might use Prealloc and might reopen the file with
// SequentialReadsOption.
type fileReadable struct {
file vfs.File
size int64
@@ -25,7 +31,7 @@ type fileReadable struct {
fs vfs.FS
}
var _ Readable = (*fileReadable)(nil)
var _ objstorage.Readable = (*fileReadable)(nil)
func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, error) {
info, err := file.Stat()
@@ -48,8 +54,12 @@ func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable,
}
// ReadAt is part of the objstorage.Readable interface.
func (r *fileReadable) ReadAt(p []byte, off int64) (n int, err error) {
return r.file.ReadAt(p, off)
func (r *fileReadable) ReadAt(_ context.Context, p []byte, off int64) error {
n, err := r.file.ReadAt(p, off)
if invariants.Enabled && err == nil && n != len(p) {
panic("short read")
}
return err
}
// Close is part of the objstorage.Readable interface.
@@ -64,10 +74,10 @@ func (r *fileReadable) Size() int64 {
}
// NewReadHandle is part of the objstorage.Readable interface.
func (r *fileReadable) NewReadHandle() ReadHandle {
func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
rh := readHandlePool.Get().(*vfsReadHandle)
rh.r = r
rh.rs = makeReadaheadState()
rh.rs = makeReadaheadState(fileMaxReadaheadSize)
return rh
}
@@ -82,7 +92,7 @@ type vfsReadHandle struct {
sequentialFile vfs.File
}
var _ ReadHandle = (*vfsReadHandle)(nil)
var _ objstorage.ReadHandle = (*vfsReadHandle)(nil)
var readHandlePool = sync.Pool{
New: func() interface{} {
@@ -110,25 +120,36 @@ func (rh *vfsReadHandle) Close() error {
}
// ReadAt is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) ReadAt(p []byte, offset int64) (n int, err error) {
func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error {
var n int
var err error
if rh.sequentialFile != nil {
// Use OS-level read-ahead.
return rh.sequentialFile.ReadAt(p, offset)
}
if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
if readaheadSize >= maxReadaheadSize {
// We've reached the maximum readahead size. Beyond this point, rely on
// OS-level readahead.
rh.MaxReadahead()
} else {
_ = rh.r.file.Prefetch(offset, readaheadSize)
n, err = rh.sequentialFile.ReadAt(p, offset)
} else {
if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
if readaheadSize >= fileMaxReadaheadSize {
// We've reached the maximum readahead size. Beyond this point, rely on
// OS-level readahead.
rh.switchToOSReadahead()
} else {
_ = rh.r.file.Prefetch(offset, readaheadSize)
}
}
n, err = rh.r.file.ReadAt(p, offset)
}
return rh.r.file.ReadAt(p, offset)
if invariants.Enabled && err == nil && n != len(p) {
panic("short read")
}
return err
}
// MaxReadahead is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) MaxReadahead() {
// SetupForCompaction is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) SetupForCompaction() {
rh.switchToOSReadahead()
}
func (rh *vfsReadHandle) switchToOSReadahead() {
if rh.sequentialFile != nil {
return
}
@@ -142,7 +163,7 @@ func (rh *vfsReadHandle) MaxReadahead() {
}
// RecordCacheHit is part of the objstorage.ReadHandle interface.
func (rh *vfsReadHandle) RecordCacheHit(offset, size int64) {
func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
if rh.sequentialFile != nil {
// Using OS-level readahead, so do nothing.
return
@@ -150,60 +171,9 @@ func (rh *vfsReadHandle) RecordCacheHit(offset, size int64) {
rh.rs.recordCacheHit(offset, size)
}
// genericFileReadable implements objstorage.Readable on top of any vfs.File.
// This implementation does not support read-ahead.
type genericFileReadable struct {
file vfs.File
size int64
rh NoopReadHandle
}
var _ Readable = (*genericFileReadable)(nil)
func newGenericFileReadable(file vfs.File) (*genericFileReadable, error) {
info, err := file.Stat()
if err != nil {
return nil, err
}
r := &genericFileReadable{
file: file,
size: info.Size(),
rh: MakeNoopReadHandle(file),
}
invariants.SetFinalizer(r, func(obj interface{}) {
if obj.(*genericFileReadable).file != nil {
fmt.Fprintf(os.Stderr, "Readable was not closed")
os.Exit(1)
}
})
return r, nil
}
// ReadAt is part of the objstorage.Readable interface.
func (r *genericFileReadable) ReadAt(p []byte, off int64) (n int, err error) {
return r.file.ReadAt(p, off)
}
// Close is part of the objstorage.Readable interface.
func (r *genericFileReadable) Close() error {
defer func() { r.file = nil }()
return r.file.Close()
}
// Size is part of the objstorage.Readable interface.
func (r *genericFileReadable) Size() int64 {
return r.size
}
// NewReadHandle is part of the objstorage.Readable interface.
func (r *genericFileReadable) NewReadHandle() ReadHandle {
return &r.rh
}
// TestingCheckMaxReadahead returns true if the ReadHandle has switched to
// OS-level read-ahead.
func TestingCheckMaxReadahead(rh ReadHandle) bool {
func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool {
switch rh := rh.(type) {
case *vfsReadHandle:
return rh.sequentialFile != nil
@@ -234,11 +204,13 @@ func (rh *PreallocatedReadHandle) Close() error {
// but uses the existing storage of a PreallocatedReadHandle when possible
// (currently this happens if we are reading from a local file).
// The returned handle still needs to be closed.
func UsePreallocatedReadHandle(readable Readable, rh *PreallocatedReadHandle) ReadHandle {
func UsePreallocatedReadHandle(
ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
) objstorage.ReadHandle {
if r, ok := readable.(*fileReadable); ok {
// See fileReadable.NewReadHandle.
rh.vfsReadHandle = vfsReadHandle{r: r}
return rh
}
return readable.NewReadHandle()
return readable.NewReadHandle(ctx)
}

7
vendor/github.com/cockroachdb/pebble/objstorage/vfs_writable.go → vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go generated vendored
View File

@@ -2,16 +2,17 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
package objstorageprovider
import (
"bufio"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/vfs"
)
// NewFileWritable returns a Writable that uses a file as underlying storage.
func NewFileWritable(file vfs.File) Writable {
func NewFileWritable(file vfs.File) objstorage.Writable {
return newFileBufferedWritable(file)
}
@@ -20,7 +21,7 @@ type fileBufferedWritable struct {
bw *bufio.Writer
}
var _ Writable = (*fileBufferedWritable)(nil)
var _ objstorage.Writable = (*fileBufferedWritable)(nil)
func newFileBufferedWritable(file vfs.File) *fileBufferedWritable {
return &fileBufferedWritable{

446
vendor/github.com/cockroachdb/pebble/objstorage/provider.go generated vendored
View File

@@ -1,446 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import (
"io"
"os"
"sort"
"sync"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/objstorage/sharedobjcat"
"github.com/cockroachdb/pebble/vfs"
)
// Provider is a singleton object used to access and manage objects.
//
// An object is conceptually like a large immutable file. The main use of
// objects is for storing sstables; in the future it could also be used for blob
// storage.
//
// The Provider can only manage objects that it knows about - either objects
// created by the provider, or existing objects the Provider was informed about
// via AddObjects.
//
// Objects are currently backed by a vfs.File.
type Provider struct {
st Settings
fsDir vfs.File
shared sharedSubsystem
mu struct {
sync.RWMutex
shared struct {
// catalogBatch accumulates shared object creations and deletions until
// Sync is called.
catalogBatch sharedobjcat.Batch
}
// localObjectsChanged is set if non-shared objects were created or deleted
// but Sync was not yet called.
localObjectsChanged bool
// knownObjects maintains information about objects that are known to the provider.
// It is initialized with the list of files in the manifest when we open a DB.
knownObjects map[base.FileNum]ObjectMetadata
}
}
// Readable is the handle for an object that is open for reading.
type Readable interface {
io.ReaderAt
io.Closer
// Size returns the size of the object.
Size() int64
// NewReadHandle creates a read handle for ReadAt requests that are related
// and can benefit from optimizations like read-ahead.
//
// The ReadHandle must be closed before the Readable is closed.
//
// Multiple separate ReadHandles can be used.
NewReadHandle() ReadHandle
}
// ReadHandle is used to perform reads that are related and might benefit from
// optimizations like read-ahead.
type ReadHandle interface {
io.ReaderAt
io.Closer
// MaxReadahead configures the implementation to expect large sequential
// reads. Used to skip any initial read-ahead ramp-up.
MaxReadahead()
// RecordCacheHit informs the implementation that we were able to retrieve a
// block from cache.
RecordCacheHit(offset, size int64)
}
// Writable is the handle for an object that is open for writing.
// Either Finish or Abort must be called.
type Writable interface {
// Write writes len(p) bytes from p to the underlying object. The data is not
// guaranteed to be durable until Finish is called.
//
// Note that Write *is* allowed to modify the slice passed in, whether
// temporarily or permanently. Callers of Write need to take this into
// account.
Write(p []byte) error
// Finish completes the object and makes the data durable.
// No further calls are allowed after calling Finish.
Finish() error
// Abort gives up on finishing the object. There is no guarantee about whether
// the object exists after calling Abort.
// No further calls are allowed after calling Abort.
Abort()
}
// Settings that must be specified when creating the Provider.
type Settings struct {
Logger base.Logger
// Local filesystem configuration.
FS vfs.FS
FSDirName string
// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
//
// This is an optional optimization to avoid double listing on Open when the
// higher layer already has a listing. When nil, we obtain the listing on
// Open.
FSDirInitialListing []string
// Cleaner cleans obsolete files from the local filesystem.
//
// The default cleaner uses the DeleteCleaner.
FSCleaner base.Cleaner
// NoSyncOnClose decides whether the implementation will enforce a
// close-time synchronization (e.g., fdatasync() or sync_file_range())
// on files it writes to. Setting this to true removes the guarantee for a
// sync on close. Some implementations can still issue a non-blocking sync.
NoSyncOnClose bool
// BytesPerSync enables periodic syncing of files in order to smooth out
// writes to disk. This option does not provide any persistence guarantee, but
// is used to avoid latency spikes if the OS automatically decides to write
// out a large chunk of dirty filesystem buffers.
BytesPerSync int
// Fields here are set only if the provider is to support shared objects
// (experimental).
Shared struct {
Storage shared.Storage
}
}
// DefaultSettings initializes default settings (with no shared storage),
// suitable for tests and tools.
func DefaultSettings(fs vfs.FS, dirName string) Settings {
return Settings{
Logger: base.DefaultLogger,
FS: fs,
FSDirName: dirName,
FSCleaner: base.DeleteCleaner{},
NoSyncOnClose: false,
BytesPerSync: 512 * 1024, // 512KB
}
}
// ObjectMetadata contains the metadata required to be able to access an object.
type ObjectMetadata struct {
FileNum base.FileNum
FileType base.FileType
// The fields below are only set if the object is on shared storage.
Shared struct {
// CreatorID identifies the DB instance that originally created the object.
CreatorID CreatorID
// CreatorFileNum is the identifier for the object within the context of the
// DB instance that originally created the object.
CreatorFileNum base.FileNum
}
}
// CreatorID identifies the DB instance that originally created a shared object.
// This ID is incorporated in backing object names.
// Must be non-zero.
type CreatorID = sharedobjcat.CreatorID
// IsShared returns true if the object is on shared storage.
func (meta *ObjectMetadata) IsShared() bool {
return meta.Shared.CreatorID.IsSet()
}
// Open creates the Provider.
func Open(settings Settings) (p *Provider, _ error) {
fsDir, err := settings.FS.OpenDir(settings.FSDirName)
if err != nil {
return nil, err
}
defer func() {
if p == nil {
fsDir.Close()
}
}()
p = &Provider{
st: settings,
fsDir: fsDir,
}
p.mu.knownObjects = make(map[base.FileNum]ObjectMetadata)
// Add local FS objects.
if err := p.vfsInit(); err != nil {
return nil, err
}
// Initialize shared subsystem (if configured) and add shared objects.
if err := p.sharedInit(); err != nil {
return nil, err
}
return p, nil
}
// Close the provider.
func (p *Provider) Close() error {
var err error
if p.fsDir != nil {
err = p.fsDir.Close()
p.fsDir = nil
}
return err
}
// OpenForReading opens an existing object.
func (p *Provider) OpenForReading(fileType base.FileType, fileNum base.FileNum) (Readable, error) {
meta, err := p.Lookup(fileType, fileNum)
if err != nil {
return nil, err
}
if !meta.IsShared() {
return p.vfsOpenForReading(fileType, fileNum, false /* mustExist */)
}
return p.sharedOpenForReading(meta)
}
// OpenForReadingMustExist is a variant of OpenForReading which causes a fatal
// error if the file does not exist. The fatal error message contains
// information helpful for debugging.
func (p *Provider) OpenForReadingMustExist(
fileType base.FileType, fileNum base.FileNum,
) (Readable, error) {
meta, err := p.Lookup(fileType, fileNum)
if err != nil {
p.st.Logger.Fatalf("%v", err)
return nil, err
}
if !meta.IsShared() {
return p.vfsOpenForReading(fileType, fileNum, true /* mustExist */)
}
// TODO(radu): implement "must exist" behavior.
return p.sharedOpenForReading(meta)
}
// CreateOptions contains optional arguments for Create.
type CreateOptions struct {
// PreferSharedStorage causes the object to be created on shared storage if
// the provider has shared storage configured.
PreferSharedStorage bool
}
// Create creates a new object and opens it for writing.
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
func (p *Provider) Create(
fileType base.FileType, fileNum base.FileNum, opts CreateOptions,
) (w Writable, meta ObjectMetadata, err error) {
if opts.PreferSharedStorage && p.st.Shared.Storage != nil {
w, meta, err = p.sharedCreate(fileType, fileNum)
} else {
w, meta, err = p.vfsCreate(fileType, fileNum)
}
if err != nil {
err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum))
return nil, ObjectMetadata{}, err
}
p.addMetadata(meta)
return w, meta, nil
}
// Remove removes an object.
//
// The object is not guaranteed to be durably removed until Sync is called.
func (p *Provider) Remove(fileType base.FileType, fileNum base.FileNum) error {
meta, err := p.Lookup(fileType, fileNum)
if err != nil {
return err
}
if !meta.IsShared() {
err = p.vfsRemove(fileType, fileNum)
}
// TODO(radu): implement shared object removal (i.e. deref).
if err != nil && !IsNotExistError(err) {
// We want to be able to retry a Remove, so we keep the object in our list.
// TODO(radu): we should mark the object as "zombie" and not allow any other
// operations.
return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum))
}
p.removeMetadata(fileNum)
return err
}
// IsNotExistError indicates whether the error is known to report that a file or
// directory does not exist.
func IsNotExistError(err error) bool {
return oserror.IsNotExist(err)
}
// Sync flushes the metadata from creation or removal of objects since the last Sync.
func (p *Provider) Sync() error {
if err := p.vfsSync(); err != nil {
return err
}
if err := p.sharedSync(); err != nil {
return err
}
return nil
}
// LinkOrCopyFromLocal creates a new object that is either a copy of a given
// local file or a hard link (if the new object is created on the same FS, and
// if the FS supports it).
//
// The object is not guaranteed to be durable (accessible in case of crashes)
// until Sync is called.
func (p *Provider) LinkOrCopyFromLocal(
srcFS vfs.FS, srcFilePath string, dstFileType base.FileType, dstFileNum base.FileNum,
) (ObjectMetadata, error) {
if srcFS == p.st.FS {
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{
NoSyncOnClose: p.st.NoSyncOnClose,
BytesPerSync: p.st.BytesPerSync,
})
dstPath := p.vfsPath(dstFileType, dstFileNum)
if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
return ObjectMetadata{}, err
}
meta := ObjectMetadata{
FileNum: dstFileNum,
FileType: dstFileType,
}
p.addMetadata(meta)
return meta, nil
}
// TODO(radu): for the copy case, we should use `p.Create` and do the copy ourselves.
panic("unimplemented")
}
// Lookup returns the metadata of an object that is already known to the Provider.
// Does not perform any I/O.
func (p *Provider) Lookup(fileType base.FileType, fileNum base.FileNum) (ObjectMetadata, error) {
p.mu.RLock()
defer p.mu.RUnlock()
meta, ok := p.mu.knownObjects[fileNum]
if !ok {
return ObjectMetadata{}, errors.Wrapf(
os.ErrNotExist,
"file %s (type %d) unknown to the objstorage provider",
errors.Safe(fileNum), errors.Safe(fileType),
)
}
if meta.FileType != fileType {
return ObjectMetadata{}, errors.AssertionFailedf(
"file %s type mismatch (known type %d, expected type %d)",
errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType),
)
}
return meta, nil
}
// Path returns an internal, implementation-dependent path for the object. It is
// meant to be used for informational purposes (like logging).
func (p *Provider) Path(meta ObjectMetadata) string {
if !meta.IsShared() {
return p.vfsPath(meta.FileType, meta.FileNum)
}
return p.sharedPath(meta)
}
// Size returns the size of the object.
func (p *Provider) Size(meta ObjectMetadata) (int64, error) {
if !meta.IsShared() {
return p.vfsSize(meta.FileType, meta.FileNum)
}
return p.sharedSize(meta)
}
// List returns the objects currently known to the provider. Does not perform any I/O.
func (p *Provider) List() []ObjectMetadata {
p.mu.RLock()
defer p.mu.RUnlock()
res := make([]ObjectMetadata, 0, len(p.mu.knownObjects))
for _, meta := range p.mu.knownObjects {
res = append(res, meta)
}
sort.Slice(res, func(i, j int) bool {
return res[i].FileNum < res[j].FileNum
})
return res
}
func (p *Provider) addMetadata(meta ObjectMetadata) {
p.mu.Lock()
defer p.mu.Unlock()
p.mu.knownObjects[meta.FileNum] = meta
if meta.IsShared() {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
FileNum: meta.FileNum,
FileType: meta.FileType,
CreatorID: meta.Shared.CreatorID,
CreatorFileNum: meta.Shared.CreatorFileNum,
})
} else {
p.mu.localObjectsChanged = true
}
}
func (p *Provider) removeMetadata(fileNum base.FileNum) {
p.mu.Lock()
defer p.mu.Unlock()
meta, ok := p.mu.knownObjects[fileNum]
if !ok {
return
}
delete(p.mu.knownObjects, fileNum)
if meta.IsShared() {
p.mu.shared.catalogBatch.DeleteObject(fileNum)
} else {
p.mu.localObjectsChanged = true
}
}

25
vendor/github.com/cockroachdb/pebble/objstorage/remote/factory.go generated vendored Normal file
View File

@@ -0,0 +1,25 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package remote
import "github.com/pkg/errors"
// MakeSimpleFactory returns a StorageFactory implementation that produces the given
// Storage objects.
func MakeSimpleFactory(m map[Locator]Storage) StorageFactory {
return simpleFactory(m)
}
type simpleFactory map[Locator]Storage
var _ StorageFactory = simpleFactory{}
// CreateStorage is part of the StorageFactory interface.
func (sf simpleFactory) CreateStorage(locator Locator) (Storage, error) {
if s, ok := sf[locator]; ok {
return s, nil
}
return nil, errors.Errorf("unknown locator '%s'", locator)
}

118
vendor/github.com/cockroachdb/pebble/objstorage/remote/localfs.go generated vendored Normal file
View File

@@ -0,0 +1,118 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package remote
import (
"context"
"io"
"os"
"path"
"github.com/cockroachdb/pebble/vfs"
)
// NewLocalFS returns a vfs-backed implementation of the remote.Storage
// interface (for testing). All objects will be stored at the directory
// dirname.
func NewLocalFS(dirname string, fs vfs.FS) Storage {
store := &localFSStore{
dirname: dirname,
vfs: fs,
}
return store
}
// localFSStore is a vfs-backed implementation of the remote.Storage
// interface (for testing).
type localFSStore struct {
dirname string
vfs vfs.FS
}
var _ Storage = (*localFSStore)(nil)
// Close is part of the remote.Storage interface.
func (s *localFSStore) Close() error {
*s = localFSStore{}
return nil
}
// ReadObject is part of the remote.Storage interface.
func (s *localFSStore) ReadObject(
ctx context.Context, objName string,
) (_ ObjectReader, objSize int64, _ error) {
f, err := s.vfs.Open(path.Join(s.dirname, objName))
if err != nil {
return nil, 0, err
}
stat, err := f.Stat()
if err != nil {
return nil, 0, err
}
return &localFSReader{f}, stat.Size(), nil
}
type localFSReader struct {
file vfs.File
}
var _ ObjectReader = (*localFSReader)(nil)
// ReadAt is part of the shared.ObjectReader interface.
func (r *localFSReader) ReadAt(_ context.Context, p []byte, offset int64) error {
n, err := r.file.ReadAt(p, offset)
// https://pkg.go.dev/io#ReaderAt
if err == io.EOF && n == len(p) {
return nil
}
return err
}
// Close is part of the shared.ObjectReader interface.
func (r *localFSReader) Close() error {
r.file.Close()
r.file = nil
return nil
}
// CreateObject is part of the remote.Storage interface.
func (s *localFSStore) CreateObject(objName string) (io.WriteCloser, error) {
file, err := s.vfs.Create(path.Join(s.dirname, objName))
return file, err
}
// List is part of the remote.Storage interface.
func (s *localFSStore) List(prefix, delimiter string) ([]string, error) {
// TODO(josh): For the intended use case of localfs.go of running 'pebble bench',
// List can always return <nil, nil>, since this indicates a file has only one ref,
// and since `pebble bench` implies running in a single-pebble-instance context.
// https://github.com/cockroachdb/pebble/blob/a9a079d4fb6bf4a9ebc52e4d83a76ad4cbf676cb/objstorage/objstorageprovider/shared.go#L292
return nil, nil
}
// Delete is part of the remote.Storage interface.
func (s *localFSStore) Delete(objName string) error {
return s.vfs.Remove(path.Join(s.dirname, objName))
}
// Size is part of the remote.Storage interface.
func (s *localFSStore) Size(objName string) (int64, error) {
f, err := s.vfs.Open(path.Join(s.dirname, objName))
if err != nil {
return 0, err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return 0, err
}
return stat.Size(), nil
}
// IsNotExistError is part of the remote.Storage interface.
func (s *localFSStore) IsNotExistError(err error) bool {
return err == os.ErrNotExist
}

66
vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go → vendor/github.com/cockroachdb/pebble/objstorage/remote/logging.go generated vendored
View File

@@ -2,9 +2,10 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
package remote
import (
"context"
"fmt"
"io"
"sort"
@@ -19,7 +20,7 @@ func WithLogging(wrapped Storage, logf func(fmt string, args ...interface{})) St
}
}
// loggingStore wraps a shared.Storage implementation and emits logs of the
// loggingStore wraps a remote.Storage implementation and emits logs of the
// operations.
type loggingStore struct {
logf func(fmt string, args ...interface{})
@@ -33,41 +34,52 @@ func (l *loggingStore) Close() error {
return l.wrapped.Close()
}
func (l *loggingStore) ReadObjectAt(
basename string, offset int64,
) (_ io.ReadCloser, totalSize int64, _ error) {
r, totalSize, err := l.wrapped.ReadObjectAt(basename, offset)
l.logf("read object %q at %d: %s", basename, offset, errOrPrintf(err, "%d bytes", totalSize))
func (l *loggingStore) ReadObject(
ctx context.Context, objName string,
) (_ ObjectReader, objSize int64, _ error) {
r, size, err := l.wrapped.ReadObject(ctx, objName)
l.logf("create reader for object %q: %s", objName, errOrPrintf(err, "%d bytes", size))
if err != nil {
return nil, 0, err
}
return &loggingReader{
l: l,
name: basename,
ReadCloser: r,
}, totalSize, nil
l: l,
name: objName,
wrapped: r,
}, size, nil
}
type loggingReader struct {
l *loggingStore
name string
io.ReadCloser
l *loggingStore
name string
wrapped ObjectReader
}
var _ ObjectReader = (*loggingReader)(nil)
func (l *loggingReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
if err := l.wrapped.ReadAt(ctx, p, offset); err != nil {
l.l.logf("read object %q at %d (length %d): error %v", l.name, offset, len(p), err)
return err
}
l.l.logf("read object %q at %d (length %d)", l.name, offset, len(p))
return nil
}
func (l *loggingReader) Close() error {
l.l.logf("close reader for %q", l.name)
return l.ReadCloser.Close()
return l.wrapped.Close()
}
func (l *loggingStore) CreateObject(basename string) (io.WriteCloser, error) {
l.logf("create object %q", basename)
writer, err := l.wrapped.CreateObject(basename)
func (l *loggingStore) CreateObject(objName string) (io.WriteCloser, error) {
l.logf("create object %q", objName)
writer, err := l.wrapped.CreateObject(objName)
if err != nil {
return nil, err
}
return &loggingWriter{
l: l,
name: basename,
name: objName,
WriteCloser: writer,
}, nil
}
@@ -104,14 +116,14 @@ func (l *loggingStore) List(prefix, delimiter string) ([]string, error) {
return res, nil
}
func (l *loggingStore) Delete(basename string) error {
l.logf("delete object %q", basename)
return l.wrapped.Delete(basename)
func (l *loggingStore) Delete(objName string) error {
l.logf("delete object %q", objName)
return l.wrapped.Delete(objName)
}
func (l *loggingStore) Size(basename string) (int64, error) {
size, err := l.wrapped.Size(basename)
l.logf("size of object %q: %s", basename, errOrPrintf(err, "%d", size))
func (l *loggingStore) Size(objName string) (int64, error) {
size, err := l.wrapped.Size(objName)
l.logf("size of object %q: %s", objName, errOrPrintf(err, "%d", size))
return size, err
}
@@ -121,3 +133,7 @@ func errOrPrintf(err error, format string, args ...interface{}) string {
}
return fmt.Sprintf(format, args...)
}
func (l *loggingStore) IsNotExistError(err error) bool {
return l.wrapped.IsNotExistError(err)
}

57
vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go → vendor/github.com/cockroachdb/pebble/objstorage/remote/mem.go generated vendored
View File

@@ -2,17 +2,18 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
package remote
import (
"bytes"
"context"
"io"
"os"
"strings"
"sync"
)
// NewInMem returns an in-memory implementation of the shared.Storage
// NewInMem returns an in-memory implementation of the remote.Storage
// interface (for testing).
func NewInMem() Storage {
store := &inMemStore{}
@@ -20,7 +21,7 @@ func NewInMem() Storage {
return store
}
// inMemStore is an in-memory implementation of the shared.Storage interface
// inMemStore is an in-memory implementation of the remote.Storage interface
// (for testing).
type inMemStore struct {
mu struct {
@@ -41,39 +42,39 @@ func (s *inMemStore) Close() error {
return nil
}
func (s *inMemStore) ReadObjectAt(basename string, offset int64) (io.ReadCloser, int64, error) {
obj, err := s.getObj(basename)
func (s *inMemStore) ReadObject(
ctx context.Context, objName string,
) (_ ObjectReader, objSize int64, _ error) {
obj, err := s.getObj(objName)
if err != nil {
return nil, 0, err
}
remaining := int64(len(obj.data)) - offset
if remaining < 0 {
return nil, 0, io.EOF
}
return newInMemReader(obj.data[int(offset):]), remaining, nil
return &inMemReader{data: obj.data}, int64(len(obj.data)), nil
}
type inMemReader struct {
bytes.Reader
data []byte
}
func newInMemReader(b []byte) *inMemReader {
r := &inMemReader{}
r.Reset(b)
return r
}
var _ ObjectReader = (*inMemReader)(nil)
var _ io.ReadCloser = (*inMemReader)(nil)
func (r *inMemReader) Close() error {
r.Reset(nil)
func (r *inMemReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
if offset+int64(len(p)) > int64(len(r.data)) {
return io.EOF
}
copy(p, r.data[offset:])
return nil
}
func (s *inMemStore) CreateObject(basename string) (io.WriteCloser, error) {
func (r *inMemReader) Close() error {
r.data = nil
return nil
}
func (s *inMemStore) CreateObject(objName string) (io.WriteCloser, error) {
return &inMemWriter{
store: s,
name: basename,
name: objName,
}, nil
}
@@ -119,20 +120,24 @@ func (s *inMemStore) List(prefix, delimiter string) ([]string, error) {
return res, nil
}
func (s *inMemStore) Delete(basename string) error {
s.rmObj(basename)
func (s *inMemStore) Delete(objName string) error {
s.rmObj(objName)
return nil
}
// Size returns the length of the named object in bytesWritten.
func (s *inMemStore) Size(basename string) (int64, error) {
obj, err := s.getObj(basename)
func (s *inMemStore) Size(objName string) (int64, error) {
obj, err := s.getObj(objName)
if err != nil {
return 0, err
}
return int64(len(obj.data)), nil
}
func (s *inMemStore) IsNotExistError(err error) bool {
return err == os.ErrNotExist
}
func (s *inMemStore) getObj(name string) (*inMemObj, error) {
s.mu.Lock()
defer s.mu.Unlock()

126
vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go generated vendored Normal file
View File

@@ -0,0 +1,126 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package remote
import (
"context"
"io"
)
// Locator is an opaque string identifying a remote.Storage implementation.
//
// The Locator must not contain secrets (like authentication keys). Locators are
// stored on disk in the shared object catalog and are passed around as part of
// RemoteObjectBacking; they can also appear in error messages.
type Locator string
// StorageFactory is used to return Storage implementations based on locators. A
// Pebble store that uses shared storage is configured with a StorageFactory.
type StorageFactory interface {
CreateStorage(locator Locator) (Storage, error)
}
// SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will
// have sstables shared across Pebble instances when doing skip-shared
// iteration (see db.ScanInternal) or shared file ingestion (see
// db.IngestAndExcise).
const SharedLevelsStart = 5
// CreateOnSharedStrategy specifies what table files should be created on shared
// storage. For use with CreateOnShared in options.
type CreateOnSharedStrategy int
const (
// CreateOnSharedNone denotes no files being created on shared storage.
CreateOnSharedNone CreateOnSharedStrategy = iota
// CreateOnSharedLower denotes the creation of files in lower levels of the
// LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on
// shared storage, and higher levels on local storage.
CreateOnSharedLower
// CreateOnSharedAll denotes the creation of all sstables on shared storage.
CreateOnSharedAll
)
// ShouldCreateShared returns whether new table files at the specified level
// should be created on shared storage.
func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool {
switch strategy {
case CreateOnSharedAll:
return true
case CreateOnSharedNone:
return false
case CreateOnSharedLower:
return level >= SharedLevelsStart
default:
panic("unexpected CreateOnSharedStrategy value")
}
}
// Storage is an interface for a blob storage driver. This is lower-level
// than an FS-like interface, however FS/File-like abstractions can be built on
// top of these methods.
//
// TODO(bilal): Consider pushing shared file obsoletion as well as path
// generation behind this interface.
type Storage interface {
io.Closer
// ReadObject returns an ObjectReader that can be used to perform reads on an
// object, along with the total size of the object.
ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error)
// CreateObject returns a writer for the object at the request name. A new
// empty object is created if CreateObject is called on an existing object.
//
// A Writer *must* be closed via either Close, and if closing returns a
// non-nil error, that error should be handled or reported to the user -- an
// implementation may buffer written data until Close and only then return
// an error, or Write may return an opaque io.EOF with the underlying cause
// returned by the subsequent Close().
//
// TODO(radu): if we encounter some unrelated error while writing to the
// WriteCloser, we'd want to abort the whole thing rather than letting Close
// finalize the upload.
CreateObject(objName string) (io.WriteCloser, error)
// List enumerates files within the supplied prefix, returning a list of
// objects within that prefix. If delimiter is non-empty, names which have the
// same prefix, prior to the delimiter but after the prefix, are grouped into a
// single result which is that prefix. The order that results are returned is
// undefined. If a prefix is specified, the prefix is trimmed from the result
// list.
//
// An example would be, if the storage contains objects a, b/4, b/5 and b/6,
// these would be the return values:
// List("", "") -> ["a", "b/4", "b/5", "b/6"]
// List("", "/") -> ["a", "b"]
// List("b", "/") -> ["4", "5", "6"]
// List("b", "") -> ["/4", "/5", "/6"]
List(prefix, delimiter string) ([]string, error)
// Delete removes the named object from the store.
Delete(objName string) error
// Size returns the length of the named object in bytesWritten.
Size(objName string) (int64, error)
// IsNotExistError returns true if the given error (returned by a method in
// this interface) indicates that the object does not exist.
IsNotExistError(err error) bool
}
// ObjectReader is used to perform reads on an object.
type ObjectReader interface {
// ReadAt reads len(p) bytes into p starting at offset off.
//
// Does not return partial results; if offset + len(p) is past the end of the
// object, an error is returned.
//
// Clients of ReadAt can execute parallel ReadAt calls on the same
// ObjectReader.
ReadAt(ctx context.Context, p []byte, offset int64) error
Close() error
}

176
vendor/github.com/cockroachdb/pebble/objstorage/shared.go generated vendored
View File

@@ -1,176 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import (
"fmt"
"sync"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage/sharedobjcat"
)
// sharedSubsystem contains the provider fields related to shared storage.
// All fields remain unset if shared storage is not configured.
type sharedSubsystem struct {
catalog *sharedobjcat.Catalog
// initialized guards access to the creatorID field.
initialized atomic.Bool
creatorID CreatorID
initOnce sync.Once
}
func (ss *sharedSubsystem) init(creatorID CreatorID) {
ss.initOnce.Do(func() {
ss.creatorID = creatorID
ss.initialized.Store(true)
})
}
// sharedInit initializes the shared object subsystem (if configured) and finds
// any shared objects.
func (p *Provider) sharedInit() error {
if p.st.Shared.Storage == nil {
return nil
}
catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName)
if err != nil {
return errors.Wrapf(err, "pebble: could not open shared object catalog")
}
p.shared.catalog = catalog
// The creator ID may or may not be initialized yet.
if contents.CreatorID.IsSet() {
p.shared.init(contents.CreatorID)
p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID)
} else {
p.st.Logger.Infof("shared storage configured; no creatorID yet")
}
for _, meta := range contents.Objects {
o := ObjectMetadata{
FileNum: meta.FileNum,
FileType: meta.FileType,
}
o.Shared.CreatorID = meta.CreatorID
o.Shared.CreatorFileNum = meta.CreatorFileNum
p.mu.knownObjects[o.FileNum] = o
}
return nil
}
// SetCreatorID sets the CreatorID which is needed in order to use shared
// objects. Shared object usage is disabled until this method is called the
// first time. Once set, the Creator ID is persisted and cannot change.
//
// Cannot be called if shared storage is not configured for the provider.
func (p *Provider) SetCreatorID(creatorID CreatorID) error {
if p.st.Shared.Storage == nil {
return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled")
}
// Note: this call is a cheap no-op if the creator ID was already set. This
// call also checks if we are trying to change the ID.
if err := p.shared.catalog.SetCreatorID(creatorID); err != nil {
return err
}
if !p.shared.initialized.Load() {
p.st.Logger.Infof("shared storage creatorID set to %s", creatorID)
p.shared.init(creatorID)
}
return nil
}
func (p *Provider) sharedCheckInitialized() error {
if p.st.Shared.Storage == nil {
return errors.Errorf("shared object support not configured")
}
if !p.shared.initialized.Load() {
return errors.Errorf("shared object support not available: shared creator ID not yet set")
}
return nil
}
func (p *Provider) sharedSync() error {
batch := func() sharedobjcat.Batch {
p.mu.Lock()
defer p.mu.Unlock()
res := p.mu.shared.catalogBatch.Copy()
p.mu.shared.catalogBatch.Reset()
return res
}()
if batch.IsEmpty() {
return nil
}
err := p.shared.catalog.ApplyBatch(batch)
if err != nil {
// We have to put back the batch (for the next Sync).
p.mu.Lock()
defer p.mu.Unlock()
batch.Append(p.mu.shared.catalogBatch)
p.mu.shared.catalogBatch = batch
return err
}
return nil
}
func (p *Provider) sharedPath(meta ObjectMetadata) string {
return "shared://" + sharedObjectName(meta)
}
func sharedObjectName(meta ObjectMetadata) string {
// TODO(radu): prepend a "shard" value for better distribution within the bucket?
return fmt.Sprintf(
"%s-%s",
meta.Shared.CreatorID, base.MakeFilename(meta.FileType, meta.Shared.CreatorFileNum),
)
}
func (p *Provider) sharedCreate(
fileType base.FileType, fileNum base.FileNum,
) (Writable, ObjectMetadata, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, ObjectMetadata{}, err
}
meta := ObjectMetadata{
FileNum: fileNum,
FileType: fileType,
}
meta.Shared.CreatorID = p.shared.creatorID
meta.Shared.CreatorFileNum = fileNum
objName := sharedObjectName(meta)
writer, err := p.st.Shared.Storage.CreateObject(objName)
if err != nil {
return nil, ObjectMetadata{}, err
}
return &sharedWritable{
storageWriter: writer,
}, meta, nil
}
func (p *Provider) sharedOpenForReading(meta ObjectMetadata) (Readable, error) {
if err := p.sharedCheckInitialized(); err != nil {
return nil, err
}
objName := sharedObjectName(meta)
size, err := p.st.Shared.Storage.Size(objName)
if err != nil {
return nil, err
}
return newSharedReadable(p.st.Shared.Storage, objName, size), nil
}
func (p *Provider) sharedSize(meta ObjectMetadata) (int64, error) {
if err := p.sharedCheckInitialized(); err != nil {
return 0, err
}
objName := sharedObjectName(meta)
return p.st.Shared.Storage.Size(objName)
}

61
vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go generated vendored
View File

@@ -1,61 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package shared
import "io"
// Storage is an interface for a blob storage driver. This is lower-level
// than an FS-like interface, however FS/File-like abstractions can be built on
// top of these methods.
//
// TODO(bilal): Consider pushing shared file obsoletion as well as path
// generation behind this interface.
type Storage interface {
io.Closer
// ReadObjectAt returns a Reader for reading the object at the requested name
// and offset, along with the total size of the object.
ReadObjectAt(basename string, offset int64) (_ io.ReadCloser, totalSize int64, _ error)
// CreateObject returns a writer for the object at the request name. A new
// empty object is created if CreateObject is called on an existing object.
//
// A Writer *must* be closed via either Close, and if closing returns a
// non-nil error, that error should be handled or reported to the user -- an
// implementation may buffer written data until Close and only then return
// an error, or Write may return an opaque io.EOF with the underlying cause
// returned by the subsequent Close().
//
// TODO(radu): if we encounter some unrelated error while writing to the
// WriteCloser, we'd want to abort the whole thing rather than letting Close
// finalize the upload.
CreateObject(basename string) (io.WriteCloser, error)
// List enumerates files within the supplied prefix, returning a list of
// objects within that prefix. If delimiter is non-empty, names which have the
// same prefix, prior to the delimiter but after the prefix, are grouped into a
// single result which is that prefix. The order that results are returned is
// undefined. If a prefix is specified, the prefix is trimmed from the result
// list.
//
// An example would be, if the storage contains objects a, b/4, b/5 and b/6,
// these would be the return values:
// List("", "") -> ["a", "b/4", "b/5", "b/6"]
// List("", "/") -> ["a", "b"]
// List("b", "/") -> ["4", "5", "6"]
// List("b", "") -> ["/4", "/5", "/6"]
List(prefix, delimiter string) ([]string, error)
// Delete removes the named object from the store.
//
// TODO(radu): do we get an error when the object isn't there? How can we
// check for that error?
Delete(basename string) error
// Size returns the length of the named object in bytesWritten.
//
// TODO(radu): same as above - how can we tell if it's a "no such object" error?
Size(basename string) (int64, error)
}

145
vendor/github.com/cockroachdb/pebble/objstorage/shared_backing.go generated vendored
View File

@@ -1,145 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import (
"bytes"
"encoding/binary"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/objstorage/sharedobjcat"
)
// SharedObjectBacking encodes the metadata necessary to incorporate a shared
// object into a different Pebble instance.
type SharedObjectBacking []byte
const (
tagCreatorID = 1
tagCreatorFileNum = 2
// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
// followed by the length of the data (so they can be skipped).
// Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if
// they are encountered by earlier code that doesn't know the tag.
tagNotSafeToIgnoreMask = 64
)
// SharedObjectBacking encodes the shared object metadata.
func (meta *ObjectMetadata) SharedObjectBacking() (SharedObjectBacking, error) {
if !meta.IsShared() {
return nil, errors.AssertionFailedf("object %s not on shared storage", meta.FileNum)
}
buf := make([]byte, 0, binary.MaxVarintLen64*4)
buf = binary.AppendUvarint(buf, tagCreatorID)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID))
buf = binary.AppendUvarint(buf, tagCreatorFileNum)
buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum))
return buf, nil
}
// fromSharedObjectBacking decodes the shared object metadata.
func fromSharedObjectBacking(
fileType base.FileType, fileNum base.FileNum, buf SharedObjectBacking,
) (ObjectMetadata, error) {
var creatorID uint64
var creatorFileNum uint64
br := bytes.NewReader(buf)
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return ObjectMetadata{}, err
}
switch tag {
case tagCreatorID:
creatorID, err = binary.ReadUvarint(br)
case tagCreatorFileNum:
creatorFileNum, err = binary.ReadUvarint(br)
// TODO(radu): encode file type as well?
default:
// Ignore unknown tags, unless they're not safe to ignore.
if tag&tagNotSafeToIgnoreMask != 0 {
return ObjectMetadata{}, errors.Newf("unknown tag %d", tag)
}
var dataLen uint64
dataLen, err = binary.ReadUvarint(br)
if err == nil {
_, err = br.Seek(int64(dataLen), io.SeekCurrent)
}
}
if err != nil {
return ObjectMetadata{}, err
}
}
if creatorID == 0 {
return ObjectMetadata{}, errors.Newf("shared object backing missing creator ID")
}
if creatorFileNum == 0 {
return ObjectMetadata{}, errors.Newf("shared object backing missing creator file num")
}
meta := ObjectMetadata{
FileNum: fileNum,
FileType: fileType,
}
meta.Shared.CreatorID = CreatorID(creatorID)
meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum)
return meta, nil
}
// SharedObjectToAttach contains the arguments needed to attach an existing shared object.
type SharedObjectToAttach struct {
// FileNum is the file number that will be used to refer to this object (in
// the context of this instance).
FileNum base.FileNum
FileType base.FileType
// Backing contains the metadata for the share dobject backing (normally
// generated from a different instance).
Backing SharedObjectBacking
}
// AttachSharedObjects registers existing shared objects with this provider.
func (p *Provider) AttachSharedObjects(objs []SharedObjectToAttach) ([]ObjectMetadata, error) {
metas := make([]ObjectMetadata, len(objs))
for i, o := range objs {
meta, err := fromSharedObjectBacking(o.FileType, o.FileNum, o.Backing)
if err != nil {
return nil, err
}
metas[i] = meta
}
func() {
p.mu.Lock()
defer p.mu.Unlock()
for _, meta := range metas {
p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
FileNum: meta.FileNum,
FileType: meta.FileType,
CreatorID: meta.Shared.CreatorID,
CreatorFileNum: meta.Shared.CreatorFileNum,
})
}
}()
if err := p.sharedSync(); err != nil {
return nil, err
}
p.mu.Lock()
defer p.mu.Unlock()
for _, meta := range metas {
p.mu.knownObjects[meta.FileNum] = meta
}
return metas, nil
}

97
vendor/github.com/cockroachdb/pebble/objstorage/shared_readable.go generated vendored
View File

@@ -1,97 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import (
"io"
"github.com/cockroachdb/pebble/objstorage/shared"
)
// sharedReadable is a very simple implementation of Readable on top of the
// ReadCloser returned by shared.Storage.CreateObject.
type sharedReadable struct {
storage shared.Storage
objName string
size int64
// rh is used for direct ReadAt calls without a read handle.
rh sharedReadHandle
}
var _ Readable = (*sharedReadable)(nil)
func newSharedReadable(storage shared.Storage, objName string, size int64) *sharedReadable {
r := &sharedReadable{
storage: storage,
objName: objName,
size: size,
}
r.rh.readable = r
return r
}
func (r *sharedReadable) ReadAt(p []byte, offset int64) (n int, err error) {
return r.rh.ReadAt(p, offset)
}
func (r *sharedReadable) Close() error {
err := r.rh.Close()
r.storage = nil
return err
}
func (r *sharedReadable) Size() int64 {
return r.size
}
func (r *sharedReadable) NewReadHandle() ReadHandle {
// TODO(radu): use a pool.
return &sharedReadHandle{readable: r}
}
type sharedReadHandle struct {
readable *sharedReadable
lastReader io.ReadCloser
lastOffset int64
}
var _ ReadHandle = (*sharedReadHandle)(nil)
func (r *sharedReadHandle) ReadAt(p []byte, offset int64) (n int, err error) {
// See if this continues the previous read so that we can reuse the last reader.
if r.lastReader == nil || r.lastOffset != offset {
// We need to create a new reader.
if r.lastReader != nil {
if err := r.lastReader.Close(); err != nil {
return 0, err
}
r.lastReader = nil
}
reader, _, err := r.readable.storage.ReadObjectAt(r.readable.objName, offset)
if err != nil {
return 0, err
}
r.lastReader = reader
r.lastOffset = offset
}
n, err = io.ReadFull(r.lastReader, p)
r.lastOffset += int64(n)
return n, err
}
func (r *sharedReadHandle) Close() error {
var err error
if r.lastReader != nil {
err = r.lastReader.Close()
r.lastReader = nil
}
r.readable = nil
return err
}
func (r *sharedReadHandle) MaxReadahead() {}
func (r *sharedReadHandle) RecordCacheHit(offset, size int64) {}

34
vendor/github.com/cockroachdb/pebble/objstorage/shared_writable.go generated vendored
View File

@@ -1,34 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package objstorage
import "io"
// sharedWritable is a very simple implementation of Writable on top of the
// WriteCloser returned by shared.Storage.CreateObject.
type sharedWritable struct {
storageWriter io.WriteCloser
}
var _ Writable = (*sharedWritable)(nil)
// Write is part of the Writable interface.
func (w *sharedWritable) Write(p []byte) error {
_, err := w.storageWriter.Write(p)
return err
}
// Finish is part of the Writable interface.
func (w *sharedWritable) Finish() error {
err := w.storageWriter.Close()
w.storageWriter = nil
return err
}
// Abort is part of the Writable interface.
func (w *sharedWritable) Abort() {
_ = w.storageWriter.Close()
w.storageWriter = nil
}

158
vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/version_edit.go generated vendored
View File

@@ -1,158 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package sharedobjcat
import (
"bufio"
"encoding/binary"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
)
// versionEdit is a modification to the shared object state which can be encoded
// into a record.
//
// TODO(radu): consider adding creation and deletion time for debugging purposes.
type versionEdit struct {
NewObjects []SharedObjectMetadata
DeletedObjects []base.FileNum
CreatorID CreatorID
}
const (
// tagNewObject is followed by the FileNum, creator ID and creator FileNum.
tagNewObject = 1
// tagDeletedObject is followed by the FileNum.
tagDeletedObject = 2
// tagCreatorID is followed by the Creator ID for this store. This ID can
// never change.
tagCreatorID = 3
)
// Object type values. We don't want to encode FileType directly because it is
// more general (and we want freedom to change it in the future).
const (
objTypeTable = 1
)
func objTypeToFileType(objType uint64) (base.FileType, error) {
switch objType {
case objTypeTable:
return base.FileTypeTable, nil
default:
return 0, errors.Newf("unknown object type %d", objType)
}
}
func fileTypeToObjType(fileType base.FileType) (uint64, error) {
switch fileType {
case base.FileTypeTable:
return objTypeTable, nil
default:
return 0, errors.Newf("unknown object type for file type %d", fileType)
}
}
// Encode encodes an edit to the specified writer.
func (v *versionEdit) Encode(w io.Writer) error {
buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2))
for _, meta := range v.NewObjects {
objType, err := fileTypeToObjType(meta.FileType)
if err != nil {
return err
}
buf = binary.AppendUvarint(buf, uint64(tagNewObject))
buf = binary.AppendUvarint(buf, uint64(meta.FileNum))
buf = binary.AppendUvarint(buf, objType)
buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum))
}
for _, fileNum := range v.DeletedObjects {
buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
buf = binary.AppendUvarint(buf, uint64(fileNum))
}
if v.CreatorID.IsSet() {
buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
}
_, err := w.Write(buf)
return err
}
// Decode decodes an edit from the specified reader.
func (v *versionEdit) Decode(r io.Reader) error {
br, ok := r.(io.ByteReader)
if !ok {
br = bufio.NewReader(r)
}
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return err
}
err = nil
switch tag {
case tagNewObject:
var fileNum, creatorID, creatorFileNum uint64
var fileType base.FileType
fileNum, err = binary.ReadUvarint(br)
if err == nil {
var objType uint64
objType, err = binary.ReadUvarint(br)
if err == nil {
fileType, err = objTypeToFileType(objType)
}
}
if err == nil {
creatorID, err = binary.ReadUvarint(br)
}
if err == nil {
creatorFileNum, err = binary.ReadUvarint(br)
}
if err == nil {
v.NewObjects = append(v.NewObjects, SharedObjectMetadata{
FileNum: base.FileNum(fileNum),
FileType: fileType,
CreatorID: CreatorID(creatorID),
CreatorFileNum: base.FileNum(creatorFileNum),
})
}
case tagDeletedObject:
var fileNum uint64
fileNum, err = binary.ReadUvarint(br)
if err == nil {
v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum))
}
case tagCreatorID:
var id uint64
id, err = binary.ReadUvarint(br)
if err == nil {
v.CreatorID = CreatorID(id)
}
default:
// Ignore unknown tags.
}
if err != nil {
if err == io.EOF {
return errCorruptCatalog
}
return err
}
}
return nil
}
var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog")

224
vendor/github.com/cockroachdb/pebble/open.go generated vendored
View File

@@ -6,6 +6,7 @@ package pebble
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
@@ -20,11 +21,14 @@ import (
"github.com/cockroachdb/pebble/internal/arenaskl"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/constants"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/internal/manual"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
"github.com/prometheus/client_golang/prometheus"
)
@@ -34,11 +38,23 @@ const (
// The max batch size is limited by the uint32 offsets stored in
// internal/batchskl.node, DeferredBatchOp, and flushableBatchEntry.
maxBatchSize = 4 << 30 // 4 GB
//
// We limit the size to MaxUint32 (just short of 4GB) so that the exclusive
// end of an allocation fits in uint32.
//
// On 32-bit systems, slices are naturally limited to MaxInt (just short of
// 2GB).
maxBatchSize = constants.MaxUint32OrInt
// The max memtable size is limited by the uint32 offsets stored in
// internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry.
maxMemTableSize = 4 << 30 // 4 GB
//
// We limit the size to MaxUint32 (just short of 4GB) so that the exclusive
// end of an allocation fits in uint32.
//
// On 32-bit systems, slices are naturally limited to MaxInt (just short of
// 2GB).
maxMemTableSize = constants.MaxUint32OrInt
)
// TableCacheSize can be used to determine the table
@@ -157,7 +173,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
merge: opts.Merger.Merge,
split: opts.Comparer.Split,
abbreviatedKey: opts.Comparer.AbbreviatedKey,
largeBatchThreshold: (opts.MemTableSize - int(memTableEmptySize)) / 2,
largeBatchThreshold: (opts.MemTableSize - uint64(memTableEmptySize)) / 2,
fileLock: fileLock,
dataDir: dataDir,
walDir: walDir,
@@ -166,8 +182,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
closedCh: make(chan struct{}),
}
d.mu.versions = &versionSet{}
d.atomic.diskAvailBytes = math.MaxUint64
d.mu.versions.diskAvailBytes = d.getDiskAvailableBytesCached
d.diskAvailBytes.Store(math.MaxUint64)
defer func() {
// If an error or panic occurs during open, attempt to release the manually
@@ -192,6 +207,9 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
t.arenaBuf = nil
}
}
if d.cleanupManager != nil {
d.cleanupManager.Close()
}
if d.objProvider != nil {
d.objProvider.Close()
}
@@ -202,34 +220,30 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}()
d.commit = newCommitPipeline(commitEnv{
logSeqNum: &d.mu.versions.atomic.logSeqNum,
visibleSeqNum: &d.mu.versions.atomic.visibleSeqNum,
logSeqNum: &d.mu.versions.logSeqNum,
visibleSeqNum: &d.mu.versions.visibleSeqNum,
apply: d.commitApply,
write: d.commitWrite,
})
if d.opts.Experimental.MinDeletionRate > 0 {
d.deletionPacer = newDeletionPacer(int64(d.opts.Experimental.MinDeletionRate), d.getDeletionPacerInfo)
} else {
d.deletionPacer = nilPacer
}
d.mu.nextJobID = 1
d.mu.mem.nextSize = opts.MemTableSize
if d.mu.mem.nextSize > initialMemTableSize {
d.mu.mem.nextSize = initialMemTableSize
}
d.mu.mem.cond.L = &d.mu.Mutex
d.mu.cleaner.cond.L = &d.mu.Mutex
d.mu.compact.cond.L = &d.mu.Mutex
d.mu.compact.inProgress = make(map[*compaction]struct{})
d.mu.compact.noOngoingFlushStartTime = time.Now()
d.mu.snapshots.init()
// logSeqNum is the next sequence number that will be assigned. Start
// assigning sequence numbers from 1 to match rocksdb.
d.mu.versions.atomic.logSeqNum = 1
// logSeqNum is the next sequence number that will be assigned.
// Start assigning sequence numbers from base.SeqNumStart to leave
// room for reserved sequence numbers (see comments around
// SeqNumStart).
d.mu.versions.logSeqNum.Store(base.SeqNumStart)
d.mu.formatVers.vers.Store(uint64(formatVersion))
d.mu.formatVers.marker = formatVersionMarker
d.timeNow = time.Now
d.openedAt = d.timeNow()
d.mu.Lock()
defer d.mu.Unlock()
@@ -246,7 +260,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
// Create the DB.
if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
return nil, err
}
} else {
@@ -254,11 +268,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname)
}
// Load the version set.
if err := d.mu.versions.load(dirname, opts, manifestFileNum, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
return nil, err
}
if opts.ErrorIfNotPristine {
liveFileNums := make(map[FileNum]struct{})
liveFileNums := make(map[base.DiskFileNum]struct{})
d.mu.versions.addLiveFileNums(liveFileNums)
if len(liveFileNums) != 0 {
return nil, errors.Wrapf(ErrDBNotPristine, "dirname=%q", dirname)
@@ -271,7 +285,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
// sequence number of the first batch that will be inserted.
if !d.opts.ReadOnly {
var entry *flushableEntry
d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.atomic.logSeqNum)
d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.logSeqNum.Load())
d.mu.mem.queue = append(d.mu.mem.queue, entry)
}
@@ -287,7 +301,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
ls = append(ls, ls2...)
}
providerSettings := objstorage.Settings{
providerSettings := objstorageprovider.Settings{
Logger: opts.Logger,
FS: opts.FS,
FSDirName: dirname,
@@ -296,13 +310,18 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
NoSyncOnClose: opts.NoSyncOnClose,
BytesPerSync: opts.BytesPerSync,
}
providerSettings.Shared.Storage = opts.Experimental.SharedStorage
providerSettings.Remote.StorageFactory = opts.Experimental.RemoteStorage
providerSettings.Remote.CreateOnShared = opts.Experimental.CreateOnShared
providerSettings.Remote.CreateOnSharedLocator = opts.Experimental.CreateOnSharedLocator
providerSettings.Remote.CacheSizeBytes = opts.Experimental.SecondaryCacheSizeBytes
d.objProvider, err = objstorage.Open(providerSettings)
d.objProvider, err = objstorageprovider.Open(providerSettings)
if err != nil {
return nil, err
}
d.cleanupManager = openCleanupManager(opts, d.objProvider, d.onObsoleteTableDelete, d.getDeletionPacerInfo)
if manifestExists {
curVersion := d.mu.versions.currentVersion()
if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil {
@@ -331,21 +350,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
// Don't reuse any obsolete file numbers to avoid modifying an
// ingested sstable's original external file.
if d.mu.versions.nextFileNum <= fn {
d.mu.versions.nextFileNum = fn + 1
if d.mu.versions.nextFileNum <= fn.FileNum() {
d.mu.versions.nextFileNum = fn.FileNum() + 1
}
switch ft {
case fileTypeLog:
if fn >= d.mu.versions.minUnflushedLogNum {
logFiles = append(logFiles, fileNumAndName{fn, filename})
if fn.FileNum() >= d.mu.versions.minUnflushedLogNum {
logFiles = append(logFiles, fileNumAndName{fn.FileNum(), filename})
}
if d.logRecycler.minRecycleLogNum <= fn {
d.logRecycler.minRecycleLogNum = fn + 1
if d.logRecycler.minRecycleLogNum <= fn.FileNum() {
d.logRecycler.minRecycleLogNum = fn.FileNum() + 1
}
case fileTypeOptions:
if previousOptionsFileNum < fn {
previousOptionsFileNum = fn
if previousOptionsFileNum < fn.FileNum() {
previousOptionsFileNum = fn.FileNum()
previousOptionsFilename = filename
}
case fileTypeTemp, fileTypeOldTemp:
@@ -362,6 +381,15 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
}
// Ratchet d.mu.versions.nextFileNum ahead of all known objects in the
// objProvider. This avoids FileNum collisions with obsolete sstables.
objects := d.objProvider.List()
for _, obj := range objects {
if d.mu.versions.nextFileNum <= obj.DiskFileNum.FileNum() {
d.mu.versions.nextFileNum = obj.DiskFileNum.FileNum() + 1
}
}
// Validate the most-recent OPTIONS file, if there is one.
var strictWALTail bool
if previousOptionsFilename != "" {
@@ -387,11 +415,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
}
toFlush = append(toFlush, flush...)
d.mu.versions.markFileNumUsed(lf.num)
if d.mu.versions.atomic.logSeqNum < maxSeqNum {
d.mu.versions.atomic.logSeqNum = maxSeqNum
if d.mu.versions.logSeqNum.Load() < maxSeqNum {
d.mu.versions.logSeqNum.Store(maxSeqNum)
}
}
d.mu.versions.atomic.visibleSeqNum = d.mu.versions.atomic.logSeqNum
d.mu.versions.visibleSeqNum.Store(d.mu.versions.logSeqNum.Load())
if !d.opts.ReadOnly {
// Create an empty .log file.
@@ -417,8 +445,8 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
entry.readerUnrefLocked(true)
}
newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum)
d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: 0})
newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum.DiskFileNum())
d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum.DiskFileNum(), fileSize: 0})
logFile, err := opts.FS.Create(newLogName)
if err != nil {
return nil, err
@@ -469,7 +497,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
if !d.opts.ReadOnly {
// Write the current options to disk.
d.optionsFileNum = d.mu.versions.getNextFileNum()
d.optionsFileNum = d.mu.versions.getNextFileNum().DiskFileNum()
tmpPath := base.MakeFilepath(opts.FS, dirname, fileTypeTemp, d.optionsFileNum)
optionsPath := base.MakeFilepath(opts.FS, dirname, fileTypeOptions, d.optionsFileNum)
@@ -504,7 +532,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
if !d.opts.ReadOnly {
d.scanObsoleteFiles(ls)
d.deleteObsoleteFiles(jobID, true /* waitForOngoing */)
d.deleteObsoleteFiles(jobID)
} else {
// All the log files are obsolete.
d.mu.versions.metrics.WAL.Files = int64(len(logFiles))
@@ -613,9 +641,9 @@ func GetVersion(dir string, fs vfs.FS) (string, error) {
// processed, reset version. This is because rocksdb often
// writes multiple options files without deleting previous ones.
// Otherwise, skip parsing this options file.
if fn > lastOptionsSeen {
if fn.FileNum() > lastOptionsSeen {
version = ""
lastOptionsSeen = fn
lastOptionsSeen = fn.FileNum()
} else {
continue
}
@@ -677,8 +705,15 @@ func (d *DB) replayWAL(
rr = record.NewReader(file, logNum)
offset int64 // byte offset in rr
lastFlushOffset int64
keysReplayed int64 // number of keys replayed
batchesReplayed int64 // number of batches replayed
)
// TODO(jackson): This function is interspersed with panics, in addition to
// corruption error propagation. Audit them to ensure we're truly only
// panicking where the error points to Pebble bug and not user or
// hardware-induced corruption.
if d.opts.ReadOnly {
// In read-only mode, we replay directly into the mutable memtable which will
// never be flushed.
@@ -725,7 +760,7 @@ func (d *DB) replayWAL(
// TODO(bananabrick): See if we can use the actual base level here,
// instead of using 1.
c := newFlush(d.opts, d.mu.versions.currentVersion(),
1 /* base level */, toFlush)
1 /* base level */, toFlush, d.timeNow())
newVE, _, _, err := d.runCompaction(jobID, c)
if err != nil {
return errors.Wrapf(err, "running compaction during WAL replay")
@@ -733,6 +768,11 @@ func (d *DB) replayWAL(
ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...)
return nil
}
defer func() {
if err != nil {
err = errors.WithDetailf(err, "replaying log %s, offset %d", logNum, offset)
}
}()
for {
offset = rr.Offset()
@@ -765,26 +805,33 @@ func (d *DB) replayWAL(
// Specify Batch.db so that Batch.SetRepr will compute Batch.memTableSize
// which is used below.
b = Batch{db: d}
b = Batch{}
b.db = d
b.SetRepr(buf.Bytes())
seqNum := b.SeqNum()
maxSeqNum = seqNum + uint64(b.Count())
keysReplayed += int64(b.Count())
batchesReplayed++
{
br := b.Reader()
if kind, encodedFileNum, _, _ := br.Next(); kind == InternalKeyKindIngestSST {
fileNums := make([]FileNum, 0, b.Count())
if kind, encodedFileNum, _, ok, err := br.Next(); err != nil {
return nil, 0, err
} else if ok && kind == InternalKeyKindIngestSST {
fileNums := make([]base.DiskFileNum, 0, b.Count())
addFileNum := func(encodedFileNum []byte) {
fileNum, n := binary.Uvarint(encodedFileNum)
if n <= 0 {
panic("pebble: ingest sstable file num is invalid.")
}
fileNums = append(fileNums, base.FileNum(fileNum))
fileNums = append(fileNums, base.FileNum(fileNum).DiskFileNum())
}
addFileNum(encodedFileNum)
for i := 1; i < int(b.Count()); i++ {
kind, encodedFileNum, _, ok := br.Next()
kind, encodedFileNum, _, ok, err := br.Next()
if err != nil {
return nil, 0, err
}
if kind != InternalKeyKindIngestSST {
panic("pebble: invalid batch key kind.")
}
@@ -794,21 +841,41 @@ func (d *DB) replayWAL(
addFileNum(encodedFileNum)
}
if _, _, _, ok := br.Next(); ok {
if _, _, _, ok, err := br.Next(); err != nil {
return nil, 0, err
} else if ok {
panic("pebble: invalid number of entries in batch.")
}
paths := make([]string, len(fileNums))
meta := make([]*fileMetadata, len(fileNums))
for i, n := range fileNums {
paths[i] = base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
}
var readable objstorage.Readable
objMeta, err := d.objProvider.Lookup(fileTypeTable, n)
if err != nil {
return nil, 0, errors.Wrap(err, "pebble: error when looking up ingested SSTs")
}
if objMeta.IsRemote() {
readable, err = d.objProvider.OpenForReading(context.TODO(), fileTypeTable, n, objstorage.OpenOptions{MustExist: true})
if err != nil {
return nil, 0, errors.Wrap(err, "pebble: error when opening flushable ingest files")
}
} else {
path := base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
f, err := d.opts.FS.Open(path)
if err != nil {
return nil, 0, err
}
var meta []*manifest.FileMetadata
meta, _, err = ingestLoad(
d.opts, d.FormatMajorVersion(), paths, d.cacheID, fileNums,
)
if err != nil {
return nil, 0, err
readable, err = sstable.NewSimpleReadable(f)
if err != nil {
return nil, 0, err
}
}
// NB: ingestLoad1 will close readable.
meta[i], err = ingestLoad1(d.opts, d.FormatMajorVersion(), readable, d.cacheID, n)
if err != nil {
return nil, 0, errors.Wrap(err, "pebble: error when loading flushable ingest files")
}
}
if uint32(len(meta)) != b.Count() {
@@ -854,9 +921,10 @@ func (d *DB) replayWAL(
d.opts, d.mu.versions.currentVersion(),
1, /* base level */
[]*flushableEntry{entry},
d.timeNow(),
)
for _, file := range c.flushing[0].flushable.(*ingestedFlushable).files {
ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file})
ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file.FileMetadata})
}
}
return toFlush, maxSeqNum, nil
@@ -868,11 +936,14 @@ func (d *DB) replayWAL(
// Make a copy of the data slice since it is currently owned by buf and will
// be reused in the next iteration.
b.data = append([]byte(nil), b.data...)
b.flushable = newFlushableBatch(&b, d.opts.Comparer)
b.flushable, err = newFlushableBatch(&b, d.opts.Comparer)
if err != nil {
return nil, 0, err
}
entry := d.newFlushableEntry(b.flushable, logNum, b.SeqNum())
// Disable memory accounting by adding a reader ref that will never be
// removed.
entry.readerRefs++
entry.readerRefs.Add(1)
if d.opts.ReadOnly {
d.mu.mem.queue = append(d.mu.mem.queue, entry)
// We added the flushable batch to the flushable to the queue.
@@ -910,7 +981,10 @@ func (d *DB) replayWAL(
}
buf.Reset()
}
d.opts.Logger.Infof("[JOB %d] WAL file %s with log number %s stopped reading at offset: %d; replayed %d keys in %d batches", jobID, filename, logNum.String(), offset, keysReplayed, batchesReplayed)
flushMem()
// mem is nil here.
if !d.opts.ReadOnly {
err = updateVE()
@@ -990,7 +1064,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) {
// LockDirectory may be used to expand the critical section protected by the
// database lock to include setup before the call to Open.
func LockDirectory(dirname string, fs vfs.FS) (*Lock, error) {
fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, 0))
fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, base.FileNum(0).DiskFileNum()))
if err != nil {
return nil, err
}
@@ -1068,28 +1142,42 @@ func IsCorruptionError(err error) bool {
return errors.Is(err, base.ErrCorruption)
}
func checkConsistency(v *manifest.Version, dirname string, objProvider *objstorage.Provider) error {
func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error {
var buf bytes.Buffer
var args []interface{}
dedup := make(map[base.DiskFileNum]struct{})
for level, files := range v.Levels {
iter := files.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
meta, err := objProvider.Lookup(base.FileTypeTable, f.FileNum)
backingState := f.FileBacking
if _, ok := dedup[backingState.DiskFileNum]; ok {
continue
}
dedup[backingState.DiskFileNum] = struct{}{}
fileNum := backingState.DiskFileNum
fileSize := backingState.Size
// We allow foreign objects to have a mismatch between sizes. This is
// because we might skew the backing size stored by our objprovider
// to prevent us from over-prioritizing this file for compaction.
meta, err := objProvider.Lookup(base.FileTypeTable, fileNum)
var size int64
if err == nil {
if objProvider.IsSharedForeign(meta) {
continue
}
size, err = objProvider.Size(meta)
}
if err != nil {
buf.WriteString("L%d: %s: %v\n")
args = append(args, errors.Safe(level), errors.Safe(f.FileNum), err)
args = append(args, errors.Safe(level), errors.Safe(fileNum), err)
continue
}
if size != int64(f.Size) {
if size != int64(fileSize) {
buf.WriteString("L%d: %s: object size mismatch (%s): %d (disk) != %d (MANIFEST)\n")
args = append(args, errors.Safe(level), errors.Safe(f.FileNum), objProvider.Path(meta),
errors.Safe(size), errors.Safe(f.Size))
args = append(args, errors.Safe(level), errors.Safe(fileNum), objProvider.Path(meta),
errors.Safe(size), errors.Safe(fileSize))
continue
}
}

232
vendor/github.com/cockroachdb/pebble/options.go generated vendored
View File

@@ -17,8 +17,10 @@ import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/objstorage/shared"
"github.com/cockroachdb/pebble/objstorage/remote"
"github.com/cockroachdb/pebble/rangekey"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
)
@@ -117,11 +119,24 @@ type IterOptions struct {
// false to skip scanning. This function must be thread-safe since the same
// function can be used by multiple iterators, if the iterator is cloned.
TableFilter func(userProps map[string]string) bool
// SkipPoint may be used to skip over point keys that don't match an
// arbitrary predicate during iteration. If set, the Iterator invokes
// SkipPoint for keys encountered. If SkipPoint returns true, the iterator
// will skip the key without yielding it to the iterator operation in
// progress.
//
// SkipPoint must be a pure function and always return the same result when
// provided the same arguments. The iterator may call SkipPoint multiple
// times for the same user key.
SkipPoint func(userKey []byte) bool
// PointKeyFilters can be used to avoid scanning tables and blocks in tables
// when iterating over point keys. It is requires that this slice is sorted in
// increasing order of the BlockPropertyFilter.ShortID. This slice represents
// an intersection across all filters, i.e., all filters must indicate that the
// block is relevant.
// when iterating over point keys. This slice represents an intersection
// across all filters, i.e., all filters must indicate that the block is
// relevant.
//
// Performance note: When len(PointKeyFilters) > 0, the caller should ensure
// that cap(PointKeyFilters) is at least len(PointKeyFilters)+1. This helps
// avoid allocations in Pebble internal code that mutates the slice.
PointKeyFilters []BlockPropertyFilter
// RangeKeyFilters can be usefd to avoid scanning tables and blocks in tables
// when iterating over range keys. The same requirements that apply to
@@ -180,6 +195,10 @@ type IterOptions struct {
level manifest.Level
// disableLazyCombinedIteration is an internal testing option.
disableLazyCombinedIteration bool
// snapshotForHideObsoletePoints is specified for/by levelIter when opening
// files and is used to decide whether to hide obsolete points. A value of 0
// implies obsolete points should not be hidden.
snapshotForHideObsoletePoints uint64
// NB: If adding new Options, you must account for them in iterator
// construction and Iterator.SetOptions.
@@ -222,6 +241,38 @@ func (o *IterOptions) getLogger() Logger {
return o.logger
}
// SpanIterOptions creates a SpanIterOptions from this IterOptions.
func (o *IterOptions) SpanIterOptions() keyspan.SpanIterOptions {
if o == nil {
return keyspan.SpanIterOptions{}
}
return keyspan.SpanIterOptions{
RangeKeyFilters: o.RangeKeyFilters,
}
}
// scanInternalOptions is similar to IterOptions, meant for use with
// scanInternalIterator.
type scanInternalOptions struct {
IterOptions
visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error
visitRangeDel func(start, end []byte, seqNum uint64) error
visitRangeKey func(start, end []byte, keys []rangekey.Key) error
visitSharedFile func(sst *SharedSSTMeta) error
// skipSharedLevels skips levels that are shareable (level >=
// sharedLevelStart).
skipSharedLevels bool
// includeObsoleteKeys specifies whether keys shadowed by newer internal keys
// are exposed. If false, only one internal key per user key is exposed.
includeObsoleteKeys bool
// rateLimitFunc is used to limit the amount of bytes read per second.
rateLimitFunc func(key *InternalKey, value LazyValue) error
}
// RangeKeyMasking configures automatic hiding of point keys by range keys. A
// non-nil Suffix enables range-key masking. When enabled, range keys with
// suffixes ≥ Suffix behave as masks. All point keys that are contained within a
@@ -497,18 +548,12 @@ type Options struct {
// concurrent compaction is added. This works "on top" of
// L0CompactionConcurrency, so the higher of the count of compaction
// concurrency slots as determined by the two options is chosen.
CompactionDebtConcurrency int
CompactionDebtConcurrency uint64
// MinDeletionRate is the minimum number of bytes per second that would
// be deleted. Deletion pacing is used to slow down deletions when
// compactions finish up or readers close, and newly-obsolete files need
// cleaning up. Deleting lots of files at once can cause disk latency to
// go up on some SSDs, which this functionality guards against. This is a
// minimum as the maximum is theoretically unlimited; pacing is disabled
// when there are too many obsolete files relative to live bytes, or
// there isn't enough disk space available. Setting this to 0 disables
// deletion pacing, which is also the default.
MinDeletionRate int
// IngestSplit, if it returns true, allows for ingest-time splitting of
// existing sstables into two virtual sstables to allow ingestion sstables to
// slot into a lower level than they otherwise would have.
IngestSplit func() bool
// ReadCompactionRate controls the frequency of read triggered
// compactions by adjusting `AllowedSeeks` in manifest.FileMetadata:
@@ -573,10 +618,10 @@ type Options struct {
// desired size of each level of the LSM. Defaults to 10.
LevelMultiplier int
// MultiLevelCompactionHueristic determines whether to add an additional
// MultiLevelCompactionHeuristic determines whether to add an additional
// level to a conventional two level compaction. If nil, a multilevel
// compaction will never get triggered.
MultiLevelCompactionHueristic MultiLevelHeuristic
MultiLevelCompactionHeuristic MultiLevelHeuristic
// MaxWriterConcurrency is used to indicate the maximum number of
// compression workers the compression queue is allowed to use. If
@@ -597,8 +642,11 @@ type Options struct {
CPUWorkPermissionGranter CPUWorkPermissionGranter
// EnableValueBlocks is used to decide whether to enable writing
// TableFormatPebblev3 sstables. WARNING: do not return true yet, since
// support for TableFormatPebblev3 is incomplete and not production ready.
// TableFormatPebblev3 sstables. This setting is only respected by a
// specific subset of format major versions: FormatSSTableValueBlocks,
// FormatFlushableIngest and FormatPrePebblev1MarkedCompacted. In lower
// format major versions, value blocks are never enabled. In higher
// format major versions, value blocks are always enabled.
EnableValueBlocks func() bool
// ShortAttributeExtractor is used iff EnableValueBlocks() returns true
@@ -607,18 +655,24 @@ type Options struct {
ShortAttributeExtractor ShortAttributeExtractor
// RequiredInPlaceValueBound specifies an optional span of user key
// prefixes for which the values must be stored with the key. This is
// useful for statically known exclusions to value separation. In
// CockroachDB, this will be used for the lock table key space that has
// non-empty suffixes, but those locks don't represent actual MVCC
// versions (the suffix ordering is arbitrary). We will also need to add
// support for dynamically configured exclusions (we want the default to
// be to allow Pebble to decide whether to separate the value or not,
// hence this is structured as exclusions), for example, for users of
// CockroachDB to dynamically exclude certain tables.
// prefixes that are not-MVCC, but have a suffix. For these the values
// must be stored with the key, since the concept of "older versions" is
// not defined. It is also useful for statically known exclusions to value
// separation. In CockroachDB, this will be used for the lock table key
// space that has non-empty suffixes, but those locks don't represent
// actual MVCC versions (the suffix ordering is arbitrary). We will also
// need to add support for dynamically configured exclusions (we want the
// default to be to allow Pebble to decide whether to separate the value
// or not, hence this is structured as exclusions), for example, for users
// of CockroachDB to dynamically exclude certain tables.
//
// Any change in exclusion behavior takes effect only on future written
// sstables, and does not start rewriting existing sstables.
//
// Even ignoring changes in this setting, exclusions are interpreted as a
// guidance by Pebble, and not necessarily honored. Specifically, user
// keys with multiple Pebble-versions *may* have the older versions stored
// in value blocks.
RequiredInPlaceValueBound UserKeyPrefixBound
// DisableIngestAsFlushable disables lazy ingestion of sstables through
@@ -626,15 +680,49 @@ type Options struct {
// major version is at least `FormatFlushableIngest`.
DisableIngestAsFlushable func() bool
// SharedStorage is a second FS-like storage medium that can be shared
// between multiple Pebble instances. It is used to store sstables only, and
// is managed by objstorage.Provider. Each sstable might only be written to
// by one Pebble instance, but other Pebble instances can possibly read the
// same files if they have the path to get to them. The pebble instance that
// wrote a file should not delete it if other Pebble instances are known to
// be reading this file. This FS is expected to have slower read/write
// performance than the default FS above.
SharedStorage shared.Storage
// RemoteStorage enables use of remote storage (e.g. S3) for storing
// sstables. Setting this option enables use of CreateOnShared option and
// allows ingestion of external files.
RemoteStorage remote.StorageFactory
// If CreateOnShared is non-zero, new sstables are created on remote storage
// (using CreateOnSharedLocator and with the appropriate
// CreateOnSharedStrategy). These sstables can be shared between different
// Pebble instances; the lifecycle of such objects is managed by the
// remote.Storage constructed by options.RemoteStorage.
//
// Can only be used when RemoteStorage is set (and recognizes
// CreateOnSharedLocator).
CreateOnShared remote.CreateOnSharedStrategy
CreateOnSharedLocator remote.Locator
// CacheSizeBytesBytes is the size of the on-disk block cache for objects
// on shared storage in bytes. If it is 0, no cache is used.
SecondaryCacheSizeBytes int64
// IneffectualPointDeleteCallback is called in compactions/flushes if any
// single delete is being elided without deleting a point set/merge.
IneffectualSingleDeleteCallback func(userKey []byte)
// SingleDeleteInvariantViolationCallback is called in compactions/flushes if any
// single delete has consumed a Set/Merge, and there is another immediately older
// Set/SetWithDelete/Merge. The user of Pebble has violated the invariant under
// which SingleDelete can be used correctly.
//
// Consider the sequence SingleDelete#3, Set#2, Set#1. There are three
// ways some of these keys can first meet in a compaction.
//
// - All 3 keys in the same compaction: this callback will detect the
// violation.
//
// - SingleDelete#3, Set#2 meet in a compaction first: Both keys will
// disappear. The violation will not be detected, and the DB will have
// Set#1 which is likely incorrect (from the user's perspective).
//
// - Set#2, Set#1 meet in a compaction first: The output will be Set#2,
// which will later be consumed by SingleDelete#3. The violation will
// not be detected and the DB will be correct.
SingleDeleteInvariantViolationCallback func(userKey []byte)
}
// Filters is a map from filter policy name to filter policy. It is used for
@@ -749,12 +837,18 @@ type Options struct {
// writing the contents of the old one in the
// background. MemTableStopWritesThreshold places a hard limit on the size of
// the queued MemTables.
MemTableSize int
//
// The default value is 4MB.
MemTableSize uint64
// Hard limit on the size of queued of MemTables. Writes are stopped when the
// sum of the queued memtable sizes exceeds
// MemTableStopWritesThreshold*MemTableSize. This value should be at least 2
// or writes will stop whenever a MemTable is being flushed.
// Hard limit on the number of queued of MemTables. Writes are stopped when
// the sum of the queued memtable sizes exceeds:
// MemTableStopWritesThreshold * MemTableSize.
//
// This value should be at least 2 or writes will stop whenever a MemTable is
// being flushed.
//
// The default value is 2.
MemTableStopWritesThreshold int
// Merger defines the associative merge operation to use for merging values
@@ -839,6 +933,20 @@ type Options struct {
// changing options dynamically?
WALMinSyncInterval func() time.Duration
// TargetByteDeletionRate is the rate (in bytes per second) at which sstable file
// deletions are limited to (under normal circumstances).
//
// Deletion pacing is used to slow down deletions when compactions finish up
// or readers close and newly-obsolete files need cleaning up. Deleting lots
// of files at once can cause disk latency to go up on some SSDs, which this
// functionality guards against.
//
// This value is only a best-effort target; the effective rate can be
// higher if deletions are falling behind or disk space is running low.
//
// Setting this to 0 disables deletion pacing, which is also the default.
TargetByteDeletionRate int
// private options are only used by internal tests or are used internally
// for facilitating upgrade paths of unconfigurable functionality.
private struct {
@@ -872,6 +980,10 @@ type Options struct {
// A private option to disable stats collection.
disableTableStats bool
// testingAlwaysWaitForCleanup is set by some tests to force waiting for
// obsolete file deletion (to make events deterministic).
testingAlwaysWaitForCleanup bool
// fsCloser holds a closer that should be invoked after a DB using these
// Options is closed. This is used to automatically stop the
// long-running goroutine associated with the disk-health-checking FS.
@@ -983,7 +1095,7 @@ func (o *Options) EnsureDefaults() *Options {
o.MaxOpenFiles = 1000
}
if o.MemTableSize <= 0 {
o.MemTableSize = 4 << 20
o.MemTableSize = 4 << 20 // 4 MB
}
if o.MemTableStopWritesThreshold <= 0 {
o.MemTableStopWritesThreshold = 2
@@ -1024,8 +1136,8 @@ func (o *Options) EnsureDefaults() *Options {
if o.Experimental.CPUWorkPermissionGranter == nil {
o.Experimental.CPUWorkPermissionGranter = defaultCPUWorkGranter{}
}
if o.Experimental.MultiLevelCompactionHueristic == nil {
o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{}
if o.Experimental.MultiLevelCompactionHeuristic == nil {
o.Experimental.MultiLevelCompactionHeuristic = WriteAmpHeuristic{}
}
o.initMaps()
@@ -1144,7 +1256,7 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, " max_open_files=%d\n", o.MaxOpenFiles)
fmt.Fprintf(&buf, " mem_table_size=%d\n", o.MemTableSize)
fmt.Fprintf(&buf, " mem_table_stop_writes_threshold=%d\n", o.MemTableStopWritesThreshold)
fmt.Fprintf(&buf, " min_deletion_rate=%d\n", o.Experimental.MinDeletionRate)
fmt.Fprintf(&buf, " min_deletion_rate=%d\n", o.TargetByteDeletionRate)
fmt.Fprintf(&buf, " merger=%s\n", o.Merger.Name)
fmt.Fprintf(&buf, " read_compaction_rate=%d\n", o.Experimental.ReadCompactionRate)
fmt.Fprintf(&buf, " read_sampling_multiplier=%d\n", o.Experimental.ReadSamplingMultiplier)
@@ -1165,6 +1277,8 @@ func (o *Options) String() string {
fmt.Fprintf(&buf, " wal_bytes_per_sync=%d\n", o.WALBytesPerSync)
fmt.Fprintf(&buf, " max_writer_concurrency=%d\n", o.Experimental.MaxWriterConcurrency)
fmt.Fprintf(&buf, " force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism)
fmt.Fprintf(&buf, " secondary_cache_size_bytes=%d\n", o.Experimental.SecondaryCacheSizeBytes)
fmt.Fprintf(&buf, " create_on_shared=%d\n", o.Experimental.CreateOnShared)
// Private options.
//
@@ -1322,7 +1436,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
}
}
case "compaction_debt_concurrency":
o.Experimental.CompactionDebtConcurrency, err = strconv.Atoi(value)
o.Experimental.CompactionDebtConcurrency, err = strconv.ParseUint(value, 10, 64)
case "delete_range_flush_delay":
// NB: This is a deprecated serialization of the
// `flush_delay_delete_range`.
@@ -1354,7 +1468,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
// version is valid right here.
var v uint64
v, err = strconv.ParseUint(value, 10, 64)
if vers := FormatMajorVersion(v); vers > FormatNewest || vers == FormatDefault {
if vers := FormatMajorVersion(v); vers > internalFormatNewest || vers == FormatDefault {
err = errors.Newf("unknown format major version %d", o.FormatMajorVersion)
}
if err == nil {
@@ -1387,14 +1501,14 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
case "max_open_files":
o.MaxOpenFiles, err = strconv.Atoi(value)
case "mem_table_size":
o.MemTableSize, err = strconv.Atoi(value)
o.MemTableSize, err = strconv.ParseUint(value, 10, 64)
case "mem_table_stop_writes_threshold":
o.MemTableStopWritesThreshold, err = strconv.Atoi(value)
case "min_compaction_rate":
// Do nothing; option existed in older versions of pebble, and
// may be meaningful again eventually.
case "min_deletion_rate":
o.Experimental.MinDeletionRate, err = strconv.Atoi(value)
o.TargetByteDeletionRate, err = strconv.Atoi(value)
case "min_flush_rate":
// Do nothing; option existed in older versions of pebble, and
// may be meaningful again eventually.
@@ -1438,6 +1552,12 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
o.Experimental.MaxWriterConcurrency, err = strconv.Atoi(value)
case "force_writer_parallelism":
o.Experimental.ForceWriterParallelism, err = strconv.ParseBool(value)
case "secondary_cache_size_bytes":
o.Experimental.SecondaryCacheSizeBytes, err = strconv.ParseInt(value, 10, 64)
case "create_on_shared":
var createOnSharedInt int64
createOnSharedInt, err = strconv.ParseInt(value, 10, 64)
o.Experimental.CreateOnShared = remote.CreateOnSharedStrategy(createOnSharedInt)
default:
if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
return nil
@@ -1568,15 +1688,15 @@ func (o *Options) Validate() error {
}
if uint64(o.MemTableSize) >= maxMemTableSize {
fmt.Fprintf(&buf, "MemTableSize (%s) must be < %s\n",
humanize.Uint64(uint64(o.MemTableSize)), humanize.Uint64(maxMemTableSize))
humanize.Bytes.Uint64(uint64(o.MemTableSize)), humanize.Bytes.Uint64(maxMemTableSize))
}
if o.MemTableStopWritesThreshold < 2 {
fmt.Fprintf(&buf, "MemTableStopWritesThreshold (%d) must be >= 2\n",
o.MemTableStopWritesThreshold)
}
if o.FormatMajorVersion > FormatNewest {
if o.FormatMajorVersion > internalFormatNewest {
fmt.Fprintf(&buf, "FormatMajorVersion (%d) must be <= %d\n",
o.FormatMajorVersion, FormatNewest)
o.FormatMajorVersion, internalFormatNewest)
}
if o.TableCache != nil && o.Cache != o.TableCache.cache {
fmt.Fprintf(&buf, "underlying cache in the TableCache and the Cache dont match\n")
@@ -1596,6 +1716,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions {
readerOpts.Comparer = o.Comparer
readerOpts.Filters = o.Filters
if o.Merger != nil {
readerOpts.Merge = o.Merger.Merge
readerOpts.MergerName = o.Merger.Name
}
readerOpts.LoggerAndTracer = o.LoggerAndTracer
@@ -1620,6 +1741,9 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab
if format >= sstable.TableFormatPebblev3 {
writerOpts.ShortAttributeExtractor = o.Experimental.ShortAttributeExtractor
writerOpts.RequiredInPlaceValueBound = o.Experimental.RequiredInPlaceValueBound
if format >= sstable.TableFormatPebblev4 && level == numLevels-1 {
writerOpts.WritingToLowestLevel = true
}
}
levelOpts := o.Level(level)
writerOpts.BlockRestartInterval = levelOpts.BlockRestartInterval

179
vendor/github.com/cockroachdb/pebble/pacer.go generated vendored
View File

@@ -7,30 +7,11 @@ package pebble
import (
"sync"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/rate"
)
var nilPacer = &noopPacer{}
type limiter interface {
DelayN(now time.Time, n int) time.Duration
AllowN(now time.Time, n int) bool
Burst() int
}
// pacer is the interface for flush and compaction rate limiters. The rate limiter
// is possible applied on each iteration step of a flush or compaction. This is to
// limit background IO usage so that it does not contend with foreground traffic.
type pacer interface {
maybeThrottle(bytesIterated uint64) error
reportDeletion(bytesToDelete uint64)
}
// deletionPacerInfo contains any info from the db necessary to make deletion
// pacing decisions.
// pacing decisions (to limit background IO usage so that it does not contend
// with foreground traffic).
type deletionPacerInfo struct {
freeBytes uint64
obsoleteBytes uint64
@@ -43,12 +24,17 @@ type deletionPacerInfo struct {
// negatively impacted if too many blocks are deleted very quickly, so this
// mechanism helps mitigate that.
type deletionPacer struct {
limiter limiter
freeSpaceThreshold uint64
obsoleteBytesMaxRatio float64
targetByteDeletionRate int64
// If there are less than freeSpaceThreshold bytes of free space on
// disk, increase the pace of deletions such that we delete enough bytes to
// get back to the threshold within the freeSpaceTimeframe.
freeSpaceThreshold uint64
freeSpaceTimeframe time.Duration
getInfo func() deletionPacerInfo
// If the ratio of obsolete bytes to live bytes is greater than
// obsoleteBytesMaxRatio, increase the pace of deletions such that we delete
// enough bytes to get back to the threshold within the obsoleteBytesTimeframe.
obsoleteBytesMaxRatio float64
obsoleteBytesTimeframe time.Duration
mu struct {
sync.Mutex
@@ -57,110 +43,93 @@ type deletionPacer struct {
// deletion rate to match the pace of deletions.
history history
}
targetByteDeletionRate int64
getInfo func() deletionPacerInfo
}
const deletePacerHistory = 5 * time.Minute
// newDeletionPacer instantiates a new deletionPacer for use when deleting
// obsolete files. The limiter passed in must be a singleton shared across this
// pebble instance.
// obsolete files.
//
// targetByteDeletionRate is the rate (in bytes/sec) at which we want to
// normally limit deletes (when we are not falling behind or running out of
// space). A value of 0.0 disables pacing.
func newDeletionPacer(
targetByteDeletionRate int64, getInfo func() deletionPacerInfo,
now time.Time, targetByteDeletionRate int64, getInfo func() deletionPacerInfo,
) *deletionPacer {
d := &deletionPacer{
limiter: rate.NewLimiter(rate.Limit(targetByteDeletionRate), int(targetByteDeletionRate)),
// If there are less than freeSpaceThreshold bytes of free space on
// disk, do not pace deletions at all.
freeSpaceThreshold: 16 << 30, // 16 GB
// If the ratio of obsolete bytes to live bytes is greater than
// obsoleteBytesMaxRatio, do not pace deletions at all.
obsoleteBytesMaxRatio: 0.20,
freeSpaceTimeframe: 10 * time.Second,
obsoleteBytesMaxRatio: 0.20,
obsoleteBytesTimeframe: 5 * time.Minute,
targetByteDeletionRate: targetByteDeletionRate,
getInfo: getInfo,
getInfo: getInfo,
}
d.mu.history.Init(time.Now(), deletePacerHistory)
d.mu.history.Init(now, deletePacerHistory)
return d
}
// limit applies rate limiting if the current free disk space is more than
// freeSpaceThreshold, and the ratio of obsolete to live bytes is less than
// obsoleteBytesMaxRatio.
func (p *deletionPacer) limit(amount uint64, info deletionPacerInfo) error {
obsoleteBytesRatio := float64(1.0)
if info.liveBytes > 0 {
obsoleteBytesRatio = float64(info.obsoleteBytes) / float64(info.liveBytes)
}
paceDeletions := info.freeBytes > p.freeSpaceThreshold &&
obsoleteBytesRatio < p.obsoleteBytesMaxRatio
p.mu.Lock()
if historyRate := p.mu.history.Sum(time.Now()) / int64(deletePacerHistory/time.Second); historyRate > p.targetByteDeletionRate {
// Deletions have been happening at a rate higher than the target rate; we
// want to speed up deletions so they match the historic rate. We do this by
// decreasing the amount accordingly.
amount = uint64(float64(p.targetByteDeletionRate) / float64(historyRate) * float64(amount))
}
p.mu.Unlock()
if paceDeletions {
burst := p.limiter.Burst()
for amount > uint64(burst) {
d := p.limiter.DelayN(time.Now(), burst)
if d == rate.InfDuration {
return errors.Errorf("pacing failed")
}
time.Sleep(d)
amount -= uint64(burst)
}
d := p.limiter.DelayN(time.Now(), int(amount))
if d == rate.InfDuration {
return errors.Errorf("pacing failed")
}
time.Sleep(d)
} else {
burst := p.limiter.Burst()
for amount > uint64(burst) {
// AllowN will subtract burst if there are enough tokens available,
// else leave the tokens untouched. That is, we are making a
// best-effort to account for this activity in the limiter, but by
// ignoring the return value, we do the activity instantaneously
// anyway.
p.limiter.AllowN(time.Now(), burst)
amount -= uint64(burst)
}
p.limiter.AllowN(time.Now(), int(amount))
}
return nil
}
// maybeThrottle slows down a deletion of this file if it's faster than
// opts.Experimental.MinDeletionRate.
func (p *deletionPacer) maybeThrottle(bytesToDelete uint64) error {
return p.limit(bytesToDelete, p.getInfo())
}
// reportDeletion is used to report a deletion to the pacer. The pacer uses it
// ReportDeletion is used to report a deletion to the pacer. The pacer uses it
// to keep track of the recent rate of deletions and potentially increase the
// deletion rate accordingly.
//
// reportDeletion is thread-safe.
func (p *deletionPacer) reportDeletion(bytesToDelete uint64) {
// ReportDeletion is thread-safe.
func (p *deletionPacer) ReportDeletion(now time.Time, bytesToDelete uint64) {
p.mu.Lock()
defer p.mu.Unlock()
p.mu.history.Add(time.Now(), int64(bytesToDelete))
p.mu.history.Add(now, int64(bytesToDelete))
}
type noopPacer struct{}
// PacingDelay returns the recommended pacing wait time (in seconds) for
// deleting the given number of bytes.
//
// PacingDelay is thread-safe.
func (p *deletionPacer) PacingDelay(now time.Time, bytesToDelete uint64) (waitSeconds float64) {
if p.targetByteDeletionRate == 0 {
// Pacing disabled.
return 0.0
}
func (p *noopPacer) maybeThrottle(_ uint64) error {
return nil
baseRate := float64(p.targetByteDeletionRate)
// If recent deletion rate is more than our target, use that so that we don't
// fall behind.
historicRate := func() float64 {
p.mu.Lock()
defer p.mu.Unlock()
return float64(p.mu.history.Sum(now)) / deletePacerHistory.Seconds()
}()
if historicRate > baseRate {
baseRate = historicRate
}
// Apply heuristics to increase the deletion rate.
var extraRate float64
info := p.getInfo()
if info.freeBytes <= p.freeSpaceThreshold {
// Increase the rate so that we can free up enough bytes within the timeframe.
extraRate = float64(p.freeSpaceThreshold-info.freeBytes) / p.freeSpaceTimeframe.Seconds()
}
if info.liveBytes == 0 {
// We don't know the obsolete bytes ratio. Disable pacing altogether.
return 0.0
}
obsoleteBytesRatio := float64(info.obsoleteBytes) / float64(info.liveBytes)
if obsoleteBytesRatio >= p.obsoleteBytesMaxRatio {
// Increase the rate so that we can free up enough bytes within the timeframe.
r := (obsoleteBytesRatio - p.obsoleteBytesMaxRatio) * float64(info.liveBytes) / p.obsoleteBytesTimeframe.Seconds()
if extraRate < r {
extraRate = r
}
}
return float64(bytesToDelete) / (baseRate + extraRate)
}
func (p *noopPacer) reportDeletion(_ uint64) {}
// history is a helper used to keep track of the recent history of a set of
// data points (in our case deleted bytes), at limited granularity.
// Specifically, we split the desired timeframe into 100 "epochs" and all times

50
vendor/github.com/cockroachdb/pebble/range_keys.go generated vendored
View File

@@ -17,7 +17,7 @@ import (
func (i *Iterator) constructRangeKeyIter() {
i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
&i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
&i.hasPrefix, &i.prefixOrFullSeekKey, true /* onlySets */, &i.rangeKey.rangeKeyBuffers.internal)
&i.hasPrefix, &i.prefixOrFullSeekKey, false /* internalKeys */, &i.rangeKey.rangeKeyBuffers.internal)
// If there's an indexed batch with range keys, include it.
if i.batch != nil {
@@ -36,19 +36,24 @@ func (i *Iterator) constructRangeKeyIter() {
}
// Next are the flushables: memtables and large batches.
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
if i.readState != nil {
for j := len(i.readState.memtables) - 1; j >= 0; j-- {
mem := i.readState.memtables[j]
// We only need to read from memtables which contain sequence numbers older
// than seqNum.
if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
continue
}
if rki := mem.newRangeKeyIter(&i.opts); rki != nil {
i.rangeKey.iterConfig.AddLevel(rki)
}
}
}
current := i.readState.current
current := i.version
if current == nil {
current = i.readState.current
}
// Next are the file levels: L0 sub-levels followed by lower levels.
//
// Add file-specific iterators for L0 files containing range keys. This is less
@@ -63,8 +68,7 @@ func (i *Iterator) constructRangeKeyIter() {
// around Key Trailer order.
iter := current.RangeKeyLevels[0].Iter()
for f := iter.Last(); f != nil; f = iter.Prev() {
spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
spanIter, err := i.newIterRangeKey(f, spanIterOpts)
spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions())
if err != nil {
i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
continue
@@ -78,7 +82,7 @@ func (i *Iterator) constructRangeKeyIter() {
continue
}
li := i.rangeKey.iterConfig.NewLevelIter()
spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
spanIterOpts := i.opts.SpanIterOptions()
li.Init(spanIterOpts, i.cmp, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
manifest.Level(level), manifest.KeyTypeRange)
i.rangeKey.iterConfig.AddLevel(li)
@@ -366,24 +370,24 @@ func (m *rangeKeyMasking) Intersects(prop []byte) (bool, error) {
// KeyIsWithinLowerBound implements the limitedBlockPropertyFilter interface
// defined in the sstable package. It's used to restrict the masking block
// property filter to only applying within the bounds of the active range key.
func (m *rangeKeyMasking) KeyIsWithinLowerBound(ik *InternalKey) bool {
func (m *rangeKeyMasking) KeyIsWithinLowerBound(key []byte) bool {
// Invariant: m.maskSpan != nil
//
// The provided `ik` is an inclusive lower bound of the block we're
// The provided `key` is an inclusive lower bound of the block we're
// considering skipping.
return m.cmp(m.maskSpan.Start, ik.UserKey) <= 0
return m.cmp(m.maskSpan.Start, key) <= 0
}
// KeyIsWithinUpperBound implements the limitedBlockPropertyFilter interface
// defined in the sstable package. It's used to restrict the masking block
// property filter to only applying within the bounds of the active range key.
func (m *rangeKeyMasking) KeyIsWithinUpperBound(ik *InternalKey) bool {
func (m *rangeKeyMasking) KeyIsWithinUpperBound(key []byte) bool {
// Invariant: m.maskSpan != nil
//
// The provided `ik` is an *inclusive* upper bound of the block we're
// The provided `key` is an *inclusive* upper bound of the block we're
// considering skipping, so the range key's end must be strictly greater
// than the block bound for the block to be within bounds.
return m.cmp(m.maskSpan.End, ik.UserKey) > 0
return m.cmp(m.maskSpan.End, key) > 0
}
// lazyCombinedIter implements the internalIterator interface, wrapping a
@@ -512,7 +516,11 @@ func (i *lazyCombinedIter) initCombinedIteration(
// Initialize the Iterator's interleaving iterator.
i.parent.rangeKey.iiter.Init(
&i.parent.comparer, i.parent.pointIter, i.parent.rangeKey.rangeKeyIter,
&i.parent.rangeKeyMasking, i.parent.opts.LowerBound, i.parent.opts.UpperBound)
keyspan.InterleavingIterOpts{
Mask: &i.parent.rangeKeyMasking,
LowerBound: i.parent.opts.LowerBound,
UpperBound: i.parent.opts.UpperBound,
})
// Set the parent's primary iterator to point to the combined, interleaving
// iterator that's now initialized with our current state.

33
vendor/github.com/cockroachdb/pebble/rangekey/rangekey.go generated vendored Normal file
View File

@@ -0,0 +1,33 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package rangekey provides functionality for working with range keys.
package rangekey
import (
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/rangekey"
"github.com/cockroachdb/pebble/sstable"
)
// Fragmenter exports the keyspan.Fragmenter type.
type Fragmenter = keyspan.Fragmenter
// Key exports the keyspan.Key type.
type Key = keyspan.Key
// Span exports the keyspan.Span type.
type Span = keyspan.Span
// IsRangeKey returns if this InternalKey is a range key. Alias for
// rangekey.IsRangeKey.
func IsRangeKey(ik sstable.InternalKey) bool {
return rangekey.IsRangeKey(ik.Kind())
}
// Decode decodes an InternalKey into a keyspan.Span, if it is a range key. If
// keysDst is provided, keys will be appended to keysDst to reduce allocations.
func Decode(ik sstable.InternalKey, val []byte, keysDst []keyspan.Key) (Span, error) {
return rangekey.Decode(ik, val, keysDst)
}

20
vendor/github.com/cockroachdb/pebble/read_state.go generated vendored
View File

@@ -20,14 +20,14 @@ import "sync/atomic"
// in Go.
type readState struct {
db *DB
refcnt int32
refcnt atomic.Int32
current *version
memtables flushableList
}
// ref adds a reference to the readState.
func (s *readState) ref() {
atomic.AddInt32(&s.refcnt, 1)
s.refcnt.Add(1)
}
// unref removes a reference to the readState. If this was the last reference,
@@ -35,7 +35,7 @@ func (s *readState) ref() {
// is NOT held as version.unref() will acquire it. See unrefLocked() if DB.mu
// is held by the caller.
func (s *readState) unref() {
if atomic.AddInt32(&s.refcnt, -1) != 0 {
if s.refcnt.Add(-1) != 0 {
return
}
s.current.Unref()
@@ -50,10 +50,11 @@ func (s *readState) unref() {
// unrefLocked removes a reference to the readState. If this was the last
// reference, the reference the readState holds on the version is
// released. Requires DB.mu is held as version.unrefLocked() requires it. See
// unref() if DB.mu is NOT held by the caller.
// released.
//
// DB.mu must be held. See unref() if DB.mu is NOT held by the caller.
func (s *readState) unrefLocked() {
if atomic.AddInt32(&s.refcnt, -1) != 0 {
if s.refcnt.Add(-1) != 0 {
return
}
s.current.UnrefLocked()
@@ -61,9 +62,8 @@ func (s *readState) unrefLocked() {
mem.readerUnrefLocked(true)
}
// NB: Unlike readState.unref(), we don't attempt to cleanup newly obsolete
// tables as unrefLocked() is only called during DB shutdown to release the
// current readState.
// In this code path, the caller is responsible for scheduling obsolete table
// deletion as necessary.
}
// loadReadState returns the current readState. The returned readState must be
@@ -82,10 +82,10 @@ func (d *DB) loadReadState() *readState {
func (d *DB) updateReadStateLocked(checker func(*DB) error) {
s := &readState{
db: d,
refcnt: 1,
current: d.mu.versions.currentVersion(),
memtables: d.mu.mem.queue,
}
s.refcnt.Store(1)
s.current.Ref()
for _, mem := range s.memtables {
mem.readerRef()

156
vendor/github.com/cockroachdb/pebble/record/log_writer.go generated vendored
View File

@@ -20,10 +20,11 @@ import (
)
var walSyncLabels = pprof.Labels("pebble", "wal-sync")
var errClosedWriter = errors.New("pebble/record: closed LogWriter")
type block struct {
// buf[:written] has already been filled with fragments. Updated atomically.
written int32
written atomic.Int32
// buf[:flushed] has already been flushed to w.
flushed int32
buf [blockSize]byte
@@ -69,7 +70,7 @@ type syncQueue struct {
//
// The head index is stored in the most-significant bits so that we can
// atomically add to it and the overflow is harmless.
headTail uint64
headTail atomic.Uint64
// slots is a ring buffer of values stored in this queue. The size must be a
// power of 2. A slot is in use until the tail index has moved beyond it.
@@ -78,7 +79,7 @@ type syncQueue struct {
// blocked is an atomic boolean which indicates whether syncing is currently
// blocked or can proceed. It is used by the implementation of
// min-sync-interval to block syncing until the min interval has passed.
blocked uint32
blocked atomic.Bool
}
const dequeueBits = 32
@@ -91,7 +92,7 @@ func (q *syncQueue) unpack(ptrs uint64) (head, tail uint32) {
}
func (q *syncQueue) push(wg *sync.WaitGroup, err *error) {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
panic("pebble: queue is full")
@@ -103,15 +104,15 @@ func (q *syncQueue) push(wg *sync.WaitGroup, err *error) {
// Increment head. This passes ownership of slot to dequeue and acts as a
// store barrier for writing the slot.
atomic.AddUint64(&q.headTail, 1<<dequeueBits)
q.headTail.Add(1 << dequeueBits)
}
func (q *syncQueue) setBlocked() {
atomic.StoreUint32(&q.blocked, 1)
q.blocked.Store(true)
}
func (q *syncQueue) clearBlocked() {
atomic.StoreUint32(&q.blocked, 0)
q.blocked.Store(false)
}
func (q *syncQueue) empty() bool {
@@ -124,10 +125,10 @@ func (q *syncQueue) empty() bool {
// min-sync-interval. It additionally returns the real length of this queue,
// regardless of whether syncing is blocked.
func (q *syncQueue) load() (head, tail, realLength uint32) {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail = q.unpack(ptrs)
realLength = head - tail
if atomic.LoadUint32(&q.blocked) == 1 {
if q.blocked.Load() {
return 0, 0, realLength
}
return head, tail, realLength
@@ -152,7 +153,7 @@ func (q *syncQueue) pop(head, tail uint32, err error, queueSemChan chan struct{}
// We need to bump the tail count before signalling the wait group as
// signalling the wait group can trigger release a blocked goroutine which
// will try to enqueue before we've "freed" space in the queue.
atomic.AddUint64(&q.headTail, 1)
q.headTail.Add(1)
wg.Done()
// Is always non-nil in production.
if queueSemChan != nil {
@@ -265,10 +266,7 @@ type LogWriter struct {
block *block
free struct {
sync.Mutex
// Condition variable used to signal a block is freed.
cond sync.Cond
blocks []*block
allocated int
blocks []*block
}
flusher struct {
@@ -312,18 +310,17 @@ type LogWriterConfig struct {
QueueSemChan chan struct{}
}
// CapAllocatedBlocks is the maximum number of blocks allocated by the
// LogWriter. With blockSize=32KiB, this allows the LogWriter to allocate up to
// 16 MiB in blocks. This is 25% of the default memtable size of 64 MiB.
// However, this is a maximum that will only be allocated if the commit pipeline
// would otherwise stall.
//
// TODO(jackson): Do we need to even cap the number of allocated blocks? The 1:1
// relationship between memtables and WALs already bounds the amount of memory
// allocated for log blocks to the size of the memtable. More buffering
// insulates from the effects of spikes in fsync latency. If we're unable to
// write to the WAL fast enough, at least command.
const CapAllocatedBlocks = 512
// initialAllocatedBlocksCap is the initial capacity of the various slices
// intended to hold LogWriter blocks. The LogWriter may allocate more blocks
// than this threshold allows.
const initialAllocatedBlocksCap = 32
// blockPool pools *blocks to avoid allocations. Blocks are only added to the
// Pool when a LogWriter is closed. Before that, free blocks are maintained
// within a LogWriter's own internal free list `w.free.blocks`.
var blockPool = sync.Pool{
New: func() any { return &block{} },
}
// NewLogWriter returns a new LogWriter.
func NewLogWriter(w io.Writer, logNum base.FileNum, logWriterConfig LogWriterConfig) *LogWriter {
@@ -343,10 +340,8 @@ func NewLogWriter(w io.Writer, logNum base.FileNum, logWriterConfig LogWriterCon
},
queueSemChan: logWriterConfig.QueueSemChan,
}
r.free.cond.L = &r.free.Mutex
r.free.blocks = make([]*block, 0, CapAllocatedBlocks)
r.free.allocated = 1
r.block = &block{}
r.free.blocks = make([]*block, 0, initialAllocatedBlocksCap)
r.block = blockPool.Get().(*block)
r.flusher.ready.init(&r.flusher.Mutex, &r.flusher.syncQ)
r.flusher.closed = make(chan struct{})
r.flusher.pending = make([]*block, 0, cap(r.free.blocks))
@@ -410,20 +405,22 @@ func (w *LogWriter) flushLoop(context.Context) {
// before syncing work. The guarantee of this code is that when a sync is
// requested, any previously queued flush work will be synced. This
// motivates reading the syncing work (f.syncQ.load()) before picking up
// the flush work (atomic.LoadInt32(&w.block.written)).
// the flush work (w.block.written.Load()).
// The list of full blocks that need to be written. This is copied from
// f.pending on every loop iteration, though the number of elements is
// usually small (usually 1, max 512). If WAL throughput is high, there may
// be a non-neglible number of blocks, but the cost of copying the block
// pointers will be a small portion of the overall work performed.
// usually small (most frequently 1). In the case of the WAL LogWriter, the
// number of blocks is bounded by the size of the WAL's corresponding
// memtable (MemtableSize/BlockSize). With the default 64 MiB memtables,
// this works out to at most 2048 elements if the entirety of the memtable's
// contents are queued.
pending := make([]*block, 0, cap(f.pending))
for {
for {
// Grab the portion of the current block that requires flushing. Note that
// the current block can be added to the pending blocks list after we release
// the flusher lock, but it won't be part of pending.
written := atomic.LoadInt32(&w.block.written)
written := w.block.written.Load()
if len(f.pending) > 0 || written > w.block.flushed || !f.syncQ.empty() {
break
}
@@ -442,8 +439,7 @@ func (w *LogWriter) flushLoop(context.Context) {
// Found work to do, so no longer idle.
workStartTime := time.Now()
idleDuration := workStartTime.Sub(idleStartTime)
pending = pending[:len(f.pending)]
copy(pending, f.pending)
pending = append(pending[:0], f.pending...)
f.pending = f.pending[:0]
f.metrics.PendingBufferLen.AddSample(int64(len(pending)))
@@ -459,7 +455,7 @@ func (w *LogWriter) flushLoop(context.Context) {
// be ordered after we get the list of sync waiters from syncQ in order to
// prevent a race where a waiter adds itself to syncQ, but this thread
// picks up the entry in syncQ and not the buffered data.
written := atomic.LoadInt32(&w.block.written)
written := w.block.written.Load()
data := w.block.buf[w.block.flushed:written]
w.block.flushed = written
@@ -562,32 +558,22 @@ func (w *LogWriter) flushBlock(b *block) error {
if _, err := w.w.Write(b.buf[b.flushed:]); err != nil {
return err
}
b.written = 0
b.written.Store(0)
b.flushed = 0
w.free.Lock()
w.free.blocks = append(w.free.blocks, b)
w.free.cond.Signal()
w.free.Unlock()
return nil
}
// queueBlock queues the current block for writing to the underlying writer,
// allocates a new block and reserves space for the next header.
func (w *LogWriter) queueBlock() (waitDuration time.Duration) {
func (w *LogWriter) queueBlock() {
// Allocate a new block, blocking until one is available. We do this first
// because w.block is protected by w.flusher.Mutex.
w.free.Lock()
if len(w.free.blocks) == 0 {
if w.free.allocated < cap(w.free.blocks) {
w.free.allocated++
w.free.blocks = append(w.free.blocks, &block{})
} else {
now := time.Now()
for len(w.free.blocks) == 0 {
w.free.cond.Wait()
}
waitDuration = time.Since(now)
}
w.free.blocks = append(w.free.blocks, blockPool.Get().(*block))
}
nextBlock := w.free.blocks[len(w.free.blocks)-1]
w.free.blocks = w.free.blocks[:len(w.free.blocks)-1]
@@ -602,28 +588,6 @@ func (w *LogWriter) queueBlock() (waitDuration time.Duration) {
f.Unlock()
w.blockNum++
return waitDuration
}
// ReserveAllFreeBlocksForTesting is used to only for testing.
func (w *LogWriter) ReserveAllFreeBlocksForTesting() (releaseFunc func()) {
w.free.Lock()
defer w.free.Unlock()
free := w.free.blocks
w.free.blocks = nil
return func() {
w.free.Lock()
defer w.free.Unlock()
// It is possible that someone has pushed a free block and w.free.blocks
// is no longer nil. That is harmless. Also, the waiter loops on the
// condition len(w.free.blocks) == 0, so to actually unblock it we need to
// give it a free block.
if len(free) == 0 {
free = append(free, &block{})
}
w.free.blocks = free
w.free.cond.Broadcast()
}
}
// Close flushes and syncs any unwritten data and closes the writer.
@@ -658,6 +622,7 @@ func (w *LogWriter) Close() error {
if f.fsyncLatency != nil {
f.fsyncLatency.Observe(float64(syncLatency))
}
free := w.free.blocks
f.Unlock()
if w.c != nil {
@@ -667,7 +632,14 @@ func (w *LogWriter) Close() error {
return cerr
}
}
w.err = errors.New("pebble/record: closed LogWriter")
for _, b := range free {
b.flushed = 0
b.written.Store(0)
blockPool.Put(b)
}
w.err = errClosedWriter
return err
}
@@ -675,20 +647,20 @@ func (w *LogWriter) Close() error {
// of the record.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) WriteRecord(p []byte) (int64, error) {
logSize, _, err := w.SyncRecord(p, nil, nil)
logSize, err := w.SyncRecord(p, nil, nil)
return logSize, err
}
// SyncRecord writes a complete record. If wg!= nil the record will be
// SyncRecord writes a complete record. If wg != nil the record will be
// asynchronously persisted to the underlying writer and done will be called on
// the wait group upon completion. Returns the offset just past the end of the
// record.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) SyncRecord(
p []byte, wg *sync.WaitGroup, err *error,
) (logSize int64, waitDuration time.Duration, err2 error) {
) (logSize int64, err2 error) {
if w.err != nil {
return -1, 0, w.err
return -1, w.err
}
// The `i == 0` condition ensures we handle empty records. Such records can
@@ -696,9 +668,7 @@ func (w *LogWriter) SyncRecord(
// MANIFEST is currently written using Writer, it is good to support the same
// semantics with LogWriter.
for i := 0; i == 0 || len(p) > 0; i++ {
var wd time.Duration
p, wd = w.emitFragment(i, p)
waitDuration += wd
p = w.emitFragment(i, p)
}
if wg != nil {
@@ -712,35 +682,35 @@ func (w *LogWriter) SyncRecord(
f.ready.Signal()
}
offset := w.blockNum*blockSize + int64(w.block.written)
offset := w.blockNum*blockSize + int64(w.block.written.Load())
// Note that we don't return w.err here as a concurrent call to Close would
// race with our read. That's ok because the only error we could be seeing is
// one to syncing for which the caller can receive notification of by passing
// in a non-nil err argument.
return offset, waitDuration, nil
return offset, nil
}
// Size returns the current size of the file.
// External synchronisation provided by commitPipeline.mu.
func (w *LogWriter) Size() int64 {
return w.blockNum*blockSize + int64(w.block.written)
return w.blockNum*blockSize + int64(w.block.written.Load())
}
func (w *LogWriter) emitEOFTrailer() {
// Write a recyclable chunk header with a different log number. Readers
// will treat the header as EOF when the log number does not match.
b := w.block
i := b.written
i := b.written.Load()
binary.LittleEndian.PutUint32(b.buf[i+0:i+4], 0) // CRC
binary.LittleEndian.PutUint16(b.buf[i+4:i+6], 0) // Size
b.buf[i+6] = recyclableFullChunkType
binary.LittleEndian.PutUint32(b.buf[i+7:i+11], w.logNum+1) // Log number
atomic.StoreInt32(&b.written, i+int32(recyclableHeaderSize))
b.written.Store(i + int32(recyclableHeaderSize))
}
func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDuration time.Duration) {
func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte) {
b := w.block
i := b.written
i := b.written.Load()
first := n == 0
last := blockSize-i-recyclableHeaderSize >= int32(len(p))
@@ -764,17 +734,17 @@ func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDurati
j := i + int32(recyclableHeaderSize+r)
binary.LittleEndian.PutUint32(b.buf[i+0:i+4], crc.New(b.buf[i+6:j]).Value())
binary.LittleEndian.PutUint16(b.buf[i+4:i+6], uint16(r))
atomic.StoreInt32(&b.written, j)
b.written.Store(j)
if blockSize-b.written < recyclableHeaderSize {
if blockSize-b.written.Load() < recyclableHeaderSize {
// There is no room for another fragment in the block, so fill the
// remaining bytes with zeros and queue the block for flushing.
for i := b.written; i < blockSize; i++ {
for i := b.written.Load(); i < blockSize; i++ {
b.buf[i] = 0
}
waitDuration = w.queueBlock()
w.queueBlock()
}
return p[r:], waitDuration
return p[r:]
}
// Metrics must be called after Close. The callee will no longer modify the

1007
vendor/github.com/cockroachdb/pebble/scan_internal.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

419
vendor/github.com/cockroachdb/pebble/snapshot.go generated vendored
View File

@@ -8,16 +8,29 @@ import (
"context"
"io"
"math"
"sync"
"sync/atomic"
"time"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/rangekey"
)
// ErrSnapshotExcised is returned from WaitForFileOnlySnapshot if an excise
// overlapping with one of the EventuallyFileOnlySnapshot's KeyRanges gets
// applied before the transition of that EFOS to a file-only snapshot.
var ErrSnapshotExcised = errors.New("pebble: snapshot excised before conversion to file-only snapshot")
// Snapshot provides a read-only point-in-time view of the DB state.
type Snapshot struct {
// The db the snapshot was created from.
db *DB
seqNum uint64
// Set if part of an EventuallyFileOnlySnapshot.
efos *EventuallyFileOnlySnapshot
// The list the snapshot is linked into.
list *snapshotList
@@ -44,17 +57,17 @@ func (s *Snapshot) Get(key []byte) ([]byte, io.Closer, error) {
// NewIter returns an iterator that is unpositioned (Iterator.Valid() will
// return false). The iterator can be positioned via a call to SeekGE,
// SeekLT, First or Last.
func (s *Snapshot) NewIter(o *IterOptions) *Iterator {
func (s *Snapshot) NewIter(o *IterOptions) (*Iterator, error) {
return s.NewIterWithContext(context.Background(), o)
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
// tracing.
func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator {
func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) (*Iterator, error) {
if s.db == nil {
panic(ErrClosed)
}
return s.db.newIter(ctx, nil /* batch */, s, o)
return s.db.newIter(ctx, nil /* batch */, snapshotIterOpts{seqNum: s.seqNum}, o), nil
}
// ScanInternal scans all internal keys within the specified bounds, truncating
@@ -64,33 +77,38 @@ func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iter
// See comment on db.ScanInternal for the behaviour that can be expected of
// point keys deleted by range dels and keys masked by range keys.
func (s *Snapshot) ScanInternal(
ctx context.Context,
lower, upper []byte,
visitPointKey func(key *InternalKey, value LazyValue) error,
visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []keyspan.Key) error,
visitRangeKey func(start, end []byte, keys []rangekey.Key) error,
visitSharedFile func(sst *SharedSSTMeta) error,
) error {
if s.db == nil {
panic(ErrClosed)
}
iter := s.db.newInternalIter(s, &IterOptions{
KeyTypes: IterKeyTypePointsAndRanges,
LowerBound: lower,
UpperBound: upper,
})
scanInternalOpts := &scanInternalOptions{
visitPointKey: visitPointKey,
visitRangeDel: visitRangeDel,
visitRangeKey: visitRangeKey,
visitSharedFile: visitSharedFile,
skipSharedLevels: visitSharedFile != nil,
IterOptions: IterOptions{
KeyTypes: IterKeyTypePointsAndRanges,
LowerBound: lower,
UpperBound: upper,
},
}
iter := s.db.newInternalIter(snapshotIterOpts{seqNum: s.seqNum}, scanInternalOpts)
defer iter.close()
return scanInternalImpl(lower, iter, visitPointKey, visitRangeDel, visitRangeKey)
return scanInternalImpl(ctx, lower, upper, iter, scanInternalOpts)
}
// Close closes the snapshot, releasing its resources. Close must be called.
// Failure to do so will result in a tiny memory leak and a large leak of
// resources on disk due to the entries the snapshot is preventing from being
// deleted.
func (s *Snapshot) Close() error {
if s.db == nil {
panic(ErrClosed)
}
s.db.mu.Lock()
// closeLocked is similar to Close(), except it requires that db.mu be held
// by the caller.
func (s *Snapshot) closeLocked() error {
s.db.mu.snapshots.remove(s)
// If s was the previous earliest snapshot, we might be able to reclaim
@@ -98,11 +116,26 @@ func (s *Snapshot) Close() error {
if e := s.db.mu.snapshots.earliest(); e > s.seqNum {
s.db.maybeScheduleCompactionPicker(pickElisionOnly)
}
s.db.mu.Unlock()
s.db = nil
return nil
}
// Close closes the snapshot, releasing its resources. Close must be called.
// Failure to do so will result in a tiny memory leak and a large leak of
// resources on disk due to the entries the snapshot is preventing from being
// deleted.
//
// d.mu must NOT be held by the caller.
func (s *Snapshot) Close() error {
db := s.db
if db == nil {
panic(ErrClosed)
}
db.mu.Lock()
defer db.mu.Unlock()
return s.closeLocked()
}
type snapshotList struct {
root Snapshot
}
@@ -170,3 +203,345 @@ func (l *snapshotList) remove(s *Snapshot) {
s.prev = nil // avoid memory leaks
s.list = nil // avoid memory leaks
}
// EventuallyFileOnlySnapshot (aka EFOS) provides a read-only point-in-time view
// of the database state, similar to Snapshot. An EventuallyFileOnlySnapshot
// induces less write amplification than Snapshot, at the cost of increased space
// amplification. While a Snapshot may increase write amplification across all
// flushes and compactions for the duration of its lifetime, an
// EventuallyFileOnlySnapshot only incurs that cost for flushes/compactions if
// memtables at the time of EFOS instantiation contained keys that the EFOS is
// interested in (i.e. its protectedRanges). In that case, the EFOS prevents
// elision of keys visible to it, similar to a Snapshot, until those memtables
// are flushed, and once that happens, the "EventuallyFileOnlySnapshot"
// transitions to a file-only snapshot state in which it pins zombies sstables
// like an open Iterator would, without pinning any memtables. Callers that can
// tolerate the increased space amplification of pinning zombie sstables until
// the snapshot is closed may prefer EventuallyFileOnlySnapshots for their
// reduced write amplification. Callers that desire the benefits of the file-only
// state that requires no pinning of memtables should call
// `WaitForFileOnlySnapshot()` (and possibly re-mint an EFOS if it returns
// ErrSnapshotExcised) before relying on the EFOS to keep producing iterators
// with zero write-amp and zero pinning of memtables in memory.
//
// EventuallyFileOnlySnapshots interact with the IngestAndExcise operation in
// subtle ways. No new iterators can be created once
// EventuallyFileOnlySnapshot.excised is set to true.
type EventuallyFileOnlySnapshot struct {
mu struct {
// NB: If both this mutex and db.mu are being grabbed, db.mu should be
// grabbed _before_ grabbing this one.
sync.Mutex
// Either the snap field is set below, or the version is set at any given
// point of time. If a snapshot is referenced, this is not a file-only
// snapshot yet, and if a version is set (and ref'd) this is a file-only
// snapshot.
// The wrapped regular snapshot, if not a file-only snapshot yet.
snap *Snapshot
// The wrapped version reference, if a file-only snapshot.
vers *version
}
// Key ranges to watch for an excise on.
protectedRanges []KeyRange
// excised, if true, signals that the above ranges were excised during the
// lifetime of this snapshot.
excised atomic.Bool
// The db the snapshot was created from.
db *DB
seqNum uint64
closed chan struct{}
}
func (d *DB) makeEventuallyFileOnlySnapshot(
keyRanges []KeyRange, internalKeyRanges []internalKeyRange,
) *EventuallyFileOnlySnapshot {
isFileOnly := true
d.mu.Lock()
defer d.mu.Unlock()
seqNum := d.mu.versions.visibleSeqNum.Load()
// Check if any of the keyRanges overlap with a memtable.
for i := range d.mu.mem.queue {
mem := d.mu.mem.queue[i]
if ingestMemtableOverlaps(d.cmp, mem, internalKeyRanges) {
isFileOnly = false
break
}
}
es := &EventuallyFileOnlySnapshot{
db: d,
seqNum: seqNum,
protectedRanges: keyRanges,
closed: make(chan struct{}),
}
if isFileOnly {
es.mu.vers = d.mu.versions.currentVersion()
es.mu.vers.Ref()
} else {
s := &Snapshot{
db: d,
seqNum: seqNum,
}
s.efos = es
es.mu.snap = s
d.mu.snapshots.pushBack(s)
}
return es
}
// Transitions this EventuallyFileOnlySnapshot to a file-only snapshot. Requires
// earliestUnflushedSeqNum and vers to correspond to the same Version from the
// current or a past acquisition of db.mu. vers must have been Ref()'d before
// that mutex was released, if it was released.
//
// NB: The caller is expected to check for es.excised before making this
// call.
//
// d.mu must be held when calling this method.
func (es *EventuallyFileOnlySnapshot) transitionToFileOnlySnapshot(vers *version) error {
es.mu.Lock()
select {
case <-es.closed:
vers.UnrefLocked()
es.mu.Unlock()
return ErrClosed
default:
}
if es.mu.snap == nil {
es.mu.Unlock()
panic("pebble: tried to transition an eventually-file-only-snapshot twice")
}
// The caller has already called Ref() on vers.
es.mu.vers = vers
// NB: The callers should have already done a check of es.excised.
oldSnap := es.mu.snap
es.mu.snap = nil
es.mu.Unlock()
return oldSnap.closeLocked()
}
// hasTransitioned returns true if this EFOS has transitioned to a file-only
// snapshot.
func (es *EventuallyFileOnlySnapshot) hasTransitioned() bool {
es.mu.Lock()
defer es.mu.Unlock()
return es.mu.vers != nil
}
// waitForFlush waits for a flush on any memtables that need to be flushed
// before this EFOS can transition to a file-only snapshot. If this EFOS is
// waiting on a flush of the mutable memtable, it forces a rotation within
// `dur` duration. For immutable memtables, it schedules a flush and waits for
// it to finish.
func (es *EventuallyFileOnlySnapshot) waitForFlush(ctx context.Context, dur time.Duration) error {
es.db.mu.Lock()
defer es.db.mu.Unlock()
earliestUnflushedSeqNum := es.db.getEarliestUnflushedSeqNumLocked()
for earliestUnflushedSeqNum < es.seqNum {
select {
case <-es.closed:
return ErrClosed
case <-ctx.Done():
return ctx.Err()
default:
}
// Check if the current mutable memtable contains keys less than seqNum.
// If so, rotate it.
if es.db.mu.mem.mutable.logSeqNum < es.seqNum && dur.Nanoseconds() > 0 {
es.db.maybeScheduleDelayedFlush(es.db.mu.mem.mutable, dur)
} else {
// Find the last memtable that contains seqNums less than es.seqNum,
// and force a flush on it.
var mem *flushableEntry
for i := range es.db.mu.mem.queue {
if es.db.mu.mem.queue[i].logSeqNum < es.seqNum {
mem = es.db.mu.mem.queue[i]
}
}
mem.flushForced = true
es.db.maybeScheduleFlush()
}
es.db.mu.compact.cond.Wait()
earliestUnflushedSeqNum = es.db.getEarliestUnflushedSeqNumLocked()
}
if es.excised.Load() {
return ErrSnapshotExcised
}
return nil
}
// WaitForFileOnlySnapshot blocks the calling goroutine until this snapshot
// has been converted into a file-only snapshot (i.e. all memtables containing
// keys < seqNum are flushed). A duration can be passed in, and if nonzero,
// a delayed flush will be scheduled at that duration if necessary.
//
// Idempotent; can be called multiple times with no side effects.
func (es *EventuallyFileOnlySnapshot) WaitForFileOnlySnapshot(
ctx context.Context, dur time.Duration,
) error {
if es.hasTransitioned() {
return nil
}
if err := es.waitForFlush(ctx, dur); err != nil {
return err
}
if invariants.Enabled {
// Since we aren't returning an error, we _must_ have transitioned to a
// file-only snapshot by now.
if !es.hasTransitioned() {
panic("expected EFOS to have transitioned to file-only snapshot after flush")
}
}
return nil
}
// Close closes the file-only snapshot and releases all referenced resources.
// Not idempotent.
func (es *EventuallyFileOnlySnapshot) Close() error {
close(es.closed)
es.db.mu.Lock()
defer es.db.mu.Unlock()
es.mu.Lock()
defer es.mu.Unlock()
if es.mu.snap != nil {
if err := es.mu.snap.closeLocked(); err != nil {
return err
}
}
if es.mu.vers != nil {
es.mu.vers.UnrefLocked()
}
return nil
}
// Get implements the Reader interface.
func (es *EventuallyFileOnlySnapshot) Get(key []byte) (value []byte, closer io.Closer, err error) {
// TODO(jackson): Use getInternal.
iter, err := es.NewIter(nil)
if err != nil {
return nil, nil, err
}
var valid bool
if es.db.opts.Comparer.Split != nil {
valid = iter.SeekPrefixGE(key)
} else {
valid = iter.SeekGE(key)
}
if !valid {
if err = firstError(iter.Error(), iter.Close()); err != nil {
return nil, nil, err
}
return nil, nil, ErrNotFound
}
if !es.db.equal(iter.Key(), key) {
return nil, nil, firstError(iter.Close(), ErrNotFound)
}
return iter.Value(), iter, nil
}
// NewIter returns an iterator that is unpositioned (Iterator.Valid() will
// return false). The iterator can be positioned via a call to SeekGE,
// SeekLT, First or Last.
func (es *EventuallyFileOnlySnapshot) NewIter(o *IterOptions) (*Iterator, error) {
return es.NewIterWithContext(context.Background(), o)
}
// NewIterWithContext is like NewIter, and additionally accepts a context for
// tracing.
func (es *EventuallyFileOnlySnapshot) NewIterWithContext(
ctx context.Context, o *IterOptions,
) (*Iterator, error) {
select {
case <-es.closed:
panic(ErrClosed)
default:
}
es.mu.Lock()
defer es.mu.Unlock()
if es.mu.vers != nil {
sOpts := snapshotIterOpts{seqNum: es.seqNum, vers: es.mu.vers}
return es.db.newIter(ctx, nil /* batch */, sOpts, o), nil
}
if es.excised.Load() {
return nil, ErrSnapshotExcised
}
sOpts := snapshotIterOpts{seqNum: es.seqNum}
iter := es.db.newIter(ctx, nil /* batch */, sOpts, o)
// If excised is true, then keys relevant to the snapshot might not be
// present in the readState being used by the iterator. Error out.
if es.excised.Load() {
iter.Close()
return nil, ErrSnapshotExcised
}
return iter, nil
}
// ScanInternal scans all internal keys within the specified bounds, truncating
// any rangedels and rangekeys to those bounds. For use when an external user
// needs to be aware of all internal keys that make up a key range.
//
// See comment on db.ScanInternal for the behaviour that can be expected of
// point keys deleted by range dels and keys masked by range keys.
func (es *EventuallyFileOnlySnapshot) ScanInternal(
ctx context.Context,
lower, upper []byte,
visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error,
visitRangeDel func(start, end []byte, seqNum uint64) error,
visitRangeKey func(start, end []byte, keys []rangekey.Key) error,
visitSharedFile func(sst *SharedSSTMeta) error,
) error {
if es.db == nil {
panic(ErrClosed)
}
if es.excised.Load() {
return ErrSnapshotExcised
}
var sOpts snapshotIterOpts
es.mu.Lock()
if es.mu.vers != nil {
sOpts = snapshotIterOpts{
seqNum: es.seqNum,
vers: es.mu.vers,
}
} else {
sOpts = snapshotIterOpts{
seqNum: es.seqNum,
}
}
es.mu.Unlock()
opts := &scanInternalOptions{
IterOptions: IterOptions{
KeyTypes: IterKeyTypePointsAndRanges,
LowerBound: lower,
UpperBound: upper,
},
visitPointKey: visitPointKey,
visitRangeDel: visitRangeDel,
visitRangeKey: visitRangeKey,
visitSharedFile: visitSharedFile,
skipSharedLevels: visitSharedFile != nil,
}
iter := es.db.newInternalIter(sOpts, opts)
defer iter.close()
// If excised is true, then keys relevant to the snapshot might not be
// present in the readState being used by the iterator. Error out.
if es.excised.Load() {
return ErrSnapshotExcised
}
return scanInternalImpl(ctx, lower, upper, iter, opts)
}

314
vendor/github.com/cockroachdb/pebble/sstable/block.go generated vendored
View File

@@ -10,7 +10,6 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/cache"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manual"
@@ -47,7 +46,10 @@ type blockWriter struct {
// ensuring that any of the keys with the same prefix can be used to
// assemble the full key when the prefix does change.
restarts []uint32
curKey []byte
// Do not read curKey directly from outside blockWriter since it can have
// the InternalKeyKindSSTableInternalObsoleteBit set. Use getCurKey() or
// getCurUserKey() instead.
curKey []byte
// curValue excludes the optional prefix provided to
// storeWithOptionalValuePrefix.
curValue []byte
@@ -79,6 +81,20 @@ const setHasSameKeyPrefixRestartMask uint32 = 1 << 31
const restartMaskLittleEndianHighByteWithoutSetHasSamePrefix byte = 0b0111_1111
const restartMaskLittleEndianHighByteOnlySetHasSamePrefix byte = 0b1000_0000
func (w *blockWriter) getCurKey() InternalKey {
k := base.DecodeInternalKey(w.curKey)
k.Trailer = k.Trailer & trailerObsoleteMask
return k
}
func (w *blockWriter) getCurUserKey() []byte {
n := len(w.curKey) - base.InternalTrailerLen
if n < 0 {
panic(errors.AssertionFailedf("corrupt key in blockWriter buffer"))
}
return w.curKey[:n:n]
}
// If !addValuePrefix, the valuePrefix is ignored.
func (w *blockWriter) storeWithOptionalValuePrefix(
keySize int,
@@ -191,12 +207,21 @@ func (w *blockWriter) storeWithOptionalValuePrefix(
func (w *blockWriter) add(key InternalKey, value []byte) {
w.addWithOptionalValuePrefix(
key, value, len(key.UserKey), false, 0, false)
key, false, value, len(key.UserKey), false, 0, false)
}
// Callers that always set addValuePrefix to false should use add() instead.
//
// isObsolete indicates whether this key-value pair is obsolete in this
// sstable (only applicable when writing data blocks) -- see the comment in
// table.go and the longer one in format.go. addValuePrefix adds a 1 byte
// prefix to the value, specified in valuePrefix -- this is used for data
// blocks in TableFormatPebblev3 onwards for SETs (see the comment in
// format.go, with more details in value_block.go). setHasSameKeyPrefix is
// also used in TableFormatPebblev3 onwards for SETs.
func (w *blockWriter) addWithOptionalValuePrefix(
key InternalKey,
isObsolete bool,
value []byte,
maxSharedKeyLen int,
addValuePrefix bool,
@@ -210,6 +235,9 @@ func (w *blockWriter) addWithOptionalValuePrefix(
w.curKey = make([]byte, 0, size*2)
}
w.curKey = w.curKey[:size]
if isObsolete {
key.Trailer = key.Trailer | trailerObsoleteBit
}
key.Encode(w.curKey)
w.storeWithOptionalValuePrefix(
@@ -371,16 +399,16 @@ type blockIter struct {
// For a block encoded with a restart interval of 1, cached and cachedBuf
// will not be used as there are no prefix compressed entries between the
// restart points.
cached []blockEntry
cachedBuf []byte
cacheHandle cache.Handle
// The first key in the block. This is used by the caller to set bounds
cached []blockEntry
cachedBuf []byte
handle bufferHandle
// for block iteration for already loaded blocks.
firstKey InternalKey
firstUserKey []byte
lazyValueHandling struct {
vbr *valueBlockReader
hasValuePrefix bool
}
hideObsoletePoints bool
}
// blockIter implements the base.InternalIterator interface.
@@ -388,14 +416,16 @@ var _ base.InternalIterator = (*blockIter)(nil)
func newBlockIter(cmp Compare, block block) (*blockIter, error) {
i := &blockIter{}
return i, i.init(cmp, block, 0)
return i, i.init(cmp, block, 0, false)
}
func (i *blockIter) String() string {
return "block"
}
func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error {
func (i *blockIter) init(
cmp Compare, block block, globalSeqNum uint64, hideObsoletePoints bool,
) error {
numRestarts := int32(binary.LittleEndian.Uint32(block[len(block)-4:]))
if numRestarts == 0 {
return base.CorruptionErrorf("pebble/table: invalid table (block has no restart points)")
@@ -408,6 +438,7 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error {
i.data = block
i.fullKey = i.fullKey[:0]
i.val = nil
i.hideObsoletePoints = hideObsoletePoints
i.clearCache()
if i.restarts > 0 {
if err := i.readFirstKey(); err != nil {
@@ -415,15 +446,21 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error {
}
} else {
// Block is empty.
i.firstKey = InternalKey{}
i.firstUserKey = nil
}
return nil
}
func (i *blockIter) initHandle(cmp Compare, block cache.Handle, globalSeqNum uint64) error {
i.cacheHandle.Release()
i.cacheHandle = block
return i.init(cmp, block.Get(), globalSeqNum)
// NB: two cases of hideObsoletePoints:
// - Local sstable iteration: globalSeqNum will be set iff the sstable was
// ingested.
// - Foreign sstable iteration: globalSeqNum is always set.
func (i *blockIter) initHandle(
cmp Compare, block bufferHandle, globalSeqNum uint64, hideObsoletePoints bool,
) error {
i.handle.Release()
i.handle = block
return i.init(cmp, block.Get(), globalSeqNum, hideObsoletePoints)
}
func (i *blockIter) invalidate() {
@@ -590,24 +627,27 @@ func (i *blockIter) readFirstKey() error {
// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
// BlockIter benchmarks.
if n := len(firstKey) - 8; n >= 0 {
i.firstKey.Trailer = binary.LittleEndian.Uint64(firstKey[n:])
i.firstKey.UserKey = firstKey[:n:n]
if i.globalSeqNum != 0 {
i.firstKey.SetSeqNum(i.globalSeqNum)
}
i.firstUserKey = firstKey[:n:n]
} else {
i.firstKey.Trailer = uint64(InternalKeyKindInvalid)
i.firstKey.UserKey = nil
i.firstUserKey = nil
return base.CorruptionErrorf("pebble/table: invalid firstKey in block")
}
return nil
}
func (i *blockIter) decodeInternalKey(key []byte) {
// The sstable internal obsolete bit is set when writing a block and unset by
// blockIter, so no code outside block writing/reading code ever sees it.
const trailerObsoleteBit = uint64(base.InternalKeyKindSSTableInternalObsoleteBit)
const trailerObsoleteMask = (InternalKeySeqNumMax << 8) | uint64(base.InternalKeyKindSSTableInternalObsoleteMask)
func (i *blockIter) decodeInternalKey(key []byte) (hiddenPoint bool) {
// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
// BlockIter benchmarks.
if n := len(key) - 8; n >= 0 {
i.ikey.Trailer = binary.LittleEndian.Uint64(key[n:])
trailer := binary.LittleEndian.Uint64(key[n:])
hiddenPoint = i.hideObsoletePoints &&
(trailer&trailerObsoleteBit != 0)
i.ikey.Trailer = trailer & trailerObsoleteMask
i.ikey.UserKey = key[:n:n]
if i.globalSeqNum != 0 {
i.ikey.SetSeqNum(i.globalSeqNum)
@@ -616,6 +656,7 @@ func (i *blockIter) decodeInternalKey(key []byte) {
i.ikey.Trailer = uint64(InternalKeyKindInvalid)
i.ikey.UserKey = nil
}
return hiddenPoint
}
func (i *blockIter) clearCache() {
@@ -640,6 +681,10 @@ func (i *blockIter) cacheEntry() {
i.cachedBuf = append(i.cachedBuf, i.key...)
}
func (i *blockIter) getFirstUserKey() []byte {
return i.firstUserKey
}
// SeekGE implements internalIterator.SeekGE, as documented in the pebble
// package.
func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) {
@@ -647,7 +692,6 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
panic(errors.AssertionFailedf("invalidated blockIter used"))
}
ikey := base.MakeSearchKey(key)
i.clearCache()
// Find the index of the smallest restart point whose key is > the key
// sought; index will be numRestarts if there is no such restart point.
@@ -702,23 +746,23 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
ptr = unsafe.Pointer(uintptr(ptr) + 5)
}
// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
// BlockIter benchmarks.
// Manually inlining part of base.DecodeInternalKey provides a 5-10%
// speedup on BlockIter benchmarks.
s := getBytes(ptr, int(v1))
var k InternalKey
var k []byte
if n := len(s) - 8; n >= 0 {
k.Trailer = binary.LittleEndian.Uint64(s[n:])
k.UserKey = s[:n:n]
// NB: We can't have duplicate keys if the globalSeqNum != 0, so we
// leave the seqnum on this key as 0 as it won't affect our search
// since ikey has the maximum seqnum.
} else {
k.Trailer = uint64(InternalKeyKindInvalid)
k = s[:n:n]
}
// Else k is invalid, and left as nil
if base.InternalCompare(i.cmp, ikey, k) >= 0 {
if i.cmp(key, k) > 0 {
// The search key is greater than the user key at this restart point.
// Search beyond this restart point, since we are trying to find the
// first restart point with a user key >= the search key.
index = h + 1 // preserves f(i-1) == false
} else {
// k >= search key, so prune everything after index (since index
// satisfies the property we are looking for).
upper = h // preserves f(j) == true
}
}
@@ -726,21 +770,25 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
// => answer is index.
}
// Since keys are strictly increasing, if index > 0 then the restart point at
// index-1 will be the largest whose key is <= the key sought. If index ==
// 0, then all keys in this block are larger than the key sought, and offset
// remains at zero.
// index is the first restart point with key >= search key. Define the keys
// between a restart point and the next restart point as belonging to that
// restart point.
//
// Since keys are strictly increasing, if index > 0 then the restart point
// at index-1 will be the first one that has some keys belonging to it that
// could be equal to the search key. If index == 0, then all keys in this
// block are larger than the key sought, and offset remains at zero.
if index > 0 {
i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
}
i.readEntry()
i.decodeInternalKey(i.key)
hiddenPoint := i.decodeInternalKey(i.key)
// Iterate from that restart point to somewhere >= the key sought.
if !i.valid() {
return nil, base.LazyValue{}
}
if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
if !hiddenPoint && i.cmp(i.ikey.UserKey, key) >= 0 {
// Initialize i.lazyValue
if !i.lazyValueHandling.hasValuePrefix ||
base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
@@ -753,7 +801,7 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
return &i.ikey, i.lazyValue
}
for i.Next(); i.valid(); i.Next() {
if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
if i.cmp(i.ikey.UserKey, key) >= 0 {
// i.Next() has already initialized i.lazyValue.
return &i.ikey, i.lazyValue
}
@@ -777,8 +825,6 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
panic(errors.AssertionFailedf("invalidated blockIter used"))
}
ikey := base.MakeSearchKey(key)
i.clearCache()
// Find the index of the smallest restart point whose key is >= the key
// sought; index will be numRestarts if there is no such restart point.
@@ -833,23 +879,23 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
ptr = unsafe.Pointer(uintptr(ptr) + 5)
}
// Manually inlining base.DecodeInternalKey provides a 5-10% speedup on
// BlockIter benchmarks.
// Manually inlining part of base.DecodeInternalKey provides a 5-10%
// speedup on BlockIter benchmarks.
s := getBytes(ptr, int(v1))
var k InternalKey
var k []byte
if n := len(s) - 8; n >= 0 {
k.Trailer = binary.LittleEndian.Uint64(s[n:])
k.UserKey = s[:n:n]
// NB: We can't have duplicate keys if the globalSeqNum != 0, so we
// leave the seqnum on this key as 0 as it won't affect our search
// since ikey has the maximum seqnum.
} else {
k.Trailer = uint64(InternalKeyKindInvalid)
k = s[:n:n]
}
// Else k is invalid, and left as nil
if base.InternalCompare(i.cmp, ikey, k) > 0 {
if i.cmp(key, k) > 0 {
// The search key is greater than the user key at this restart point.
// Search beyond this restart point, since we are trying to find the
// first restart point with a user key >= the search key.
index = h + 1 // preserves f(i-1) == false
} else {
// k >= search key, so prune everything after index (since index
// satisfies the property we are looking for).
upper = h // preserves f(j) == true
}
}
@@ -857,8 +903,15 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
// => answer is index.
}
// Since keys are strictly increasing, if index > 0 then the restart point at
// index-1 will be the largest whose key is < the key sought.
// index is the first restart point with key >= search key. Define the keys
// between a restart point and the next restart point as belonging to that
// restart point. Note that index could be equal to i.numRestarts, i.e., we
// are past the last restart.
//
// Since keys are strictly increasing, if index > 0 then the restart point
// at index-1 will be the first one that has some keys belonging to it that
// are less than the search key. If index == 0, then all keys in this block
// are larger than the search key, so there is no match.
targetOffset := i.restarts
if index > 0 {
i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
@@ -881,24 +934,41 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
for {
i.offset = i.nextOffset
i.readEntry()
i.decodeInternalKey(i.key)
// When hidden keys are common, there is additional optimization possible
// by not caching entries that are hidden (note that some calls to
// cacheEntry don't decode the internal key before caching, but checking
// whether a key is hidden does not require full decoding). However, we do
// need to use the blockEntry.offset in the cache for the first entry at
// the reset point to do the binary search when the cache is empty -- so
// we would need to cache that first entry (though not the key) even if
// was hidden. Our current assumption is that if there are large numbers
// of hidden keys we will be able to skip whole blocks (using block
// property filters) so we don't bother optimizing.
hiddenPoint := i.decodeInternalKey(i.key)
if i.cmp(i.ikey.UserKey, ikey.UserKey) >= 0 {
// NB: we don't use the hiddenPoint return value of decodeInternalKey
// since we want to stop as soon as we reach a key >= ikey.UserKey, so
// that we can reverse.
if i.cmp(i.ikey.UserKey, key) >= 0 {
// The current key is greater than or equal to our search key. Back up to
// the previous key which was less than our search key. Note that this for
// loop will execute at least once with this if-block not being true, so
// the key we are backing up to is the last one this loop cached.
i.Prev()
// i.Prev() has already initialized i.lazyValue.
return &i.ikey, i.lazyValue
return i.Prev()
}
if i.nextOffset >= targetOffset {
// We've reached the end of the current restart block. Return the current
// key. When the restart interval is 1, the first iteration of the for
// loop will bring us here. In that case ikey is backed by the block so
// we get the desired key stability guarantee for the lifetime of the
// blockIter.
// We've reached the end of the current restart block. Return the
// current key if not hidden, else call Prev().
//
// When the restart interval is 1, the first iteration of the for loop
// will bring us here. In that case ikey is backed by the block so we
// get the desired key stability guarantee for the lifetime of the
// blockIter. That is, we never cache anything and therefore never
// return a key backed by cachedBuf.
if hiddenPoint {
return i.Prev()
}
break
}
@@ -932,7 +1002,10 @@ func (i *blockIter) First() (*InternalKey, base.LazyValue) {
}
i.clearCache()
i.readEntry()
i.decodeInternalKey(i.key)
hiddenPoint := i.decodeInternalKey(i.key)
if hiddenPoint {
return i.Next()
}
if !i.lazyValueHandling.hasValuePrefix ||
base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
i.lazyValue = base.MakeInPlaceValue(i.val)
@@ -971,7 +1044,10 @@ func (i *blockIter) Last() (*InternalKey, base.LazyValue) {
i.readEntry()
}
i.decodeInternalKey(i.key)
hiddenPoint := i.decodeInternalKey(i.key)
if hiddenPoint {
return i.Prev()
}
if !i.lazyValueHandling.hasValuePrefix ||
base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
i.lazyValue = base.MakeInPlaceValue(i.val)
@@ -1001,6 +1077,7 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
i.clearCache()
}
start:
i.offset = i.nextOffset
if !i.valid() {
return nil, base.LazyValue{}
@@ -1008,11 +1085,17 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
i.readEntry()
// Manually inlined version of i.decodeInternalKey(i.key).
if n := len(i.key) - 8; n >= 0 {
i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
trailer := binary.LittleEndian.Uint64(i.key[n:])
hiddenPoint := i.hideObsoletePoints &&
(trailer&trailerObsoleteBit != 0)
i.ikey.Trailer = trailer & trailerObsoleteMask
i.ikey.UserKey = i.key[:n:n]
if i.globalSeqNum != 0 {
i.ikey.SetSeqNum(i.globalSeqNum)
}
if hiddenPoint {
goto start
}
} else {
i.ikey.Trailer = uint64(InternalKeyKindInvalid)
i.ikey.UserKey = nil
@@ -1028,8 +1111,8 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
return &i.ikey, i.lazyValue
}
// nextPrefix is used for implementing NPrefix.
func (i *blockIter) nextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
// NextPrefix implements (base.InternalIterator).NextPrefix.
func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
if i.lazyValueHandling.hasValuePrefix {
return i.nextPrefixV3(succKey)
}
@@ -1154,6 +1237,7 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
// The trailer is written in little endian, so the key kind is the first
// byte in the trailer that is encoded in the slice [unshared-8:unshared].
keyKind := InternalKeyKind((*[manual.MaxArrayLen]byte)(ptr)[unshared-8])
keyKind = keyKind & base.InternalKeyKindSSTableInternalObsoleteMask
prefixChanged := false
if keyKind == InternalKeyKindSet {
if invariants.Enabled && value == 0 {
@@ -1269,8 +1353,12 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
i.key = i.fullKey
}
// Manually inlined version of i.decodeInternalKey(i.key).
hiddenPoint := false
if n := len(i.key) - 8; n >= 0 {
i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
trailer := binary.LittleEndian.Uint64(i.key[n:])
hiddenPoint = i.hideObsoletePoints &&
(trailer&trailerObsoleteBit != 0)
i.ikey.Trailer = trailer & trailerObsoleteMask
i.ikey.UserKey = i.key[:n:n]
if i.globalSeqNum != 0 {
i.ikey.SetSeqNum(i.globalSeqNum)
@@ -1286,6 +1374,9 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
}
if prefixChanged || i.cmp(i.ikey.UserKey, succKey) >= 0 {
// Prefix has changed.
if hiddenPoint {
return i.Next()
}
if invariants.Enabled && !i.lazyValueHandling.hasValuePrefix {
panic(errors.AssertionFailedf("nextPrefixV3 being run for non-v3 sstable"))
}
@@ -1307,23 +1398,11 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue)
return i.SeekGE(succKey, base.SeekGEFlagsNone)
}
// NextPrefix implements (base.InternalIterator).NextPrefix
func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
const nextsBeforeSeek = 3
k, v := i.Next()
for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ {
if j >= nextsBeforeSeek {
return i.SeekGE(succKey, base.SeekGEFlagsNone)
}
k, v = i.Next()
}
return k, v
}
// Prev implements internalIterator.Prev, as documented in the pebble
// package.
func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
if n := len(i.cached) - 1; n >= 0 {
start:
for n := len(i.cached) - 1; n >= 0; n-- {
i.nextOffset = i.offset
e := &i.cached[n]
i.offset = e.offset
@@ -1331,7 +1410,13 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
// Manually inlined version of i.decodeInternalKey(i.key).
i.key = i.cachedBuf[e.keyStart:e.keyEnd]
if n := len(i.key) - 8; n >= 0 {
i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
trailer := binary.LittleEndian.Uint64(i.key[n:])
hiddenPoint := i.hideObsoletePoints &&
(trailer&trailerObsoleteBit != 0)
if hiddenPoint {
continue
}
i.ikey.Trailer = trailer & trailerObsoleteMask
i.ikey.UserKey = i.key[:n:n]
if i.globalSeqNum != 0 {
i.ikey.SetSeqNum(i.globalSeqNum)
@@ -1373,6 +1458,8 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
// index ≤ h < upper
offset := decodeRestart(i.data[i.restarts+4*h:])
if offset < targetOffset {
// Looking for the first restart that has offset >= targetOffset, so
// ignore h and earlier.
index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
@@ -1382,20 +1469,35 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
// => answer is index.
}
// index is first restart with offset >= targetOffset. Note that
// targetOffset may not be at a restart point since one can call Prev()
// after Next() (so the cache was not populated) and targetOffset refers to
// the current entry. index-1 must have an offset < targetOffset (it can't
// be equal to targetOffset since the binary search would have selected that
// as the index).
i.offset = 0
if index > 0 {
i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
}
// TODO(sumeer): why is the else case not an error given targetOffset is a
// valid offset.
i.readEntry()
// We stop when i.nextOffset == targetOffset since the targetOffset is the
// entry we are stepping back from, and we don't need to cache the entry
// before it, since it is the candidate to return.
for i.nextOffset < targetOffset {
i.cacheEntry()
i.offset = i.nextOffset
i.readEntry()
}
i.decodeInternalKey(i.key)
hiddenPoint := i.decodeInternalKey(i.key)
if hiddenPoint {
// Use the cache.
goto start
}
if !i.lazyValueHandling.hasValuePrefix ||
base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
i.lazyValue = base.MakeInPlaceValue(i.val)
@@ -1425,8 +1527,8 @@ func (i *blockIter) Error() error {
// Close implements internalIterator.Close, as documented in the pebble
// package.
func (i *blockIter) Close() error {
i.cacheHandle.Release()
i.cacheHandle = cache.Handle{}
i.handle.Release()
i.handle = bufferHandle{}
i.val = nil
i.lazyValue = base.LazyValue{}
i.lazyValueHandling.vbr = nil
@@ -1468,6 +1570,10 @@ type fragmentBlockIter struct {
err error
dir int8
closeHook func(i keyspan.FragmentIterator) error
// elideSameSeqnum, if true, returns only the first-occurring (in forward
// order) Key for each sequence number.
elideSameSeqnum bool
}
func (i *fragmentBlockIter) resetForReuse() fragmentBlockIter {
@@ -1496,6 +1602,24 @@ func (i *fragmentBlockIter) decodeSpanKeys(k *InternalKey, internalValue []byte)
}
}
func (i *fragmentBlockIter) elideKeysOfSameSeqNum() {
if invariants.Enabled {
if !i.elideSameSeqnum || len(i.span.Keys) == 0 {
panic("elideKeysOfSameSeqNum called when it should not be")
}
}
lastSeqNum := i.span.Keys[0].SeqNum()
k := 1
for j := 1; j < len(i.span.Keys); j++ {
if lastSeqNum != i.span.Keys[j].SeqNum() {
lastSeqNum = i.span.Keys[j].SeqNum()
i.span.Keys[k] = i.span.Keys[j]
k++
}
}
i.span.Keys = i.span.Keys[:k]
}
// gatherForward gathers internal keys with identical bounds. Keys defined over
// spans of the keyspace are fragmented such that any overlapping key spans have
// identical bounds. When these spans are persisted to a range deletion or range
@@ -1545,6 +1669,9 @@ func (i *fragmentBlockIter) gatherForward(k *InternalKey, lazyValue base.LazyVal
k, lazyValue = i.blockIter.Next()
internalValue = lazyValue.InPlaceValue()
}
if i.elideSameSeqnum && len(i.span.Keys) > 0 {
i.elideKeysOfSameSeqNum()
}
// i.blockIter is positioned over the first internal key for the next span.
return &i.span
}
@@ -1604,6 +1731,9 @@ func (i *fragmentBlockIter) gatherBackward(k *InternalKey, lazyValue base.LazyVa
// Backwards iteration encounters internal keys in the wrong order.
keyspan.SortKeysByTrailer(&i.span.Keys)
if i.elideSameSeqnum && len(i.span.Keys) > 0 {
i.elideKeysOfSameSeqNum()
}
return &i.span
}

30
vendor/github.com/cockroachdb/pebble/sstable/block_property.go generated vendored
View File

@@ -9,6 +9,7 @@ import (
"fmt"
"math"
"sync"
"unsafe"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/rangekey"
@@ -117,7 +118,7 @@ type BlockPropertyCollector interface {
}
// SuffixReplaceableBlockCollector is an extension to the BlockPropertyCollector
// interface that allows a block property collector to indicate the it supports
// interface that allows a block property collector to indicate that it supports
// being *updated* during suffix replacement, i.e. when an existing SST in which
// all keys have the same key suffix is updated to have a new suffix.
//
@@ -189,13 +190,13 @@ type BoundLimitedBlockPropertyFilter interface {
// indicates that the filter may be used to filter blocks that exclusively
// contain keys ≥ `key`, so long as the blocks' keys also satisfy the upper
// bound.
KeyIsWithinLowerBound(key *InternalKey) bool
KeyIsWithinLowerBound(key []byte) bool
// KeyIsWithinUpperBound tests whether the provided internal key falls
// within the current upper bound of the filter. A true return value
// indicates that the filter may be used to filter blocks that exclusively
// contain keys ≤ `key`, so long as the blocks' keys also satisfy the lower
// bound.
KeyIsWithinUpperBound(key *InternalKey) bool
KeyIsWithinUpperBound(key []byte) bool
}
// BlockIntervalCollector is a helper implementation of BlockPropertyCollector
@@ -643,15 +644,21 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
// considered intersecting.
continue
}
byteProps := []byte(props)
if len(byteProps) < 1 {
if len(props) < 1 {
return false, base.CorruptionErrorf(
"block properties for %s is corrupted", f.filters[i].Name())
}
shortID := shortID(byteProps[0])
intersects, err := f.filters[i].Intersects(byteProps[1:])
if err != nil || !intersects {
return false, err
shortID := shortID(props[0])
{
// Use an unsafe conversion to avoid allocating. Intersects() is not
// supposed to modify the given slice.
// Note that unsafe.StringData only works if the string is not empty
// (which we already checked).
byteProps := unsafe.Slice(unsafe.StringData(props), len(props))
intersects, err := f.filters[i].Intersects(byteProps[1:])
if err != nil || !intersects {
return false, err
}
}
// Intersects the sstable, so need to use this filter when
// deciding whether to read blocks.
@@ -684,12 +691,11 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
// be unused within this file.
return true, nil
}
byteProps := []byte(props)
if len(byteProps) < 1 {
if len(props) < 1 {
return false, base.CorruptionErrorf(
"block properties for %s is corrupted", f.boundLimitedFilter.Name())
}
f.boundLimitedShortID = int(byteProps[0])
f.boundLimitedShortID = int(props[0])
// We don't check for table-level intersection for the bound-limited filter.
// The bound-limited filter is treated as vacuously intersecting.

Some files were not shown because too many files have changed in this diff Show More