aleesa-bot/aleesa-phrases-go
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
eleksir
2022-05-16 02:21:12 +03:00
commit 7a525eed44
986 changed files with 401228 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
vendor/github.com/cockroachdb/pebble/.editorconfig generated vendored Normal file
View File

@@ -0,0 +1,10 @@
# See http://editorconfig.org
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
# For non-go files, we indent with two spaces. In go files we indent
# with tabs but still set indent_size to control the github web viewer.
indent_size=2

4
vendor/github.com/cockroachdb/pebble/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,4 @@
# Profiling artifacts.
cpu.*.prof
heap.prof
mutex.prof

27
vendor/github.com/cockroachdb/pebble/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,27 @@
Copyright (c) 2011 The LevelDB-Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

83
vendor/github.com/cockroachdb/pebble/Makefile generated vendored Normal file
View File

@@ -0,0 +1,83 @@
GO := go
PKG := ./...
GOFLAGS :=
STRESSFLAGS :=
TAGS := invariants
TESTS := .
.PHONY: all
all:
@echo usage:
@echo " make test"
@echo " make testrace"
@echo " make stress"
@echo " make stressrace"
@echo " make stressmeta"
@echo " make mod-update"
@echo " make clean"
override testflags :=
.PHONY: test
test:
${GO} test -mod=vendor -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG}
.PHONY: testrace
testrace: testflags += -race -timeout 20m
testrace: test
.PHONY: stress stressrace
stressrace: testflags += -race
stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 -test.v
stress stressrace: test
.PHONY: stressmeta
stressmeta: override PKG = ./internal/metamorphic
stressmeta: override STRESSFLAGS += -p 1
stressmeta: override TESTS = TestMeta$$
stressmeta: stress
.PHONY: generate
generate:
${GO} generate -mod=vendor ${PKG}
mod-update:
${GO} get -u
${GO} mod tidy
${GO} mod vendor
.PHONY: clean
clean:
rm -f $(patsubst %,%.test,$(notdir $(shell go list ${PKG})))
git_dirty := $(shell git status -s)
.PHONY: git-clean-check
git-clean-check:
ifneq ($(git_dirty),)
@echo "Git repository is dirty!"
@false
else
@echo "Git repository is clean."
endif
.PHONY: mod-tidy-check
mod-tidy-check:
ifneq ($(git_dirty),)
$(error mod-tidy-check must be invoked on a clean repository)
endif
@${GO} mod tidy
$(MAKE) git-clean-check
.PHONY: format
format:
for _file in $$(gofmt -s -l . | grep -vE '^vendor/'); do \
gofmt -s -w $$_file ; \
done
.PHONY: format-check
format-check:
ifneq ($(git_dirty),)
$(error format-check must be invoked on a clean repository)
endif
$(MAKE) format
$(MAKE) git-clean-check

157
vendor/github.com/cockroachdb/pebble/README.md generated vendored Normal file
View File

@@ -0,0 +1,157 @@
# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble)
#### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
Pebble is a LevelDB/RocksDB inspired key-value store focused on
performance and internal usage by CockroachDB. Pebble inherits the
RocksDB file formats and a few extensions such as range deletion
tombstones, table-level bloom filters, and updates to the MANIFEST
format.
Pebble intentionally does not aspire to include every feature in
RocksDB and is specifically targetting the use case and feature set
needed by CockroachDB:
* Block-based tables
* Checkpoints
* Indexed batches
* Iterator options (lower/upper bound, table filter)
* Level-based compaction
* Manual compaction
* Merge operator
* Prefix bloom filters
* Prefix iteration
* Range deletion tombstones
* Reverse iteration
* SSTable ingestion
* Single delete
* Snapshots
* Table-level bloom filters
RocksDB has a large number of features that are not implemented in
Pebble:
* Backups
* Column families
* Delete files in range
* FIFO compaction style
* Forward iterator / tailing iterator
* Hash table format
* Memtable bloom filter
* Persistent cache
* Pin iterator key / value
* Plain table format
* SSTable ingest-behind
* Sub-compactions
* Transactions
* Universal compaction style
***WARNING***: Pebble may silently corrupt data or behave incorrectly if
used with a RocksDB database that uses a feature Pebble doesn't
support. Caveat emptor!
## Production Ready
Pebble was introduced as an alternative storage engine to RocksDB in
CockroachDB v20.1 (released May 2020) and was used in production
successfully at that time. Pebble was made the default storage engine
in CockroachDB v20.2 (released Nov 2020). Pebble is being used in
production by users of CockroachDB at scale and is considered stable
and production ready.
## Advantages
Pebble offers several improvements over RocksDB:
* Faster reverse iteration via backwards links in the memtable's
skiplist.
* Faster commit pipeline that achieves better concurrency.
* Seamless merged iteration of indexed batches. The mutations in the
batch conceptually occupy another memtable level.
* Smaller, more approachable code base.
See the [Pebble vs RocksDB: Implementation
Differences](docs/rocksdb.md) doc for more details on implementation
differences.
## RocksDB Compatibility
Pebble strives for forward compatibility with RocksDB 6.2.1 (the latest
version of RocksDB used by CockroachDB). Forward compatibility means
that a DB generated by RocksDB can be used by Pebble. Currently, Pebble
provides bidirectional compatibility with RocksDB (a Pebble generated DB
can be used by RocksDB) when using its FormatMostCompatible format. New
functionality that is backwards incompatible is gated behind new format
major versions. In general, Pebble only provides compatibility with the
subset of functionality and configuration used by CockroachDB. The scope
of RocksDB functionality and configuration is too large to adequately
test and document all the incompatibilities. The list below contains
known incompatibilities.
* Pebble's use of WAL recycling is only compatible with RocksDB's
`kTolerateCorruptedTailRecords` WAL recovery mode. Older versions of
RocksDB would automatically map incompatible WAL recovery modes to
`kTolerateCorruptedTailRecords`. New versions of RocksDB will
disable WAL recycling.
* Column families. Pebble does not support column families, nor does
it attempt to detect their usage when opening a DB that may contain
them.
* Hash table format. Pebble does not support the hash table sstable
format.
* Plain table format. Pebble does not support the plain table sstable
format.
* SSTable format version 3 and 4. Pebble does not currently support
version 3 and version 4 format sstables. The sstable format version
is controlled by the `BlockBasedTableOptions::format_version`
option. See [#97](https://github.com/cockroachdb/pebble/issues/97).
## Pedigree
Pebble is based on the incomplete Go version of LevelDB:
https://github.com/golang/leveldb
The Go version of LevelDB is based on the C++ original:
https://github.com/google/leveldb
Optimizations and inspiration were drawn from RocksDB:
https://github.com/facebook/rocksdb
## Getting Started
### Example Code
```go
package main
import (
"fmt"
"log"
"github.com/cockroachdb/pebble"
)
func main() {
db, err := pebble.Open("demo", &pebble.Options{})
if err != nil {
log.Fatal(err)
}
key := []byte("hello")
if err := db.Set(key, []byte("world"), pebble.Sync); err != nil {
log.Fatal(err)
}
value, closer, err := db.Get(key)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s %s\n", key, value)
if err := closer.Close(); err != nil {
log.Fatal(err)
}
if err := db.Close(); err != nil {
log.Fatal(err)
}
}
```

1791
vendor/github.com/cockroachdb/pebble/batch.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

23
vendor/github.com/cockroachdb/pebble/cache.go generated vendored Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import "github.com/cockroachdb/pebble/internal/cache"
// Cache exports the cache.Cache type.
type Cache = cache.Cache
// NewCache creates a new cache of the specified size. Memory for the cache is
// allocated on demand, not during initialization. The cache is created with a
// reference count of 1. Each DB it is associated with adds a reference, so the
// creator of the cache should usually release their reference after the DB is
// created.
//
// c := pebble.NewCache(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func NewCache(size int64) *cache.Cache {
return cache.New(size)
}

287
vendor/github.com/cockroachdb/pebble/checkpoint.go generated vendored Normal file
View File

@@ -0,0 +1,287 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"os"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
// checkpointOptions hold the optional parameters to construct checkpoint
// snapshots.
type checkpointOptions struct {
// flushWAL set to true will force a flush and sync of the WAL prior to
// checkpointing.
flushWAL bool
}
// CheckpointOption set optional parameters used by `DB.Checkpoint`.
type CheckpointOption func(*checkpointOptions)
// WithFlushedWAL enables flushing and syncing the WAL prior to constructing a
// checkpoint. This guarantees that any writes committed before calling
// DB.Checkpoint will be part of that checkpoint.
//
// Note that this setting can only be useful in cases when some writes are
// performed with Sync = false. Otherwise, the guarantee will already be met.
//
// Passing this option is functionally equivalent to calling
// DB.LogData(nil, Sync) right before DB.Checkpoint.
func WithFlushedWAL() CheckpointOption {
return func(opt *checkpointOptions) {
opt.flushWAL = true
}
}
// mkdirAllAndSyncParents creates destDir and any of its missing parents.
// Those missing parents, as well as the closest existing ancestor, are synced.
// Returns a handle to the directory created at destDir.
func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
// Collect paths for all directories between destDir (excluded) and its
// closest existing ancestor (included).
var parentPaths []string
foundExistingAncestor := false
for parentPath := fs.PathDir(destDir); parentPath != "."; parentPath = fs.PathDir(parentPath) {
parentPaths = append(parentPaths, parentPath)
_, err := fs.Stat(parentPath)
if err == nil {
// Exit loop at the closest existing ancestor.
foundExistingAncestor = true
break
}
if !oserror.IsNotExist(err) {
return nil, err
}
}
// Handle empty filesystem edge case.
if !foundExistingAncestor {
parentPaths = append(parentPaths, "")
}
// Create destDir and any of its missing parents.
if err := fs.MkdirAll(destDir, 0755); err != nil {
return nil, err
}
// Sync all the parent directories up to the closest existing ancestor,
// included.
for _, parentPath := range parentPaths {
parentDir, err := fs.OpenDir(parentPath)
if err != nil {
return nil, err
}
err = parentDir.Sync()
if err != nil {
_ = parentDir.Close()
return nil, err
}
err = parentDir.Close()
if err != nil {
return nil, err
}
}
return fs.OpenDir(destDir)
}
// Checkpoint constructs a snapshot of the DB instance in the specified
// directory. The WAL, MANIFEST, OPTIONS, and sstables will be copied into the
// snapshot. Hard links will be used when possible. Beware of the significant
// space overhead for a checkpoint if hard links are disabled. Also beware that
// even if hard links are used, the space overhead for the checkpoint will
// increase over time as the DB performs compactions.
func (d *DB) Checkpoint(
destDir string, opts ...CheckpointOption,
) (
ckErr error, /* used in deferred cleanup */
) {
opt := &checkpointOptions{}
for _, fn := range opts {
fn(opt)
}
if _, err := d.opts.FS.Stat(destDir); !oserror.IsNotExist(err) {
if err == nil {
return &os.PathError{
Op: "checkpoint",
Path: destDir,
Err: oserror.ErrExist,
}
}
return err
}
if opt.flushWAL && !d.opts.DisableWAL {
// Write an empty log-data record to flush and sync the WAL.
if err := d.LogData(nil /* data */, Sync); err != nil {
return err
}
}
// Disable file deletions.
d.mu.Lock()
d.disableFileDeletions()
defer func() {
d.mu.Lock()
defer d.mu.Unlock()
d.enableFileDeletions()
}()
// TODO(peter): RocksDB provides the option to roll the manifest if the
// MANIFEST size is too large. Should we do this too?
// Lock the manifest before getting the current version. We need the
// length of the manifest that we read to match the current version that
// we read, otherwise we might copy a versionEdit not reflected in the
// sstables we copy/link.
d.mu.versions.logLock()
// Get the unflushed log files, the current version, and the current manifest
// file number.
memQueue := d.mu.mem.queue
current := d.mu.versions.currentVersion()
formatVers := d.mu.formatVers.vers
manifestFileNum := d.mu.versions.manifestFileNum
manifestSize := d.mu.versions.manifest.Size()
optionsFileNum := d.optionsFileNum
// Release the manifest and DB.mu so we don't block other operations on
// the database.
d.mu.versions.logUnlock()
d.mu.Unlock()
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := syncingFS{
FS: d.opts.FS,
syncOpts: vfs.SyncingFileOptions{
BytesPerSync: d.opts.BytesPerSync,
},
}
// Create the dir and its parents (if necessary), and sync them.
var dir vfs.File
defer func() {
if dir != nil {
_ = dir.Close()
}
if ckErr != nil {
// Attempt to cleanup on error.
paths, _ := fs.List(destDir)
for _, path := range paths {
_ = fs.Remove(path)
}
_ = fs.Remove(destDir)
}
}()
dir, ckErr = mkdirAllAndSyncParents(fs, destDir)
if ckErr != nil {
return ckErr
}
{
// Link or copy the OPTIONS.
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeOptions, optionsFileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
if ckErr != nil {
return ckErr
}
}
{
// Set the format major version in the destination directory.
var versionMarker *atomicfs.Marker
versionMarker, _, ckErr = atomicfs.LocateMarker(fs, destDir, formatVersionMarkerName)
if ckErr != nil {
return ckErr
}
// We use the marker to encode the active format version in the
// marker filename. Unlike other uses of the atomic marker,
// there is no file with the filename `formatVers.String()` on
// the filesystem.
ckErr = versionMarker.Move(formatVers.String())
if ckErr != nil {
return ckErr
}
ckErr = versionMarker.Close()
if ckErr != nil {
return ckErr
}
}
{
// Copy the MANIFEST, and create a pointer to it. We copy rather
// than link because additional version edits added to the
// MANIFEST after we took our snapshot of the sstables will
// reference sstables that aren't in our checkpoint. For a
// similar reason, we need to limit how much of the MANIFEST we
// copy.
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LimitedCopy(fs, srcPath, destPath, manifestSize)
if ckErr != nil {
return ckErr
}
// Recent format versions use an atomic marker for setting the
// active manifest. Older versions use the CURRENT file. The
// setCurrentFunc function will return a closure that will
// take the appropriate action for the database's format
// version.
var manifestMarker *atomicfs.Marker
manifestMarker, _, ckErr = atomicfs.LocateMarker(fs, destDir, manifestMarkerName)
if ckErr != nil {
return ckErr
}
ckErr = setCurrentFunc(formatVers, manifestMarker, fs, destDir, dir)(manifestFileNum)
if ckErr != nil {
return ckErr
}
ckErr = manifestMarker.Close()
if ckErr != nil {
return ckErr
}
}
// Link or copy the sstables.
for l := range current.Levels {
iter := current.Levels[l].Iter()
for f := iter.First(); f != nil; f = iter.Next() {
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, f.FileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
if ckErr != nil {
return ckErr
}
}
}
// Copy the WAL files. We copy rather than link because WAL file recycling
// will cause the WAL files to be reused which would invalidate the
// checkpoint.
for i := range memQueue {
logNum := memQueue[i].logNum
if logNum == 0 {
continue
}
srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.Copy(fs, srcPath, destPath)
if ckErr != nil {
return ckErr
}
}
// Sync and close the checkpoint directory.
ckErr = dir.Sync()
if ckErr != nil {
return ckErr
}
ckErr = dir.Close()
dir = nil
return ckErr
}

16
vendor/github.com/cockroachdb/pebble/cleaner.go generated vendored Normal file
View File

@@ -0,0 +1,16 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import "github.com/cockroachdb/pebble/internal/base"
// Cleaner exports the base.Cleaner type.
type Cleaner = base.Cleaner
// DeleteCleaner exports the base.DeleteCleaner type.
type DeleteCleaner = base.DeleteCleaner
// ArchiveCleaner exports the base.ArchiveCleaner type.
type ArchiveCleaner = base.ArchiveCleaner

443
vendor/github.com/cockroachdb/pebble/commit.go generated vendored Normal file
View File

@@ -0,0 +1,443 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"runtime"
"sync"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/pebble/record"
)
// commitQueue is a lock-free fixed-size single-producer, multi-consumer
// queue. The single producer can enqueue (push) to the head, and consumers can
// dequeue (pop) from the tail.
//
// It has the added feature that it nils out unused slots to avoid unnecessary
// retention of objects.
type commitQueue struct {
// headTail packs together a 32-bit head index and a 32-bit tail index. Both
// are indexes into slots modulo len(slots)-1.
//
// tail = index of oldest data in queue
// head = index of next slot to fill
//
// Slots in the range [tail, head) are owned by consumers. A consumer
// continues to own a slot outside this range until it nils the slot, at
// which point ownership passes to the producer.
//
// The head index is stored in the most-significant bits so that we can
// atomically add to it and the overflow is harmless.
headTail uint64
// slots is a ring buffer of values stored in this queue. The size must be a
// power of 2. A slot is in use until *both* the tail index has moved beyond
// it and the slot value has been set to nil. The slot value is set to nil
// atomically by the consumer and read atomically by the producer.
slots [record.SyncConcurrency]unsafe.Pointer
}
const dequeueBits = 32
func (q *commitQueue) unpack(ptrs uint64) (head, tail uint32) {
const mask = 1<<dequeueBits - 1
head = uint32((ptrs >> dequeueBits) & mask)
tail = uint32(ptrs & mask)
return
}
func (q *commitQueue) pack(head, tail uint32) uint64 {
const mask = 1<<dequeueBits - 1
return (uint64(head) << dequeueBits) |
uint64(tail&mask)
}
func (q *commitQueue) enqueue(b *Batch) {
ptrs := atomic.LoadUint64(&q.headTail)
head, tail := q.unpack(ptrs)
if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
// Queue is full. This should never be reached because commitPipeline.sem
// limits the number of concurrent operations.
panic("pebble: not reached")
}
slot := &q.slots[head&uint32(len(q.slots)-1)]
// Check if the head slot has been released by dequeue.
for atomic.LoadPointer(slot) != nil {
// Another goroutine is still cleaning up the tail, so the queue is
// actually still full. We spin because this should resolve itself
// momentarily.
runtime.Gosched()
}
// The head slot is free, so we own it.
atomic.StorePointer(slot, unsafe.Pointer(b))
// Increment head. This passes ownership of slot to dequeue and acts as a
// store barrier for writing the slot.
atomic.AddUint64(&q.headTail, 1<<dequeueBits)
}
func (q *commitQueue) dequeue() *Batch {
for {
ptrs := atomic.LoadUint64(&q.headTail)
head, tail := q.unpack(ptrs)
if tail == head {
// Queue is empty.
return nil
}
slot := &q.slots[tail&uint32(len(q.slots)-1)]
b := (*Batch)(atomic.LoadPointer(slot))
if b == nil || atomic.LoadUint32(&b.applied) == 0 {
// The batch is not ready to be dequeued, or another goroutine has
// already dequeued it.
return nil
}
// Confirm head and tail (for our speculative check above) and increment
// tail. If this succeeds, then we own the slot at tail.
ptrs2 := q.pack(head, tail+1)
if atomic.CompareAndSwapUint64(&q.headTail, ptrs, ptrs2) {
// We now own slot.
//
// Tell enqueue that we're done with this slot. Zeroing the slot is also
// important so we don't leave behind references that could keep this object
// live longer than necessary.
atomic.StorePointer(slot, nil)
// At this point enqueue owns the slot.
return b
}
}
}
// commitEnv contains the environment that a commitPipeline interacts
// with. This allows fine-grained testing of commitPipeline behavior without
// construction of an entire DB.
type commitEnv struct {
// The next sequence number to give to a batch. Protected by
// commitPipeline.mu.
logSeqNum *uint64
// The visible sequence number at which reads should be performed. Ratcheted
// upwards atomically as batches are applied to the memtable.
visibleSeqNum *uint64
// Apply the batch to the specified memtable. Called concurrently.
apply func(b *Batch, mem *memTable) error
// Write the batch to the WAL. If wg != nil, the data will be persisted
// asynchronously and done will be called on wg upon completion. If wg != nil
// and err != nil, a failure to persist the WAL will populate *err. Returns
// the memtable the batch should be applied to. Serial execution enforced by
// commitPipeline.mu.
write func(b *Batch, wg *sync.WaitGroup, err *error) (*memTable, error)
}
// A commitPipeline manages the stages of committing a set of mutations
// (contained in a single Batch) atomically to the DB. The steps are
// conceptually:
//
// 1. Write the batch to the WAL and optionally sync the WAL
// 2. Apply the mutations in the batch to the memtable
//
// These two simple steps are made complicated by the desire for high
// performance. In the absence of concurrency, performance is limited by how
// fast a batch can be written (and synced) to the WAL and then added to the
// memtable, both of which are outside the purview of the commit
// pipeline. Performance under concurrency is the primary concern of the commit
// pipeline, though it also needs to maintain two invariants:
//
// 1. Batches need to be written to the WAL in sequence number order.
// 2. Batches need to be made visible for reads in sequence number order. This
// invariant arises from the use of a single sequence number which
// indicates which mutations are visible.
//
// Taking these invariants into account, let's revisit the work the commit
// pipeline needs to perform. Writing the batch to the WAL is necessarily
// serialized as there is a single WAL object. The order of the entries in the
// WAL defines the sequence number order. Note that writing to the WAL is
// extremely fast, usually just a memory copy. Applying the mutations in a
// batch to the memtable can occur concurrently as the underlying skiplist
// supports concurrent insertions. Publishing the visible sequence number is
// another serialization point, but one with a twist: the visible sequence
// number cannot be bumped until the mutations for earlier batches have
// finished applying to the memtable (the visible sequence number only ratchets
// up). Lastly, if requested, the commit waits for the WAL to sync. Note that
// waiting for the WAL sync after ratcheting the visible sequence number allows
// another goroutine to read committed data before the WAL has synced. This is
// similar behavior to RocksDB's manual WAL flush functionality. Application
// code needs to protect against this if necessary.
//
// The full outline of the commit pipeline operation is as follows:
//
// with commitPipeline mutex locked:
// assign batch sequence number
// write batch to WAL
// (optionally) add batch to WAL sync list
// apply batch to memtable (concurrently)
// wait for earlier batches to apply
// ratchet read sequence number
// (optionally) wait for the WAL to sync
//
// As soon as a batch has been written to the WAL, the commitPipeline mutex is
// released allowing another batch to write to the WAL. Each commit operation
// individually applies its batch to the memtable providing concurrency. The
// WAL sync happens concurrently with applying to the memtable (see
// commitPipeline.syncLoop).
//
// The "waits for earlier batches to apply" work is more complicated than might
// be expected. The obvious approach would be to keep a queue of pending
// batches and for each batch to wait for the previous batch to finish
// committing. This approach was tried initially and turned out to be too
// slow. The problem is that it causes excessive goroutine activity as each
// committing goroutine needs to wake up in order for the next goroutine to be
// unblocked. The approach taken in the current code is conceptually similar,
// though it avoids waking a goroutine to perform work that another goroutine
// can perform. A commitQueue (a single-producer, multiple-consumer queue)
// holds the ordered list of committing batches. Addition to the queue is done
// while holding commitPipeline.mutex ensuring the same ordering of batches in
// the queue as the ordering in the WAL. When a batch finishes applying to the
// memtable, it atomically updates its Batch.applied field. Ratcheting of the
// visible sequence number is done by commitPipeline.publish which loops
// dequeueing "applied" batches and ratcheting the visible sequence number. If
// we hit an unapplied batch at the head of the queue we can block as we know
// that committing of that unapplied batch will eventually find our (applied)
// batch in the queue. See commitPipeline.publish for additional commentary.
type commitPipeline struct {
// WARNING: The following struct `commitQueue` contains fields which will
// be accessed atomically.
//
// Go allocations are guaranteed to be 64-bit aligned which we take advantage
// of by placing the 64-bit fields which we access atomically at the beginning
// of the commitPipeline struct.
// For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
// Queue of pending batches to commit.
pending commitQueue
env commitEnv
sem chan struct{}
// The mutex to use for synchronizing access to logSeqNum and serializing
// calls to commitEnv.write().
mu sync.Mutex
}
func newCommitPipeline(env commitEnv) *commitPipeline {
p := &commitPipeline{
env: env,
// NB: the commit concurrency is one less than SyncConcurrency because we
// have to allow one "slot" for a concurrent WAL rotation which will close
// and sync the WAL.
sem: make(chan struct{}, record.SyncConcurrency-1),
}
return p
}
// Commit the specified batch, writing it to the WAL, optionally syncing the
// WAL, and applying the batch to the memtable. Upon successful return the
// batch's mutations will be visible for reading.
func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error {
if b.Empty() {
return nil
}
p.sem <- struct{}{}
// Prepare the batch for committing: enqueuing the batch in the pending
// queue, determining the batch sequence number and writing the data to the
// WAL.
//
// NB: We set Batch.commitErr on error so that the batch won't be a candidate
// for reuse. See Batch.release().
mem, err := p.prepare(b, syncWAL)
if err != nil {
b.db = nil // prevent batch reuse on error
return err
}
// Apply the batch to the memtable.
if err := p.env.apply(b, mem); err != nil {
b.db = nil // prevent batch reuse on error
return err
}
// Publish the batch sequence number.
p.publish(b)
<-p.sem
if b.commitErr != nil {
b.db = nil // prevent batch reuse on error
}
return b.commitErr
}
// AllocateSeqNum allocates count sequence numbers, invokes the prepare
// callback, then the apply callback, and then publishes the sequence
// numbers. AllocateSeqNum does not write to the WAL or add entries to the
// memtable. AllocateSeqNum can be used to sequence an operation such as
// sstable ingestion within the commit pipeline. The prepare callback is
// invoked with commitPipeline.mu held, but note that DB.mu is not held and
// must be locked if necessary.
func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(seqNum uint64)) {
// This method is similar to Commit and prepare. Be careful about trying to
// share additional code with those methods because Commit and prepare are
// performance critical code paths.
b := newBatch(nil)
defer b.release()
// Give the batch a count of 1 so that the log and visible sequence number
// are incremented correctly.
b.data = make([]byte, batchHeaderLen)
b.setCount(uint32(count))
b.commit.Add(1)
p.sem <- struct{}{}
p.mu.Lock()
// Enqueue the batch in the pending queue. Note that while the pending queue
// is lock-free, we want the order of batches to be the same as the sequence
// number order.
p.pending.enqueue(b)
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
logSeqNum := atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
seqNum := logSeqNum
if seqNum == 0 {
// We can't use the value 0 for the global seqnum during ingestion, because
// 0 indicates no global seqnum. So allocate one more seqnum.
atomic.AddUint64(p.env.logSeqNum, 1)
seqNum++
}
b.setSeqNum(seqNum)
// Wait for any outstanding writes to the memtable to complete. This is
// necessary for ingestion so that the check for memtable overlap can see any
// writes that were sequenced before the ingestion. The spin loop is
// unfortunate, but obviates the need for additional synchronization.
for {
visibleSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
if visibleSeqNum == logSeqNum {
break
}
runtime.Gosched()
}
// Invoke the prepare callback. Note the lack of error reporting. Even if the
// callback internally fails, the sequence number needs to be published in
// order to allow the commit pipeline to proceed.
prepare()
p.mu.Unlock()
// Invoke the apply callback.
apply(b.SeqNum())
// Publish the sequence number.
p.publish(b)
<-p.sem
}
func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) {
n := uint64(b.Count())
if n == invalidBatchCount {
return nil, ErrInvalidBatch
}
count := 1
if syncWAL {
count++
}
// count represents the waiting needed for publish, and optionally the
// waiting needed for the WAL sync.
b.commit.Add(count)
var syncWG *sync.WaitGroup
var syncErr *error
if syncWAL {
syncWG, syncErr = &b.commit, &b.commitErr
}
p.mu.Lock()
// Enqueue the batch in the pending queue. Note that while the pending queue
// is lock-free, we want the order of batches to be the same as the sequence
// number order.
p.pending.enqueue(b)
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
b.setSeqNum(atomic.AddUint64(p.env.logSeqNum, n) - n)
// Write the data to the WAL.
mem, err := p.env.write(b, syncWG, syncErr)
p.mu.Unlock()
return mem, err
}
func (p *commitPipeline) publish(b *Batch) {
// Mark the batch as applied.
atomic.StoreUint32(&b.applied, 1)
// Loop dequeuing applied batches from the pending queue. If our batch was
// the head of the pending queue we are guaranteed that either we'll publish
// it or someone else will dequeue and publish it. If our batch is not the
// head of the queue then either we'll dequeue applied batches and reach our
// batch or there is an unapplied batch blocking us. When that unapplied
// batch applies it will go through the same process and publish our batch
// for us.
for {
t := p.pending.dequeue()
if t == nil {
// Wait for another goroutine to publish us. We might also be waiting for
// the WAL sync to finish.
b.commit.Wait()
break
}
if atomic.LoadUint32(&t.applied) != 1 {
panic("not reached")
}
// We're responsible for publishing the sequence number for batch t, but
// another concurrent goroutine might sneak in and publish the sequence
// number for a subsequent batch. That's ok as all we're guaranteeing is
// that the sequence number ratchets up.
for {
curSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
newSeqNum := t.SeqNum() + uint64(t.Count())
if newSeqNum <= curSeqNum {
// t's sequence number has already been published.
break
}
if atomic.CompareAndSwapUint64(p.env.visibleSeqNum, curSeqNum, newSeqNum) {
// We successfully published t's sequence number.
break
}
}
t.commit.Done()
}
}
// ratchetSeqNum allocates and marks visible all sequence numbers less than
// but excluding `nextSeqNum`.
func (p *commitPipeline) ratchetSeqNum(nextSeqNum uint64) {
p.mu.Lock()
defer p.mu.Unlock()
logSeqNum := atomic.LoadUint64(p.env.logSeqNum)
if logSeqNum >= nextSeqNum {
return
}
count := nextSeqNum - logSeqNum
_ = atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
atomic.StoreUint64(p.env.visibleSeqNum, nextSeqNum)
}

2884
vendor/github.com/cockroachdb/pebble/compaction.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

835
vendor/github.com/cockroachdb/pebble/compaction_iter.go generated vendored Normal file
View File

@@ -0,0 +1,835 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"io"
"sort"
"strconv"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// compactionIter provides a forward-only iterator that encapsulates the logic
// for collapsing entries during compaction. It wraps an internal iterator and
// collapses entries that are no longer necessary because they are shadowed by
// newer entries. The simplest example of this is when the internal iterator
// contains two keys: a.PUT.2 and a.PUT.1. Instead of returning both entries,
// compactionIter collapses the second entry because it is no longer
// necessary. The high-level structure for compactionIter is to iterate over
// its internal iterator and output 1 entry for every user-key. There are four
// complications to this story.
//
// 1. Eliding Deletion Tombstones
//
// Consider the entries a.DEL.2 and a.PUT.1. These entries collapse to
// a.DEL.2. Do we have to output the entry a.DEL.2? Only if a.DEL.2 possibly
// shadows an entry at a lower level. If we're compacting to the base-level in
// the LSM tree then a.DEL.2 is definitely not shadowing an entry at a lower
// level and can be elided.
//
// We can do slightly better than only eliding deletion tombstones at the base
// level by observing that we can elide a deletion tombstone if there are no
// sstables that contain the entry's key. This check is performed by
// elideTombstone.
//
// 2. Merges
//
// The MERGE operation merges the value for an entry with the existing value
// for an entry. The logical value of an entry can be composed of a series of
// merge operations. When compactionIter sees a MERGE, it scans forward in its
// internal iterator collapsing MERGE operations for the same key until it
// encounters a SET or DELETE operation. For example, the keys a.MERGE.4,
// a.MERGE.3, a.MERGE.2 will be collapsed to a.MERGE.4 and the values will be
// merged using the specified Merger.
//
// An interesting case here occurs when MERGE is combined with SET. Consider
// the entries a.MERGE.3 and a.SET.2. The collapsed key will be a.SET.3. The
// reason that the kind is changed to SET is because the SET operation acts as
// a barrier preventing further merging. This can be seen better in the
// scenario a.MERGE.3, a.SET.2, a.MERGE.1. The entry a.MERGE.1 may be at lower
// (older) level and not involved in the compaction. If the compaction of
// a.MERGE.3 and a.SET.2 produced a.MERGE.3, a subsequent compaction with
// a.MERGE.1 would merge the values together incorrectly.
//
// 3. Snapshots
//
// Snapshots are lightweight point-in-time views of the DB state. At its core,
// a snapshot is a sequence number along with a guarantee from Pebble that it
// will maintain the view of the database at that sequence number. Part of this
// guarantee is relatively straightforward to achieve. When reading from the
// database Pebble will ignore sequence numbers that are larger than the
// snapshot sequence number. The primary complexity with snapshots occurs
// during compaction: the collapsing of entries that are shadowed by newer
// entries is at odds with the guarantee that Pebble will maintain the view of
// the database at the snapshot sequence number. Rather than collapsing entries
// up to the next user key, compactionIter can only collapse entries up to the
// next snapshot boundary. That is, every snapshot boundary potentially causes
// another entry for the same user-key to be emitted. Another way to view this
// is that snapshots define stripes and entries are collapsed within stripes,
// but not across stripes. Consider the following scenario:
//
// a.PUT.9
// a.DEL.8
// a.PUT.7
// a.DEL.6
// a.PUT.5
//
// In the absence of snapshots these entries would be collapsed to
// a.PUT.9. What if there is a snapshot at sequence number 7? The entries can
// be divided into two stripes and collapsed within the stripes:
//
// a.PUT.9 a.PUT.9
// a.DEL.8 --->
// a.PUT.7
// -- --
// a.DEL.6 ---> a.DEL.6
// a.PUT.5
//
// All of the rules described earlier still apply, but they are confined to
// operate within a snapshot stripe. Snapshots only affect compaction when the
// snapshot sequence number lies within the range of sequence numbers being
// compacted. In the above example, a snapshot at sequence number 10 or at
// sequence number 5 would not have any effect.
//
// 4. Range Deletions
//
// Range deletions provide the ability to delete all of the keys (and values)
// in a contiguous range. Range deletions are stored indexed by their start
// key. The end key of the range is stored in the value. In order to support
// lookup of the range deletions which overlap with a particular key, the range
// deletion tombstones need to be fragmented whenever they overlap. This
// fragmentation is performed by keyspan.Fragmenter. The fragments are then
// subject to the rules for snapshots. For example, consider the two range
// tombstones [a,e)#1 and [c,g)#2:
//
// 2: c-------g
// 1: a-------e
//
// These tombstones will be fragmented into:
//
// 2: c---e---g
// 1: a---c---e
//
// Do we output the fragment [c,e)#1? Since it is covered by [c-e]#2 the answer
// depends on whether it is in a new snapshot stripe.
//
// In addition to the fragmentation of range tombstones, compaction also needs
// to take the range tombstones into consideration when outputting normal
// keys. Just as with point deletions, a range deletion covering an entry can
// cause the entry to be elided.
//
// A note on the stability of keys and values.
//
// The stability guarantees of keys and values returned by the iterator tree
// that backs a compactionIter is nuanced and care must be taken when
// referencing any returned items.
//
// Keys and values returned by exported functions (i.e. First, Next, etc.) have
// lifetimes that fall into two categories:
//
// Lifetime valid for duration of compaction. Range deletion keys and values are
// stable for the duration of the compaction, due to way in which a
// compactionIter is typically constructed (i.e. via (*compaction).newInputIter,
// which wraps the iterator over the range deletion block in a noCloseIter,
// preventing the release of the backing memory until the compaction is
// finished).
//
// Lifetime limited to duration of sstable block liveness. Point keys (SET, DEL,
// etc.) and values must be cloned / copied following the return from the
// exported function, and before a subsequent call to Next advances the iterator
// and mutates the contents of the returned key and value.
type compactionIter struct {
equal Equal
merge Merge
iter internalIterator
err error
// `key.UserKey` is set to `keyBuf` caused by saving `i.iterKey.UserKey`
// and `key.Trailer` is set to `i.iterKey.Trailer`. This is the
// case on return from all public methods -- these methods return `key`.
// Additionally, it is the internal state when the code is moving to the
// next key so it can determine whether the user key has changed from
// the previous key.
key InternalKey
value []byte
valueCloser io.Closer
// Temporary buffer used for storing the previous user key in order to
// determine when iteration has advanced to a new user key and thus a new
// snapshot stripe.
keyBuf []byte
// Temporary buffer used for storing the previous value, which may be an
// unsafe, i.iter-owned slice that could be altered when the iterator is
// advanced.
valueBuf []byte
// Is the current entry valid?
valid bool
iterKey *InternalKey
iterValue []byte
// `skip` indicates whether the remaining skippable entries in the current
// snapshot stripe should be skipped or processed. An example of a non-
// skippable entry is a range tombstone as we need to return it from the
// `compactionIter`, even if a key covering its start key has already been
// seen in the same stripe. `skip` has no effect when `pos == iterPosNext`.
skip bool
// `pos` indicates the iterator position at the top of `Next()`. Its type's
// (`iterPos`) values take on the following meanings in the context of
// `compactionIter`.
//
// - `iterPosCur`: the iterator is at the last key returned.
// - `iterPosNext`: the iterator has already been advanced to the next
// candidate key. For example, this happens when processing merge operands,
// where we advance the iterator all the way into the next stripe or next
// user key to ensure we've seen all mergeable operands.
// - `iterPosPrev`: this is invalid as compactionIter is forward-only.
pos iterPos
// The index of the snapshot for the current key within the snapshots slice.
curSnapshotIdx int
curSnapshotSeqNum uint64
// The snapshot sequence numbers that need to be maintained. These sequence
// numbers define the snapshot stripes (see the Snapshots description
// above). The sequence numbers are in ascending order.
snapshots []uint64
// Reference to the range deletion tombstone fragmenter (e.g.,
// `compaction.rangeDelFrag`).
rangeDelFrag *keyspan.Fragmenter
// The fragmented tombstones.
tombstones []keyspan.Span
// Byte allocator for the tombstone keys.
alloc bytealloc.A
allowZeroSeqNum bool
elideTombstone func(key []byte) bool
elideRangeTombstone func(start, end []byte) bool
// The on-disk format major version. This informs the types of keys that
// may be written to disk during a compaction.
formatVersion FormatMajorVersion
}
func newCompactionIter(
cmp Compare,
equal Equal,
formatKey base.FormatKey,
merge Merge,
iter internalIterator,
snapshots []uint64,
rangeDelFrag *keyspan.Fragmenter,
allowZeroSeqNum bool,
elideTombstone func(key []byte) bool,
elideRangeTombstone func(start, end []byte) bool,
formatVersion FormatMajorVersion,
) *compactionIter {
i := &compactionIter{
equal: equal,
merge: merge,
iter: iter,
snapshots: snapshots,
rangeDelFrag: rangeDelFrag,
allowZeroSeqNum: allowZeroSeqNum,
elideTombstone: elideTombstone,
elideRangeTombstone: elideRangeTombstone,
formatVersion: formatVersion,
}
i.rangeDelFrag.Cmp = cmp
i.rangeDelFrag.Format = formatKey
i.rangeDelFrag.Emit = i.emitRangeDelChunk
return i
}
func (i *compactionIter) First() (*InternalKey, []byte) {
if i.err != nil {
return nil, nil
}
i.iterKey, i.iterValue = i.iter.First()
if i.iterKey != nil {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(i.iterKey.SeqNum(), i.snapshots)
}
i.pos = iterPosNext
return i.Next()
}
func (i *compactionIter) Next() (*InternalKey, []byte) {
if i.err != nil {
return nil, nil
}
// Close the closer for the current value if one was open.
if i.closeValueCloser() != nil {
return nil, nil
}
// Prior to this call to `Next()` we are in one of three situations with
// respect to `iterKey` and related state:
//
// - `!skip && pos == iterPosNext`: `iterKey` is already at the next key.
// - `!skip && pos == iterPosCur`: We are at the key that has been returned.
// To move forward we advance by one key, even if that lands us in the same
// snapshot stripe.
// - `skip && pos == iterPosCur`: We are at the key that has been returned.
// To move forward we skip skippable entries in the stripe.
if i.pos == iterPosCurForward {
if i.skip {
i.skipInStripe()
} else {
i.nextInStripe()
}
}
i.pos = iterPosCurForward
i.valid = false
for i.iterKey != nil {
if i.iterKey.Kind() == InternalKeyKindRangeDelete {
// Return the range tombstone so the compaction can use it for
// file truncation and add it to the fragmenter. We do not set `skip`
// to true before returning as there may be a forthcoming point key
// with the same user key and sequence number. Such a point key must be
// visible (i.e., not skipped over) since we promise point keys are
// not deleted by range tombstones at the same sequence number.
//
// Although, note that `skip` may already be true before reaching here
// due to an earlier key in the stripe. Then it is fine to leave it set
// to true, as the earlier key must have had a higher sequence number.
//
// NOTE: there is a subtle invariant violation here in that calling
// saveKey and returning a reference to the temporary slice violates
// the stability guarantee for range deletion keys. A potential
// mediation could return the original iterKey and iterValue
// directly, as the backing memory is guaranteed to be stable until
// the compaction completes. The violation here is only minor in
// that the caller immediately clones the range deletion InternalKey
// when passing the key to the deletion fragmenter (see the
// call-site in compaction.go).
// TODO(travers): address this violation by removing the call to
// saveKey and instead return the original iterKey and iterValue.
// This goes against the comment on i.key in the struct, and
// therefore warrants some investigation.
i.saveKey()
i.value = i.iterValue
i.valid = true
return &i.key, i.value
}
if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) {
i.saveKey()
i.skipInStripe()
continue
}
switch i.iterKey.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
// If we're at the last snapshot stripe and the tombstone can be elided
// skip skippable keys in the same stripe.
if i.curSnapshotIdx == 0 && i.elideTombstone(i.iterKey.UserKey) {
i.saveKey()
i.skipInStripe()
continue
}
switch i.iterKey.Kind() {
case InternalKeyKindDelete:
i.saveKey()
i.value = i.iterValue
i.valid = true
i.skip = true
return &i.key, i.value
case InternalKeyKindSingleDelete:
if i.singleDeleteNext() {
return &i.key, i.value
}
continue
}
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
// The key we emit for this entry is a function of the current key
// kind, and whether this entry is followed by a DEL/SINGLEDEL
// entry. setNext() does the work to move the iterator forward,
// preserving the original value, and potentially mutating the key
// kind.
i.setNext()
return &i.key, i.value
case InternalKeyKindMerge:
// Record the snapshot index before mergeNext as merging
// advances the iterator, adjusting curSnapshotIdx.
origSnapshotIdx := i.curSnapshotIdx
var valueMerger ValueMerger
valueMerger, i.err = i.merge(i.iterKey.UserKey, i.iterValue)
var change stripeChangeType
if i.err == nil {
change = i.mergeNext(valueMerger)
}
var needDelete bool
if i.err == nil {
// includesBase is true whenever we've transformed the MERGE record
// into a SET.
includesBase := i.key.Kind() == InternalKeyKindSet
i.value, needDelete, i.valueCloser, i.err = finishValueMerger(valueMerger, includesBase)
}
if i.err == nil {
if needDelete {
i.valid = false
if i.closeValueCloser() != nil {
return nil, nil
}
continue
}
// A non-skippable entry does not necessarily cover later merge
// operands, so we must not zero the current merge result's seqnum.
//
// For example, suppose the forthcoming two keys are a range
// tombstone, `[a, b)#3`, and a merge operand, `a#3`. Recall that
// range tombstones do not cover point keys at the same seqnum, so
// `a#3` is not deleted. The range tombstone will be seen first due
// to its larger value type. Since it is a non-skippable key, the
// current merge will not include `a#3`. If we zeroed the current
// merge result's seqnum, then it would conflict with the upcoming
// merge including `a#3`, whose seqnum will also be zeroed.
if change != sameStripeNonSkippable {
i.maybeZeroSeqnum(origSnapshotIdx)
}
return &i.key, i.value
}
if i.err != nil {
i.valid = false
i.err = base.MarkCorruptionError(i.err)
}
return nil, nil
default:
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
return nil, nil
}
}
return nil, nil
}
func (i *compactionIter) closeValueCloser() error {
if i.valueCloser == nil {
return nil
}
i.err = i.valueCloser.Close()
i.valueCloser = nil
if i.err != nil {
i.valid = false
}
return i.err
}
// snapshotIndex returns the index of the first sequence number in snapshots
// which is greater than or equal to seq.
func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) {
index := sort.Search(len(snapshots), func(i int) bool {
return snapshots[i] > seq
})
if index >= len(snapshots) {
return index, InternalKeySeqNumMax
}
return index, snapshots[index]
}
// skipInStripe skips over skippable keys in the same stripe and user key.
func (i *compactionIter) skipInStripe() {
i.skip = true
var change stripeChangeType
for {
change = i.nextInStripe()
if change == sameStripeNonSkippable || change == newStripe {
break
}
}
// Reset skip if we landed outside the original stripe. Otherwise, we landed
// in the same stripe on a non-skippable key. In that case we should preserve
// `i.skip == true` such that later keys in the stripe will continue to be
// skipped.
if change == newStripe {
i.skip = false
}
}
func (i *compactionIter) iterNext() bool {
i.iterKey, i.iterValue = i.iter.Next()
// We should never see an exclusive sentinel in the compaction input.
if i.iterKey != nil && i.iterKey.IsExclusiveSentinel() {
panic(fmt.Sprintf("pebble: unexpected exclusive sentinel in compaction input, trailer = %x", i.iterKey.Trailer))
}
return i.iterKey != nil
}
// stripeChangeType indicates how the snapshot stripe changed relative to the previous
// key. If no change, it also indicates whether the current entry is skippable.
type stripeChangeType int
const (
newStripe stripeChangeType = iota
sameStripeSkippable
sameStripeNonSkippable
)
// nextInStripe advances the iterator and returns one of the above const ints
// indicating how its state changed.
//
// Calls to nextInStripe must be preceded by a call to saveKey to retain a
// temporary reference to the original key, so that forward iteration can
// proceed with a reference to the original key. Care should be taken to avoid
// overwriting or mutating the saved key or value before they have been returned
// to the caller of the exported function (i.e. the caller of Next, First, etc.)
func (i *compactionIter) nextInStripe() stripeChangeType {
if !i.iterNext() {
return newStripe
}
key := i.iterKey
if !i.equal(i.key.UserKey, key.UserKey) {
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
return newStripe
}
origSnapshotIdx := i.curSnapshotIdx
i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
switch key.Kind() {
case InternalKeyKindRangeDelete:
// Range tombstones need to be exposed by the compactionIter to the upper level
// `compaction` object, so return them regardless of whether they are in the same
// snapshot stripe.
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripe
case InternalKeyKindInvalid:
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeNonSkippable
}
return newStripe
}
if i.curSnapshotIdx == origSnapshotIdx {
return sameStripeSkippable
}
return newStripe
}
func (i *compactionIter) setNext() {
// Save the current key.
i.saveKey()
i.value = i.iterValue
i.valid = true
i.maybeZeroSeqnum(i.curSnapshotIdx)
// There are two cases where we can early return and skip the remaining
// records in the stripe:
// - If the DB does not SETWITHDEL.
// - If this key is already a SETWITHDEL.
if i.formatVersion < FormatSetWithDelete ||
i.iterKey.Kind() == InternalKeyKindSetWithDelete {
i.skip = true
return
}
// We are iterating forward. Save the current value.
i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
i.value = i.valueBuf
// Else, we continue to loop through entries in the stripe looking for a
// DEL. Note that we may stop *before* encountering a DEL, if one exists.
for {
switch t := i.nextInStripe(); t {
case newStripe, sameStripeNonSkippable:
i.pos = iterPosNext
if t == sameStripeNonSkippable {
// We iterated onto a key that we cannot skip. We can
// conservatively transform the original SET into a SETWITHDEL
// as an indication that there *may* still be a DEL/SINGLEDEL
// under this SET, even if we did not actually encounter one.
//
// This is safe to do, as:
//
// - in the case that there *is not* actually a DEL/SINGLEDEL
// under this entry, any SINGLEDEL above this now-transformed
// SETWITHDEL will become a DEL when the two encounter in a
// compaction. The DEL will eventually be elided in a
// subsequent compaction. The cost for ensuring correctness is
// that this entry is kept around for an additional compaction
// cycle(s).
//
// - in the case there *is* indeed a DEL/SINGLEDEL under us
// (but in a different stripe or sstable), then we will have
// already done the work to transform the SET into a
// SETWITHDEL, and we will skip any additional iteration when
// this entry is encountered again in a subsequent compaction.
//
// Ideally, this codepath would be smart enough to handle the
// case of SET <- RANGEDEL <- ... <- DEL/SINGLEDEL <- ....
// This requires preserving any RANGEDEL entries we encounter
// along the way, then emitting the original (possibly
// transformed) key, followed by the RANGEDELs. This requires
// a sizable refactoring of the existing code, as nextInStripe
// currently returns a sameStripeNonSkippable when it
// encounters a RANGEDEL.
// TODO(travers): optimize to handle the RANGEDEL case if it
// turns out to be a performance problem.
i.key.SetKind(InternalKeyKindSetWithDelete)
// By setting i.skip=true, we are saying that after the
// non-skippable key is emitted (which is likely a RANGEDEL),
// the remaining point keys that share the same user key as this
// saved key should be skipped.
i.skip = true
}
return
case sameStripeSkippable:
// We're still in the same stripe. If this is a DEL/SINGLEDEL, we
// stop looking and emit a SETWITHDEL. Subsequent keys are
// eligible for skipping.
if i.iterKey.Kind() == InternalKeyKindDelete ||
i.iterKey.Kind() == InternalKeyKindSingleDelete {
i.key.SetKind(InternalKeyKindSetWithDelete)
i.skip = true
return
}
default:
panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(t)))
}
}
}
func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
// Save the current key.
i.saveKey()
i.valid = true
// Loop looking for older values in the current snapshot stripe and merge
// them.
for {
if change := i.nextInStripe(); change == sameStripeNonSkippable || change == newStripe {
i.pos = iterPosNext
return change
}
key := i.iterKey
switch key.Kind() {
case InternalKeyKindDelete, InternalKeyKindSingleDelete:
// We've hit a deletion tombstone. Return everything up to this point and
// then skip entries until the next snapshot stripe. We change the kind
// of the result key to a Set so that it shadows keys in lower
// levels. That is, MERGE+DEL -> SET.
// We do the same for SingleDelete since SingleDelete is only
// permitted (with deterministic behavior) for keys that have been
// set once since the last SingleDelete/Delete, so everything
// older is acceptable to shadow. Note that this is slightly
// different from singleDeleteNext() which implements stricter
// semantics in terms of applying the SingleDelete to the single
// next Set. But those stricter semantics are not observable to
// the end-user since Iterator interprets SingleDelete as Delete.
// We could do something more complicated here and consume only a
// single Set, and then merge in any following Sets, but that is
// complicated wrt code and unnecessary given the narrow permitted
// use of SingleDelete.
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
case InternalKeyKindSet, InternalKeyKindSetWithDelete:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
// MERGE+DEL.
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
}
// We've hit a Set or SetWithDel value. Merge with the existing
// value and return. We change the kind of the resulting key to a
// Set so that it shadows keys in lower levels. That is:
// MERGE + (SET*) -> SET.
i.err = valueMerger.MergeOlder(i.iterValue)
if i.err != nil {
i.valid = false
return sameStripeSkippable
}
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
case InternalKeyKindMerge:
if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) {
// We change the kind of the result key to a Set so that it shadows
// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
// strictly necessary, but provides consistency with the behavior of
// MERGE+DEL.
i.key.SetKind(InternalKeyKindSet)
i.skip = true
return sameStripeSkippable
}
// We've hit another Merge value. Merge with the existing value and
// continue looping.
i.err = valueMerger.MergeOlder(i.iterValue)
if i.err != nil {
i.valid = false
return sameStripeSkippable
}
default:
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
return sameStripeSkippable
}
}
}
func (i *compactionIter) singleDeleteNext() bool {
// Save the current key.
i.saveKey()
i.value = i.iterValue
i.valid = true
// Loop until finds a key to be passed to the next level.
for {
if change := i.nextInStripe(); change == sameStripeNonSkippable || change == newStripe {
i.pos = iterPosNext
return true
}
key := i.iterKey
switch key.Kind() {
case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
// We've hit a Delete, Merge or SetWithDelete, transform the
// SingleDelete into a full Delete.
i.key.SetKind(InternalKeyKindDelete)
i.skip = true
return true
case InternalKeyKindSet:
i.nextInStripe()
i.valid = false
return false
case InternalKeyKindSingleDelete:
continue
default:
i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
i.valid = false
return false
}
}
}
func (i *compactionIter) saveKey() {
i.keyBuf = append(i.keyBuf[:0], i.iterKey.UserKey...)
i.key.UserKey = i.keyBuf
i.key.Trailer = i.iterKey.Trailer
}
func (i *compactionIter) cloneKey(key []byte) []byte {
i.alloc, key = i.alloc.Copy(key)
return key
}
func (i *compactionIter) Key() InternalKey {
return i.key
}
func (i *compactionIter) Value() []byte {
return i.value
}
func (i *compactionIter) Valid() bool {
return i.valid
}
func (i *compactionIter) Error() error {
return i.err
}
func (i *compactionIter) Close() error {
err := i.iter.Close()
if i.err == nil {
i.err = err
}
// Close the closer for the current value if one was open.
if i.valueCloser != nil {
i.err = firstError(i.err, i.valueCloser.Close())
i.valueCloser = nil
}
return i.err
}
// Tombstones returns a list of pending range tombstones in the fragmenter
// up to the specified key, or all pending range tombstones if key = nil.
// exclude specifies if the specified key is exclusive or inclusive.
// When exclude = true, all returned range tombstones are truncated to the
// specified key.
func (i *compactionIter) Tombstones(key []byte) []keyspan.Span {
if key == nil {
i.rangeDelFrag.Finish()
} else {
// The specified end key is exclusive; no versions of the specified
// user key (including range tombstones covering that key) should
// be flushed yet.
i.rangeDelFrag.TruncateAndFlushTo(key)
}
tombstones := i.tombstones
i.tombstones = nil
return tombstones
}
func (i *compactionIter) emitRangeDelChunk(fragmented keyspan.Span) {
// Apply the snapshot stripe rules, keeping only the latest tombstone for
// each snapshot stripe.
currentIdx := -1
keys := fragmented.Keys[:0]
for _, k := range fragmented.Keys {
idx, _ := snapshotIndex(k.SeqNum(), i.snapshots)
if currentIdx == idx {
continue
}
if idx == 0 && i.elideRangeTombstone(fragmented.Start, fragmented.End) {
// This is the last snapshot stripe and the range tombstone
// can be elided.
break
}
keys = append(keys, k)
if idx == 0 {
// This is the last snapshot stripe.
break
}
currentIdx = idx
}
if len(keys) > 0 {
i.tombstones = append(i.tombstones, keyspan.Span{
Start: fragmented.Start,
End: fragmented.End,
Keys: keys,
})
}
}
// maybeZeroSeqnum attempts to set the seqnum for the current key to 0. Doing
// so improves compression and enables an optimization during forward iteration
// to skip some key comparisons. The seqnum for an entry can be zeroed if the
// entry is on the bottom snapshot stripe and on the bottom level of the LSM.
func (i *compactionIter) maybeZeroSeqnum(snapshotIdx int) {
if !i.allowZeroSeqNum {
// TODO(peter): allowZeroSeqNum applies to the entire compaction. We could
// make the determination on a key by key basis, similar to what is done
// for elideTombstone. Need to add a benchmark for compactionIter to verify
// that isn't too expensive.
return
}
if snapshotIdx > 0 {
// This is not the last snapshot
return
}
i.key.SetSeqNum(0)
}

1671
vendor/github.com/cockroachdb/pebble/compaction_picker.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

31
vendor/github.com/cockroachdb/pebble/comparer.go generated vendored Normal file
View File

@@ -0,0 +1,31 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import "github.com/cockroachdb/pebble/internal/base"
// Compare exports the base.Compare type.
type Compare = base.Compare
// Equal exports the base.Equal type.
type Equal = base.Equal
// AbbreviatedKey exports the base.AbbreviatedKey type.
type AbbreviatedKey = base.AbbreviatedKey
// Separator exports the base.Separator type.
type Separator = base.Separator
// Successor exports the base.Successor type.
type Successor = base.Successor
// Split exports the base.Split type.
type Split = base.Split
// Comparer exports the base.Comparer type.
type Comparer = base.Comparer
// DefaultComparer exports the base.DefaultComparer variable.
var DefaultComparer = base.DefaultComparer

2012
vendor/github.com/cockroachdb/pebble/db.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

88
vendor/github.com/cockroachdb/pebble/error_iter.go generated vendored Normal file
View File

@@ -0,0 +1,88 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
type errorIter struct {
err error
}
// errorIter implements the base.InternalIterator interface.
var _ internalIteratorWithStats = (*errorIter)(nil)
func newErrorIter(err error) *errorIter {
return &errorIter{err: err}
}
func (c *errorIter) SeekGE(key []byte, trySeekUsingNext bool) (*InternalKey, []byte) {
return nil, nil
}
func (c *errorIter) SeekPrefixGE(
prefix, key []byte, trySeekUsingNext bool,
) (*base.InternalKey, []byte) {
return nil, nil
}
func (c *errorIter) SeekLT(key []byte) (*InternalKey, []byte) {
return nil, nil
}
func (c *errorIter) First() (*InternalKey, []byte) {
return nil, nil
}
func (c *errorIter) Last() (*InternalKey, []byte) {
return nil, nil
}
func (c *errorIter) Next() (*InternalKey, []byte) {
return nil, nil
}
func (c *errorIter) Prev() (*InternalKey, []byte) {
return nil, nil
}
func (c *errorIter) Error() error {
return c.err
}
func (c *errorIter) Close() error {
return c.err
}
func (c *errorIter) String() string {
return "error"
}
func (c *errorIter) SetBounds(lower, upper []byte) {}
func (c *errorIter) Stats() InternalIteratorStats { return InternalIteratorStats{} }
func (c *errorIter) ResetStats() {}
type errorKeyspanIter struct {
err error
}
// errorKeyspanIter implements the keyspan.FragmentIterator interface.
var _ keyspan.FragmentIterator = (*errorKeyspanIter)(nil)
func newErrorKeyspanIter(err error) *errorKeyspanIter {
return &errorKeyspanIter{err: err}
}
func (*errorKeyspanIter) SeekGE(key []byte) keyspan.Span { return keyspan.Span{} }
func (*errorKeyspanIter) SeekLT(key []byte) keyspan.Span { return keyspan.Span{} }
func (*errorKeyspanIter) First() keyspan.Span { return keyspan.Span{} }
func (*errorKeyspanIter) Last() keyspan.Span { return keyspan.Span{} }
func (*errorKeyspanIter) Next() keyspan.Span { return keyspan.Span{} }
func (*errorKeyspanIter) Prev() keyspan.Span { return keyspan.Span{} }
func (i *errorKeyspanIter) Error() error { return i.err }
func (i *errorKeyspanIter) Close() error { return i.err }
func (*errorKeyspanIter) String() string { return "error" }

693
vendor/github.com/cockroachdb/pebble/event.go generated vendored Normal file
View File

@@ -0,0 +1,693 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"strings"
"time"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/redact"
)
// TableInfo exports the manifest.TableInfo type.
type TableInfo = manifest.TableInfo
func tablesTotalSize(tables []TableInfo) uint64 {
var size uint64
for i := range tables {
size += tables[i].Size
}
return size
}
func formatFileNums(tables []TableInfo) string {
var buf strings.Builder
for i := range tables {
if i > 0 {
buf.WriteString(" ")
}
buf.WriteString(tables[i].FileNum.String())
}
return buf.String()
}
// LevelInfo contains info pertaining to a partificular level.
type LevelInfo struct {
Level int
Tables []TableInfo
}
func (i LevelInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables))))
}
// CompactionInfo contains the info for a compaction event.
type CompactionInfo struct {
// JobID is the ID of the compaction job.
JobID int
// Reason is the reason for the compaction.
Reason string
// Input contains the input tables for the compaction organized by level.
Input []LevelInfo
// Output contains the output tables generated by the compaction. The output
// tables are empty for the compaction begin event.
Output LevelInfo
// Duration is the time spent compacting, including reading and writing
// sstables.
Duration time.Duration
// TotalDuration is the total wall-time duration of the compaction,
// including applying the compaction to the database. TotalDuration is
// always ≥ Duration.
TotalDuration time.Duration
Done bool
Err error
}
func (i CompactionInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] compaction(%s) to L%d error: %s",
redact.Safe(i.JobID), redact.SafeString(i.Reason), redact.Safe(i.Output.Level), i.Err)
return
}
if !i.Done {
w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Print(levelInfos(i.Input))
return
}
outputSize := tablesTotalSize(i.Output.Tables)
w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Print(levelInfos(i.Input))
w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Output.Level),
redact.Safe(formatFileNums(i.Output.Tables)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
type levelInfos []LevelInfo
func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) {
for j, levelInfo := range i {
if j > 0 {
w.Printf(" + ")
}
w.Print(levelInfo)
}
}
// DiskSlowInfo contains the info for a disk slowness event when writing to a
// file.
type DiskSlowInfo struct {
// Path of file being written to.
Path string
// Duration that has elapsed since this disk operation started.
Duration time.Duration
}
func (i DiskSlowInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("disk slowness detected: write to file %s has been ongoing for %0.1fs",
i.Path, redact.Safe(i.Duration.Seconds()))
}
// FlushInfo contains the info for a flush event.
type FlushInfo struct {
// JobID is the ID of the flush job.
JobID int
// Reason is the reason for the flush.
Reason string
// Input contains the count of input memtables that were flushed.
Input int
// Output contains the ouptut table generated by the flush. The output info
// is empty for the flush begin event.
Output []TableInfo
// Duration is the time spent flushing. This duration includes writing and
// syncing all of the flushed keys to sstables.
Duration time.Duration
// TotalDuration is the total wall-time duration of the flush, including
// applying the flush to the database. TotalDuration is always ≥ Duration.
TotalDuration time.Duration
Done bool
Err error
}
func (i FlushInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] flush error: %s", redact.Safe(i.JobID), i.Err)
return
}
plural := redact.SafeString("s")
if i.Input == 1 {
plural = ""
}
if !i.Done {
w.Printf("[JOB %d] flushing %d memtable", redact.Safe(i.JobID), redact.Safe(i.Input))
w.SafeString(plural)
w.Printf(" to L0")
return
}
outputSize := tablesTotalSize(i.Output)
w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.JobID), redact.Safe(i.Input), plural,
redact.Safe(formatFileNums(i.Output)),
redact.Safe(humanize.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
// ManifestCreateInfo contains info about a manifest creation event.
type ManifestCreateInfo struct {
// JobID is the ID of the job the caused the manifest to be created.
JobID int
Path string
// The file number of the new Manifest.
FileNum FileNum
Err error
}
func (i ManifestCreateInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// ManifestDeleteInfo contains the info for a Manifest deletion event.
type ManifestDeleteInfo struct {
// JobID is the ID of the job the caused the Manifest to be deleted.
JobID int
Path string
FileNum FileNum
Err error
}
func (i ManifestDeleteInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// TableCreateInfo contains the info for a table creation event.
type TableCreateInfo struct {
JobID int
// Reason is the reason for the table creation: "compacting", "flushing", or
// "ingesting".
Reason string
Path string
FileNum FileNum
}
func (i TableCreateInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] %s: sstable created %s",
redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
}
// TableDeleteInfo contains the info for a table deletion event.
type TableDeleteInfo struct {
JobID int
Path string
FileNum FileNum
Err error
}
func (i TableDeleteInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] sstable delete error %s: %s",
redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
return
}
w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// TableIngestInfo contains the info for a table ingestion event.
type TableIngestInfo struct {
// JobID is the ID of the job the caused the table to be ingested.
JobID int
Tables []struct {
TableInfo
Level int
}
// GlobalSeqNum is the sequence number that was assigned to all entries in
// the ingested table.
GlobalSeqNum uint64
Err error
}
func (i TableIngestInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] ingest error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] ingested", redact.Safe(i.JobID))
for j := range i.Tables {
t := &i.Tables[j]
if j > 0 {
w.Printf(",")
}
w.Printf(" L%d:%s (%s)", redact.Safe(t.Level), redact.Safe(t.FileNum),
redact.Safe(humanize.Uint64(t.Size)))
}
}
// TableStatsInfo contains the info for a table stats loaded event.
type TableStatsInfo struct {
// JobID is the ID of the job that finished loading the initial tables'
// stats.
JobID int
}
func (i TableStatsInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableStatsInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] all initial table stats loaded", redact.Safe(i.JobID))
}
// TableValidatedInfo contains information on the result of a validation run
// on an sstable.
type TableValidatedInfo struct {
JobID int
Meta *fileMetadata
}
func (i TableValidatedInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableValidatedInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] validated table: %s", redact.Safe(i.JobID), i.Meta)
}
// WALCreateInfo contains info about a WAL creation event.
type WALCreateInfo struct {
// JobID is the ID of the job the caused the WAL to be created.
JobID int
Path string
// The file number of the new WAL.
FileNum FileNum
// The file number of a previous WAL which was recycled to create this
// one. Zero if recycling did not take place.
RecycledFileNum FileNum
Err error
}
func (i WALCreateInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] WAL create error: %s", redact.Safe(i.JobID), i.Err)
return
}
if i.RecycledFileNum == 0 {
w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
return
}
w.Printf("[JOB %d] WAL created %s (recycled %s)",
redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
}
// WALDeleteInfo contains the info for a WAL deletion event.
type WALDeleteInfo struct {
// JobID is the ID of the job the caused the WAL to be deleted.
JobID int
Path string
FileNum FileNum
Err error
}
func (i WALDeleteInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// WriteStallBeginInfo contains the info for a write stall begin event.
type WriteStallBeginInfo struct {
Reason string
}
func (i WriteStallBeginInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i WriteStallBeginInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("write stall beginning: %s", redact.Safe(i.Reason))
}
// EventListener contains a set of functions that will be invoked when various
// significant DB events occur. Note that the functions should not run for an
// excessive amount of time as they are invoked synchronously by the DB and may
// block continued DB work. For a similar reason it is advisable to not perform
// any synchronous calls back into the DB.
type EventListener struct {
// BackgroundError is invoked whenever an error occurs during a background
// operation such as flush or compaction.
BackgroundError func(error)
// CompactionBegin is invoked after the inputs to a compaction have been
// determined, but before the compaction has produced any output.
CompactionBegin func(CompactionInfo)
// CompactionEnd is invoked after a compaction has completed and the result
// has been installed.
CompactionEnd func(CompactionInfo)
// DiskSlow is invoked after a disk write operation on a file created
// with a disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks)
// is observed to exceed the specified disk slowness threshold duration.
DiskSlow func(DiskSlowInfo)
// FlushBegin is invoked after the inputs to a flush have been determined,
// but before the flush has produced any output.
FlushBegin func(FlushInfo)
// FlushEnd is invoked after a flush has complated and the result has been
// installed.
FlushEnd func(FlushInfo)
// FormatUpgrade is invoked after the database's FormatMajorVersion
// is upgraded.
FormatUpgrade func(FormatMajorVersion)
// ManifestCreated is invoked after a manifest has been created.
ManifestCreated func(ManifestCreateInfo)
// ManifestDeleted is invoked after a manifest has been deleted.
ManifestDeleted func(ManifestDeleteInfo)
// TableCreated is invoked when a table has been created.
TableCreated func(TableCreateInfo)
// TableDeleted is invoked after a table has been deleted.
TableDeleted func(TableDeleteInfo)
// TableIngested is invoked after an externally created table has been
// ingested via a call to DB.Ingest().
TableIngested func(TableIngestInfo)
// TableStatsLoaded is invoked at most once, when the table stats
// collector has loaded statistics for all tables that existed at Open.
TableStatsLoaded func(TableStatsInfo)
// TableValidated is invoked after validation runs on an sstable.
TableValidated func(TableValidatedInfo)
// WALCreated is invoked after a WAL has been created.
WALCreated func(WALCreateInfo)
// WALDeleted is invoked after a WAL has been deleted.
WALDeleted func(WALDeleteInfo)
// WriteStallBegin is invoked when writes are intentionally delayed.
WriteStallBegin func(WriteStallBeginInfo)
// WriteStallEnd is invoked when delayed writes are released.
WriteStallEnd func()
}
// EnsureDefaults ensures that background error events are logged to the
// specified logger if a handler for those events hasn't been otherwise
// specified. Ensure all handlers are non-nil so that we don't have to check
// for nil-ness before invoking.
func (l *EventListener) EnsureDefaults(logger Logger) {
if l.BackgroundError == nil {
if logger != nil {
l.BackgroundError = func(err error) {
logger.Infof("background error: %s", err)
}
} else {
l.BackgroundError = func(error) {}
}
}
if l.CompactionBegin == nil {
l.CompactionBegin = func(info CompactionInfo) {}
}
if l.CompactionEnd == nil {
l.CompactionEnd = func(info CompactionInfo) {}
}
if l.DiskSlow == nil {
l.DiskSlow = func(info DiskSlowInfo) {}
}
if l.FlushBegin == nil {
l.FlushBegin = func(info FlushInfo) {}
}
if l.FlushEnd == nil {
l.FlushEnd = func(info FlushInfo) {}
}
if l.FormatUpgrade == nil {
l.FormatUpgrade = func(v FormatMajorVersion) {}
}
if l.ManifestCreated == nil {
l.ManifestCreated = func(info ManifestCreateInfo) {}
}
if l.ManifestDeleted == nil {
l.ManifestDeleted = func(info ManifestDeleteInfo) {}
}
if l.TableCreated == nil {
l.TableCreated = func(info TableCreateInfo) {}
}
if l.TableDeleted == nil {
l.TableDeleted = func(info TableDeleteInfo) {}
}
if l.TableIngested == nil {
l.TableIngested = func(info TableIngestInfo) {}
}
if l.TableStatsLoaded == nil {
l.TableStatsLoaded = func(info TableStatsInfo) {}
}
if l.TableValidated == nil {
l.TableValidated = func(validated TableValidatedInfo) {}
}
if l.WALCreated == nil {
l.WALCreated = func(info WALCreateInfo) {}
}
if l.WALDeleted == nil {
l.WALDeleted = func(info WALDeleteInfo) {}
}
if l.WriteStallBegin == nil {
l.WriteStallBegin = func(info WriteStallBeginInfo) {}
}
if l.WriteStallEnd == nil {
l.WriteStallEnd = func() {}
}
}
// MakeLoggingEventListener creates an EventListener that logs all events to the
// specified logger.
func MakeLoggingEventListener(logger Logger) EventListener {
if logger == nil {
logger = DefaultLogger
}
return EventListener{
BackgroundError: func(err error) {
logger.Infof("background error: %s", err)
},
CompactionBegin: func(info CompactionInfo) {
logger.Infof("%s", info)
},
CompactionEnd: func(info CompactionInfo) {
logger.Infof("%s", info)
},
DiskSlow: func(info DiskSlowInfo) {
logger.Infof("%s", info)
},
FlushBegin: func(info FlushInfo) {
logger.Infof("%s", info)
},
FlushEnd: func(info FlushInfo) {
logger.Infof("%s", info)
},
FormatUpgrade: func(v FormatMajorVersion) {
logger.Infof("upgraded to format version: %s", v)
},
ManifestCreated: func(info ManifestCreateInfo) {
logger.Infof("%s", info)
},
ManifestDeleted: func(info ManifestDeleteInfo) {
logger.Infof("%s", info)
},
TableCreated: func(info TableCreateInfo) {
logger.Infof("%s", info)
},
TableDeleted: func(info TableDeleteInfo) {
logger.Infof("%s", info)
},
TableIngested: func(info TableIngestInfo) {
logger.Infof("%s", info)
},
TableStatsLoaded: func(info TableStatsInfo) {
logger.Infof("%s", info)
},
TableValidated: func(info TableValidatedInfo) {
logger.Infof("%s", info)
},
WALCreated: func(info WALCreateInfo) {
logger.Infof("%s", info)
},
WALDeleted: func(info WALDeleteInfo) {
logger.Infof("%s", info)
},
WriteStallBegin: func(info WriteStallBeginInfo) {
logger.Infof("%s", info)
},
WriteStallEnd: func() {
logger.Infof("write stall ending")
},
}
}
// TeeEventListener wraps two EventListeners, forwarding all events to both.
func TeeEventListener(a, b EventListener) EventListener {
a.EnsureDefaults(nil)
b.EnsureDefaults(nil)
return EventListener{
BackgroundError: func(err error) {
a.BackgroundError(err)
b.BackgroundError(err)
},
CompactionBegin: func(info CompactionInfo) {
a.CompactionBegin(info)
b.CompactionBegin(info)
},
CompactionEnd: func(info CompactionInfo) {
a.CompactionEnd(info)
b.CompactionEnd(info)
},
DiskSlow: func(info DiskSlowInfo) {
a.DiskSlow(info)
b.DiskSlow(info)
},
FlushBegin: func(info FlushInfo) {
a.FlushBegin(info)
b.FlushBegin(info)
},
FlushEnd: func(info FlushInfo) {
a.FlushEnd(info)
b.FlushEnd(info)
},
FormatUpgrade: func(v FormatMajorVersion) {
a.FormatUpgrade(v)
b.FormatUpgrade(v)
},
ManifestCreated: func(info ManifestCreateInfo) {
a.ManifestCreated(info)
b.ManifestCreated(info)
},
ManifestDeleted: func(info ManifestDeleteInfo) {
a.ManifestDeleted(info)
b.ManifestDeleted(info)
},
TableCreated: func(info TableCreateInfo) {
a.TableCreated(info)
b.TableCreated(info)
},
TableDeleted: func(info TableDeleteInfo) {
a.TableDeleted(info)
b.TableDeleted(info)
},
TableIngested: func(info TableIngestInfo) {
a.TableIngested(info)
b.TableIngested(info)
},
TableStatsLoaded: func(info TableStatsInfo) {
a.TableStatsLoaded(info)
b.TableStatsLoaded(info)
},
TableValidated: func(info TableValidatedInfo) {
a.TableValidated(info)
b.TableValidated(info)
},
WALCreated: func(info WALCreateInfo) {
a.WALCreated(info)
b.WALCreated(info)
},
WALDeleted: func(info WALDeleteInfo) {
a.WALDeleted(info)
b.WALDeleted(info)
},
WriteStallBegin: func(info WriteStallBeginInfo) {
a.WriteStallBegin(info)
b.WriteStallBegin(info)
},
WriteStallEnd: func() {
a.WriteStallEnd()
b.WriteStallEnd()
},
}
}

176
vendor/github.com/cockroachdb/pebble/external_iterator.go generated vendored Normal file
View File

@@ -0,0 +1,176 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/internal/rangekey"
"github.com/cockroachdb/pebble/sstable"
)
// NewExternalIter takes an input set of sstable files which may overlap
// arbitrarily and returns an Iterator over the merged contents of the sstables.
// Input sstables may contain point keys, range keys, range deletions, etc. The
// input files slice must be sorted in reverse chronological ordering. A key in
// a file at a lower index will shadow a key with an identical user key
// contained within a file at a higher index.
//
// Input sstables must only contain keys with the zero sequence number.
func NewExternalIter(
o *Options,
iterOpts *IterOptions,
files []sstable.ReadableFile,
extraReaderOpts ...sstable.ReaderOption,
) (it *Iterator, err error) {
var readers []*sstable.Reader
// Ensure we close all the opened readers if we error out.
closeReaders := func() {
for i := range readers {
_ = readers[i].Close()
}
}
defer func() {
if err != nil {
closeReaders()
}
}()
readers, err = openExternalTables(o, files, o.MakeReaderOptions(), extraReaderOpts...)
if err != nil {
return nil, err
}
buf := iterAllocPool.Get().(*iterAlloc)
dbi := &buf.dbi
*dbi = Iterator{
alloc: buf,
cmp: o.Comparer.Compare,
equal: o.equal(),
merge: o.Merger.Merge,
split: o.Comparer.Split,
readState: nil,
keyBuf: buf.keyBuf,
prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
boundsBuf: buf.boundsBuf,
batch: nil,
newIters: func(f *manifest.FileMetadata, opts *IterOptions, bytesIterated *uint64) (internalIterator, keyspan.FragmentIterator, error) {
// NB: External iterators are currently constructed without any
// `levelIters`. newIters should never be called. When we support
// organizing multiple non-overlapping files into a single level
// (see TODO below), we'll need to adjust this tableNewIters
// implementation to open iterators by looking up f in a map
// of readers indexed by *fileMetadata.
panic("unreachable")
},
seqNum: base.InternalKeySeqNumMax,
}
if iterOpts != nil {
dbi.opts = *iterOpts
dbi.saveBounds(iterOpts.LowerBound, iterOpts.UpperBound)
}
// TODO(jackson): In some instances we could generate fewer levels by using
// L0Sublevels code to organize nonoverlapping files into the same level.
// This would allow us to use levelIters and keep a smaller set of data and
// files in-memory. However, it would also require us to identify the bounds
// of all the files upfront.
// Ensure we close all iters if error out early.
mlevels := buf.mlevels[:0]
var rangeKeyIters []keyspan.FragmentIterator
defer func() {
if err != nil {
for i := range rangeKeyIters {
_ = rangeKeyIters[i].Close()
}
for i := range mlevels {
if mlevels[i].iter != nil {
_ = mlevels[i].iter.Close()
}
if mlevels[i].rangeDelIter != nil {
_ = mlevels[i].rangeDelIter.Close()
}
}
}
}()
if iterOpts.pointKeys() {
if len(files) > cap(mlevels) {
mlevels = make([]mergingIterLevel, 0, len(files))
}
for _, r := range readers {
pointIter, err := r.NewIter(dbi.opts.LowerBound, dbi.opts.UpperBound)
if err != nil {
return nil, err
}
rangeDelIter, err := r.NewRawRangeDelIter()
if err != nil {
_ = pointIter.Close()
return nil, err
}
mlevels = append(mlevels, mergingIterLevel{
iter: base.WrapIterWithStats(pointIter),
rangeDelIter: rangeDelIter,
})
}
}
buf.merging.init(&dbi.opts, dbi.cmp, dbi.split, mlevels...)
buf.merging.snapshot = base.InternalKeySeqNumMax
buf.merging.elideRangeTombstones = true
dbi.pointIter = &buf.merging
dbi.iter = dbi.pointIter
if dbi.opts.rangeKeys() {
for _, r := range readers {
rki, err := r.NewRawRangeKeyIter()
if err != nil {
return nil, err
}
if rki != nil {
rangeKeyIters = append(rangeKeyIters, rki)
}
}
dbi.rangeKey = iterRangeKeyStateAllocPool.Get().(*iteratorRangeKeyState)
dbi.rangeKey.rangeKeyIter = rangekey.InitUserIteration(
o.Comparer.Compare,
base.InternalKeySeqNumMax,
&dbi.rangeKey.alloc.merging,
&dbi.rangeKey.alloc.defraging,
rangeKeyIters...,
)
dbi.rangeKey.iter.Init(dbi.cmp, &buf.merging, dbi.rangeKey.rangeKeyIter, keyspan.Hooks{
SpanChanged: dbi.rangeKeySpanChanged,
SkipPoint: dbi.rangeKeySkipPoint,
}, dbi.opts.LowerBound, dbi.opts.UpperBound)
dbi.iter = &dbi.rangeKey.iter
}
// Close all the opened sstable.Readers when the Iterator is closed.
dbi.closeHook = closeReaders
return dbi, nil
}
func openExternalTables(
o *Options,
files []sstable.ReadableFile,
readerOpts sstable.ReaderOptions,
extraReaderOpts ...sstable.ReaderOption,
) (readers []*sstable.Reader, err error) {
readers = make([]*sstable.Reader, 0, len(files))
for i := range files {
r, err := sstable.NewReader(files[i], readerOpts, extraReaderOpts...)
if err != nil {
return readers, err
}
// Use the index of the file in files as the sequence number for all of
// its keys.
r.Properties.GlobalSeqNum = uint64(len(files) - i)
readers = append(readers, r)
}
return readers, err
}

54
vendor/github.com/cockroachdb/pebble/filenames.go generated vendored Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/vfs"
)
type fileType = base.FileType
// FileNum is an identifier for a file within a database.
type FileNum = base.FileNum
const (
fileTypeLog = base.FileTypeLog
fileTypeLock = base.FileTypeLock
fileTypeTable = base.FileTypeTable
fileTypeManifest = base.FileTypeManifest
fileTypeCurrent = base.FileTypeCurrent
fileTypeOptions = base.FileTypeOptions
fileTypeTemp = base.FileTypeTemp
fileTypeOldTemp = base.FileTypeOldTemp
)
// setCurrentFile sets the CURRENT file to point to the manifest with
// provided file number.
//
// NB: This is a low-level routine and typically not what you want to
// use. Newer versions of Pebble running newer format major versions do
// not use the CURRENT file. See setCurrentFunc in version_set.go.
func setCurrentFile(dirname string, fs vfs.FS, fileNum FileNum) error {
newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum)
oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum)
fs.Remove(oldFilename)
f, err := fs.Create(oldFilename)
if err != nil {
return err
}
if _, err := fmt.Fprintf(f, "MANIFEST-%s\n", fileNum); err != nil {
return err
}
if err := f.Sync(); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
return fs.Rename(oldFilename, newFilename)
}

111
vendor/github.com/cockroachdb/pebble/flush_external.go generated vendored Normal file
View File

@@ -0,0 +1,111 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"time"
"github.com/cockroachdb/pebble/internal/private"
)
// flushExternalTable is installed as the private.FlushExternalTable hook.
// It's used by the internal/replay package to flush external tables into L0
// for supporting the pebble bench compact commands. Clients should use the
// replay package rather than calling this private hook directly.
func flushExternalTable(untypedDB interface{}, path string, originalMeta *fileMetadata) error {
d := untypedDB.(*DB)
if err := d.closed.Load(); err != nil {
panic(err)
}
if d.opts.ReadOnly {
return ErrReadOnly
}
d.mu.Lock()
fileNum := d.mu.versions.getNextFileNum()
jobID := d.mu.nextJobID
d.mu.nextJobID++
d.mu.Unlock()
m := &fileMetadata{
FileNum: fileNum,
Size: originalMeta.Size,
CreationTime: time.Now().Unix(),
SmallestSeqNum: originalMeta.SmallestSeqNum,
LargestSeqNum: originalMeta.LargestSeqNum,
}
if originalMeta.HasPointKeys {
m.ExtendPointKeyBounds(d.cmp, originalMeta.SmallestPointKey, originalMeta.LargestPointKey)
}
if originalMeta.HasRangeKeys {
m.ExtendRangeKeyBounds(d.cmp, originalMeta.SmallestRangeKey, originalMeta.LargestRangeKey)
}
// Hard link the sstable into the DB directory.
if err := ingestLink(jobID, d.opts, d.dirname, []string{path}, []*fileMetadata{m}); err != nil {
return err
}
if err := d.dataDir.Sync(); err != nil {
return err
}
// Assign sequence numbers from its current value through to the
// external file's largest sequence number. It's possible that the current
// sequence number has already exceeded m.LargestSeqNum if there were many
// nonoverlapping ingestions, so skip if this doesn't bump the sequence
// number.
d.commit.ratchetSeqNum(m.LargestSeqNum + 1)
d.commit.mu.Lock()
defer d.commit.mu.Unlock()
// Apply the version edit.
d.mu.Lock()
d.mu.versions.logLock()
ve := &versionEdit{
NewFiles: []newFileEntry{{Level: 0, Meta: m}},
}
metrics := map[int]*LevelMetrics{
0: {
NumFiles: 1,
Size: int64(m.Size),
BytesIngested: m.Size,
TablesIngested: 1,
},
}
err := d.mu.versions.logAndApply(jobID, ve, metrics, false /* forceRotation */, func() []compactionInfo {
return d.getInProgressCompactionInfoLocked(nil)
})
if err != nil {
// NB: logAndApply will release d.mu.versions.logLock unconditionally.
d.mu.Unlock()
if err2 := ingestCleanup(d.opts.FS, d.dirname, []*fileMetadata{m}); err2 != nil {
d.opts.Logger.Infof("flush external cleanup failed: %v", err2)
}
return err
}
d.updateReadStateLocked(d.opts.DebugCheck, nil)
d.updateTableStatsLocked(ve.NewFiles)
d.deleteObsoleteFiles(jobID, true /* waitForOngoing */)
d.maybeScheduleCompaction()
d.mu.Unlock()
return nil
}
// ratchetSeqNum is a hook for allocating and publishing sequence numbers up
// to a specific absolute value. Its first parameter is a *pebble.DB and its
// second is the new next sequence number. RatchetSeqNum does nothing if the
// next sequence is already greater than or equal to nextSeqNum.
//
// This function is used by the internal/replay package to ensure replayed
// operations receive the same absolute sequence number.
func ratchetSeqNum(untypedDB interface{}, nextSeqNum uint64) {
d := untypedDB.(*DB)
d.commit.ratchetSeqNum(nextSeqNum)
}
func init() {
private.FlushExternalTable = flushExternalTable
private.RatchetSeqNum = ratchetSeqNum
}

81
vendor/github.com/cockroachdb/pebble/flushable.go generated vendored Normal file
View File

@@ -0,0 +1,81 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"sync/atomic"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// flushable defines the interface for immutable memtables.
type flushable interface {
newIter(o *IterOptions) internalIterator
newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator
newRangeDelIter(o *IterOptions) keyspan.FragmentIterator
newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator
// inuseBytes returns the number of inuse bytes by the flushable.
inuseBytes() uint64
// totalBytes returns the total number of bytes allocated by the flushable.
totalBytes() uint64
// readyForFlush returns true when the flushable is ready for flushing. See
// memTable.readyForFlush for one implementation which needs to check whether
// there are any outstanding write references.
readyForFlush() bool
}
// flushableEntry wraps a flushable and adds additional metadata and
// functionality that is common to all flushables.
type flushableEntry struct {
flushable
// Channel which is closed when the flushable has been flushed.
flushed chan struct{}
// flushForced indicates whether a flush was forced on this memtable (either
// manual, or due to ingestion). Protected by DB.mu.
flushForced bool
// delayedFlushForced indicates whether a timer has been set to force a flush
// on this memtable at some point in the future. Protected by DB.mu
delayedFlushForced bool
// logNum corresponds to the WAL that contains the records present in the
// receiver.
logNum FileNum
// logSize is the size in bytes of the associated WAL. Protected by DB.mu.
logSize uint64
// The current logSeqNum at the time the memtable was created. This is
// guaranteed to be less than or equal to any seqnum stored in the memtable.
logSeqNum uint64
// readerRefs tracks the read references on the flushable. The two sources of
// reader references are DB.mu.mem.queue and readState.memtables. The memory
// reserved by the flushable in the cache is released when the reader refs
// drop to zero. If the flushable is a memTable, when the reader refs drops
// to zero, the writer refs will already be zero because the memtable will
// have been flushed and that only occurs once the writer refs drops to zero.
readerRefs int32
// Closure to invoke to release memory accounting.
releaseMemAccounting func()
}
func (e *flushableEntry) readerRef() {
switch v := atomic.AddInt32(&e.readerRefs, 1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
}
func (e *flushableEntry) readerUnref() {
switch v := atomic.AddInt32(&e.readerRefs, -1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
if e.releaseMemAccounting == nil {
panic("pebble: memtable reservation already released")
}
e.releaseMemAccounting()
e.releaseMemAccounting = nil
}
}
type flushableList []*flushableEntry

461
vendor/github.com/cockroachdb/pebble/format_major_version.go generated vendored Normal file
View File

@@ -0,0 +1,461 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"strconv"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
// FormatMajorVersion is a constant controlling the format of persisted
// data. Backwards incompatible changes to durable formats are gated
// behind new format major versions.
//
// At any point, a database's format major version may be bumped.
// However, once a database's format major version is increased,
// previous versions of Pebble will refuse to open the database.
//
// The zero value format is the FormatDefault constant. The exact
// FormatVersion that the default corresponds to may change with time.
type FormatMajorVersion uint64
// String implements fmt.Stringer.
func (v FormatMajorVersion) String() string {
// NB: This must not change. It's used as the value for the the
// on-disk version marker file.
//
// Specifically, this value must always parse as a base 10 integer
// that fits in a uint64. We format it as zero-padded, 3-digit
// number today, but the padding may change.
return fmt.Sprintf("%03d", v)
}
const (
// FormatDefault leaves the format version unspecified. The
// FormatDefault constant may be ratcheted upwards over time.
FormatDefault FormatMajorVersion = iota
// FormatMostCompatible maintains the most backwards compatibility,
// maintaining bi-directional compatibility with RocksDB 6.2.1 in
// the particular configuration described in the Pebble README.
FormatMostCompatible
// formatVersionedManifestMarker is the first
// backwards-incompatible change made to Pebble, introducing the
// format-version marker file for handling backwards-incompatible
// changes more broadly, and replacing the `CURRENT` file with a
// marker file.
//
// This format version is intended as an intermediary version state.
// It is deliberately unexported to discourage direct use of this
// format major version. Clients should use FormatVersioned which
// also ensures earlier versions of Pebble fail to open a database
// written in a future format major version.
formatVersionedManifestMarker
// FormatVersioned is a new format major version that replaces the
// old `CURRENT` file with a new 'marker' file scheme. Previous
// Pebble versions will be unable to open the database unless
// they're aware of format versions.
FormatVersioned
// FormatSetWithDelete is a format major version that introduces a new key
// kind, base.InternalKeyKindSetWithDelete. Previous Pebble versions will be
// unable to open this database.
FormatSetWithDelete
// FormatBlockPropertyCollector is a format major version that introduces
// BlockPropertyCollectors.
FormatBlockPropertyCollector
// FormatSplitUserKeysMarked is a format major version that guarantees that
// all files that share user keys with neighbors are marked for compaction
// in the manifest. Ratcheting to FormatSplitUserKeysMarked will block
// (without holding mutexes) until the scan of the LSM is complete and the
// manifest has been rotated.
FormatSplitUserKeysMarked
// FormatMarkedCompacted is a format major version that guarantees that all
// files explicitly marked for compaction in the manifest have been
// compacted. Combined with the FormatSplitUserKeysMarked format major
// version, this version guarantees that there are no user keys split across
// multiple files within a level L1+. Ratcheting to this format version will
// block (without holding mutexes) until all necessary compactions for files
// marked for compaction are complete.
FormatMarkedCompacted
// FormatRangeKeys is a format major version that introduces range keys.
FormatRangeKeys
// FormatNewest always contains the most recent format major version.
// NB: When adding new versions, the MaxTableFormat method should also be
// updated to return the maximum allowable version for the new
// FormatMajorVersion.
FormatNewest FormatMajorVersion = FormatRangeKeys
)
// MaxTableFormat returns the maximum sstable.TableFormat that can be used at
// this FormatMajorVersion.
func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
switch v {
case FormatDefault, FormatMostCompatible, formatVersionedManifestMarker,
FormatVersioned, FormatSetWithDelete:
return sstable.TableFormatRocksDBv2
case FormatBlockPropertyCollector, FormatSplitUserKeysMarked, FormatMarkedCompacted:
return sstable.TableFormatPebblev1
case FormatRangeKeys:
return sstable.TableFormatPebblev2
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
}
}
// formatMajorVersionMigrations defines the migrations from one format
// major version to the next. Each migration is defined as a closure
// which will be invoked on the database before the new format major
// version is committed. Migrations must be idempotent. Migrations are
// invoked with d.mu locked.
//
// Each migration is responsible for invoking finalizeFormatVersUpgrade
// to set the new format major version. RatchetFormatMajorVersion will
// panic if a migration returns a nil error but fails to finalize the
// new format major version.
var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
FormatMostCompatible: func(d *DB) error { return nil },
formatVersionedManifestMarker: func(d *DB) error {
// formatVersionedManifestMarker introduces the use of a marker
// file for pointing to the current MANIFEST file.
// Lock the manifest.
d.mu.versions.logLock()
defer d.mu.versions.logUnlock()
// Construct the filename of the currently active manifest and
// move the manifest marker to that filename. The marker is
// guaranteed to exist, because we unconditionally locate it
// during Open.
manifestFileNum := d.mu.versions.manifestFileNum
filename := base.MakeFilename(fileTypeManifest, manifestFileNum)
if err := d.mu.versions.manifestMarker.Move(filename); err != nil {
return errors.Wrap(err, "moving manifest marker")
}
// Now that we have a manifest marker file in place and pointing
// to the current MANIFEST, finalize the upgrade. If we fail for
// some reason, a retry of this migration is guaranteed to again
// move the manifest marker file to the latest manifest. If
// we're unable to finalize the upgrade, a subsequent call to
// Open will ignore the manifest marker.
if err := d.finalizeFormatVersUpgrade(formatVersionedManifestMarker); err != nil {
return err
}
// We've finalized the upgrade. All subsequent Open calls will
// ignore the CURRENT file and instead read the manifest marker.
// Before we unlock the manifest, we need to update versionSet
// to use the manifest marker on future rotations.
d.mu.versions.setCurrent = setCurrentFuncMarker(
d.mu.versions.manifestMarker,
d.mu.versions.fs,
d.mu.versions.dirname)
return nil
},
// The FormatVersioned version is split into two, each with their
// own migration to ensure the post-migration cleanup happens even
// if there's a crash immediately after finalizing the version. Once
// a new format major version is finalized, its migration will never
// run again. Post-migration cleanup like the one in the migration
// below must be performed in a separate migration or every time the
// database opens.
FormatVersioned: func(d *DB) error {
// Replace the `CURRENT` file with one that points to the
// nonexistent `MANIFEST-000000` file. If an earlier Pebble
// version that does not know about format major versions
// attempts to open the database, it will error avoiding
// accidental corruption.
if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, 0); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatVersioned)
},
// As SetWithDelete is a new key kind, there is nothing to migrate. We can
// simply finalize the format version and we're done.
FormatSetWithDelete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatSetWithDelete)
},
FormatBlockPropertyCollector: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatBlockPropertyCollector)
},
FormatSplitUserKeysMarked: func(d *DB) error {
// Mark any unmarked files with split-user keys. Note all format major
// versions migrations are invoked with DB.mu locked.
if err := d.markFilesWithSplitUserKeysLocked(); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatSplitUserKeysMarked)
},
FormatMarkedCompacted: func(d *DB) error {
// Before finalizing the format major version, rewrite any sstables
// still marked for compaction. Note all format major versions
// migrations are invoked with DB.mu locked.
if err := d.compactMarkedFilesLocked(); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatMarkedCompacted)
},
FormatRangeKeys: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatRangeKeys)
},
}
const formatVersionMarkerName = `format-version`
func lookupFormatMajorVersion(
fs vfs.FS, dirname string,
) (FormatMajorVersion, *atomicfs.Marker, error) {
m, versString, err := atomicfs.LocateMarker(fs, dirname, formatVersionMarkerName)
if err != nil {
return 0, nil, err
}
if versString == "" {
return FormatMostCompatible, m, nil
}
v, err := strconv.ParseUint(versString, 10, 64)
if err != nil {
return 0, nil, errors.Wrap(err, "parsing format major version")
}
vers := FormatMajorVersion(v)
if vers == FormatDefault {
return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers)
}
if vers > FormatNewest {
return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers)
}
return vers, m, nil
}
// FormatMajorVersion returns the database's active format major
// version. The format major version may be higher than the one
// provided in Options when the database was opened if the existing
// database was written with a higher format version.
func (d *DB) FormatMajorVersion() FormatMajorVersion {
d.mu.Lock()
defer d.mu.Unlock()
return d.mu.formatVers.vers
}
// RatchetFormatMajorVersion ratchets the opened database's format major
// version to the provided version. It errors if the provided format
// major version is below the database's current version. Once a
// database's format major version is upgraded, previous Pebble versions
// that do not know of the format version will be unable to open the
// database.
func (d *DB) RatchetFormatMajorVersion(fmv FormatMajorVersion) error {
if err := d.closed.Load(); err != nil {
panic(err)
}
d.mu.Lock()
defer d.mu.Unlock()
return d.ratchetFormatMajorVersionLocked(fmv)
}
func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) error {
if d.opts.ReadOnly {
return ErrReadOnly
}
if formatVers > FormatNewest {
// Guard against accidentally forgetting to update FormatNewest.
return errors.Errorf("pebble: unknown format version %d", formatVers)
}
if d.mu.formatVers.vers > formatVers {
return errors.Newf("pebble: database already at format major version %d; cannot reduce to %d",
d.mu.formatVers.vers, formatVers)
}
if d.mu.formatVers.ratcheting {
return errors.Newf("pebble: database format major version upgrade is in-progress")
}
d.mu.formatVers.ratcheting = true
defer func() { d.mu.formatVers.ratcheting = false }()
for nextVers := d.mu.formatVers.vers + 1; nextVers <= formatVers; nextVers++ {
if err := formatMajorVersionMigrations[nextVers](d); err != nil {
return errors.Wrapf(err, "migrating to version %d", nextVers)
}
// NB: The migration is responsible for calling
// finalizeFormatVersUpgrade to finalize the upgrade. This
// structure is necessary because some migrations may need to
// update in-memory state (without ever dropping locks) after
// the upgrade is finalized. Here we assert that the upgrade
// did occur.
if d.mu.formatVers.vers != nextVers {
d.opts.Logger.Fatalf("pebble: successful migration to format version %d never finalized the upgrade", nextVers)
}
}
return nil
}
// finalizeFormatVersUpgrade is typically only be called from within a
// format major version migration.
//
// See formatMajorVersionMigrations.
func (d *DB) finalizeFormatVersUpgrade(formatVers FormatMajorVersion) error {
// We use the marker to encode the active format version in the
// marker filename. Unlike other uses of the atomic marker, there is
// no file with the filename `formatVers.String()` on the
// filesystem.
if err := d.mu.formatVers.marker.Move(formatVers.String()); err != nil {
return err
}
d.mu.formatVers.vers = formatVers
d.opts.EventListener.FormatUpgrade(formatVers)
return nil
}
// compactMarkedFilesLocked performs a migration that schedules rewrite
// compactions to compact away any sstables marked for compaction.
// compactMarkedFilesLocked is run while ratcheting the database's format major
// version to FormatMarkedCompacted.
func (d *DB) compactMarkedFilesLocked() error {
curr := d.mu.versions.currentVersion()
for curr.Stats.MarkedForCompaction > 0 {
// Attempt to schedule a compaction to rewrite a file marked for
// compaction.
d.maybeScheduleCompactionPicker(func(picker compactionPicker, env compactionEnv) *pickedCompaction {
return picker.pickRewriteCompaction(env)
})
// The above attempt might succeed and schedule a rewrite compaction. Or
// there might not be available compaction concurrency to schedule the
// compaction. Or compaction of the file might have already been in
// progress. In any scenario, wait until there's some change in the
// state of active compactions.
// Before waiting, check that the database hasn't been closed. Trying to
// schedule the compaction may have dropped d.mu while waiting for a
// manifest write to complete. In that dropped interim, the database may
// have been closed.
if err := d.closed.Load(); err != nil {
return err.(error)
}
// NB: Waiting on this condition variable drops d.mu while blocked.
d.mu.compact.cond.Wait()
// Some flush or compaction was scheduled or completed. Loop again to
// check again for files that must be compacted. The next iteration may
// find same file again, but that's okay. It'll eventually succeed in
// scheduling the compaction and eventually be woken by its completion.
curr = d.mu.versions.currentVersion()
}
return nil
}
// markFilesWithSplitUserKeysLocked scans the LSM's levels 1 through 6 for
// adjacent files that contain the same user key. Such arrangements of files
// were permitted in RocksDB and in Pebble up to SHA a860bbad.
// markFilesWithSplitUserKeysLocked durably marks such files for compaction.
func (d *DB) markFilesWithSplitUserKeysLocked() error {
jobID := d.mu.nextJobID
d.mu.nextJobID++
// Files with split user keys are expected to be rare and performing key
// comparisons for every file within the LSM is expensive, so drop the
// database lock while scanning the file metadata.
//
// After scanning, if we found files to mark, we'll re-acquire the mutex
// before marking them, since MarkedForCompaction and the version's
// Stats.MarkedForCompaction are protected by d.mu.
var files [numLevels][]*fileMetadata
var foundSplitUserKey bool
func() {
equal := d.opts.equal()
vers := d.mu.versions.currentVersion()
// Note the unusual locking: unlock, defer Lock().
d.mu.Unlock()
defer d.mu.Lock()
for l := numLevels - 1; l > 0; l-- {
iter := vers.Levels[l].Iter()
var prevFile *fileMetadata
var prevUserKey []byte
for f := iter.First(); f != nil; f = iter.Next() {
if prevUserKey != nil && equal(prevUserKey, f.Smallest.UserKey) {
// NB: We may append a file twice, once as prevFile and once
// as f. That's okay, and handled below.
files[l] = append(files[l], prevFile, f)
foundSplitUserKey = true
}
if f.Largest.IsExclusiveSentinel() {
prevUserKey = nil
prevFile = nil
} else {
prevUserKey = f.Largest.UserKey
prevFile = f
}
}
}
}()
// The database lock has been acquired again by the defer within the above
// anonymous function.
if !foundSplitUserKey {
// Nothing to do.
return nil
}
// Lock the manifest for a coherent view of the LSM.
d.mu.versions.logLock()
vers := d.mu.versions.currentVersion()
for l, filesToMark := range files {
if len(filesToMark) == 0 {
continue
}
for _, f := range filesToMark {
if f.MarkedForCompaction {
continue
}
// We need to determine if the file still exists in the current
// version of the LSM. Note that we're making the assumption that if
// f still exists within the LSM, it's within the same level. That's
// okay, because only move compactions may move files between levels
// and move compactions only apply to compactions with a single file
// in the start level, but we already know these files have
// multi-file atomic compaction units.
if m := vers.Levels[l].Find(d.cmp, f); m != nil {
// It's still there. Mark it for compaction and update the
// version's stats.
f.MarkedForCompaction = true
vers.Stats.MarkedForCompaction++
}
}
// The compaction picker uses the markedForCompactionAnnotator to
// quickly find files marked for compaction, or to quickly determine
// that there are no such files marked for compaction within a level.
// A b-tree node may be annotated with an annotation recording that
// there are no files marked for compaction within the node's subtree,
// based on the assumption that it's static.
//
// Since we're marking files for compaction, these b-tree nodes'
// annotations will be out of date. Clear the compaction-picking
// annotation, so that it's recomputed the next time the compaction
// picker looks for a file marked for compaction.
vers.Levels[l].InvalidateAnnotation(markedForCompactionAnnotator{})
}
// The 'marked-for-compaction' bit is persisted in the MANIFEST file
// metadata. We've already modified the in-memory file metadata, but the
// manifest hasn't been updated. Force rotation to a new MANIFEST file,
// which will write every file metadata to the new manifest file and ensure
// that the now marked-for-compaction file metadata are persisted as marked.
// NB: This call to logAndApply will unlockthe MANIFEST, which we locked up
// above before obtaining `vers`.
return d.mu.versions.logAndApply(
jobID,
&manifest.VersionEdit{},
map[int]*LevelMetrics{},
true, /* forceRotation */
func() []compactionInfo { return d.getInProgressCompactionInfoLocked(nil) })
}

211
vendor/github.com/cockroachdb/pebble/get_iter.go generated vendored Normal file
View File

@@ -0,0 +1,211 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
)
// getIter is an internal iterator used to perform gets. It iterates through
// the values for a particular key, level by level. It is not a general purpose
// internalIterator, but specialized for Get operations so that it loads data
// lazily.
type getIter struct {
logger Logger
cmp Compare
equal Equal
newIters tableNewIters
snapshot uint64
key []byte
iter internalIterator
rangeDelIter keyspan.FragmentIterator
tombstone keyspan.Span
levelIter levelIter
level int
batch *Batch
mem flushableList
l0 []manifest.LevelSlice
version *version
iterKey *InternalKey
iterValue []byte
err error
}
// TODO(sumeer): CockroachDB code doesn't use getIter, but, for completeness,
// make this implement InternalIteratorWithStats.
// getIter implements the base.InternalIterator interface.
var _ base.InternalIterator = (*getIter)(nil)
func (g *getIter) String() string {
return fmt.Sprintf("len(l0)=%d, len(mem)=%d, level=%d", len(g.l0), len(g.mem), g.level)
}
func (g *getIter) SeekGE(key []byte, trySeekUsingNext bool) (*InternalKey, []byte) {
panic("pebble: SeekGE unimplemented")
}
func (g *getIter) SeekPrefixGE(
prefix, key []byte, trySeekUsingNext bool,
) (*base.InternalKey, []byte) {
panic("pebble: SeekPrefixGE unimplemented")
}
func (g *getIter) SeekLT(key []byte) (*InternalKey, []byte) {
panic("pebble: SeekLT unimplemented")
}
func (g *getIter) First() (*InternalKey, []byte) {
return g.Next()
}
func (g *getIter) Last() (*InternalKey, []byte) {
panic("pebble: Last unimplemented")
}
func (g *getIter) Next() (*InternalKey, []byte) {
if g.iter != nil {
g.iterKey, g.iterValue = g.iter.Next()
}
for {
if g.iter != nil {
// We have to check rangeDelIter on each iteration because a single
// user-key can be spread across multiple tables in a level. A range
// tombstone will appear in the table corresponding to its start
// key. Every call to levelIter.Next() potentially switches to a new
// table and thus reinitializes rangeDelIter.
if g.rangeDelIter != nil {
g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key, g.snapshot)
if g.err = g.rangeDelIter.Close(); g.err != nil {
return nil, nil
}
g.rangeDelIter = nil
}
if g.iterKey != nil {
key := g.iterKey
if g.tombstone.Covers(key.SeqNum()) {
// We have a range tombstone covering this key. Rather than return a
// point or range deletion here, we return false and close our
// internal iterator which will make Valid() return false,
// effectively stopping iteration.
g.err = g.iter.Close()
g.iter = nil
return nil, nil
}
if g.equal(g.key, key.UserKey) {
if !key.Visible(g.snapshot) {
g.iterKey, g.iterValue = g.iter.Next()
continue
}
return g.iterKey, g.iterValue
}
}
// We've advanced the iterator passed the desired key. Move on to the
// next memtable / level.
g.err = g.iter.Close()
g.iter = nil
if g.err != nil {
return nil, nil
}
}
// Create an iterator from the batch.
if g.batch != nil {
g.iter = g.batch.newInternalIter(nil)
g.rangeDelIter = g.batch.newRangeDelIter(nil)
g.batch = nil
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, false /* trySeekUsingNext */)
continue
}
// If we have a tombstone from a previous level it is guaranteed to delete
// keys in lower levels.
if !g.tombstone.Empty() {
return nil, nil
}
// Create iterators from memtables from newest to oldest.
if n := len(g.mem); n > 0 {
m := g.mem[n-1]
g.iter = m.newIter(nil)
g.rangeDelIter = m.newRangeDelIter(nil)
g.mem = g.mem[:n-1]
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, false /* trySeekUsingNext */)
continue
}
if g.level == 0 {
// Create iterators from L0 from newest to oldest.
if n := len(g.l0); n > 0 {
files := g.l0[n-1].Iter()
g.l0 = g.l0[:n-1]
iterOpts := IterOptions{logger: g.logger}
g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters,
files, manifest.L0Sublevel(n), nil)
g.levelIter.initRangeDel(&g.rangeDelIter)
g.iter = &g.levelIter
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, false /* trySeekUsingNext */)
continue
}
g.level++
}
if g.level >= numLevels {
return nil, nil
}
if g.version.Levels[g.level].Empty() {
g.level++
continue
}
iterOpts := IterOptions{logger: g.logger}
g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters,
g.version.Levels[g.level].Iter(), manifest.Level(g.level), nil)
g.levelIter.initRangeDel(&g.rangeDelIter)
g.level++
g.iter = &g.levelIter
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, false /* trySeekUsingNext */)
}
}
func (g *getIter) Prev() (*InternalKey, []byte) {
panic("pebble: Prev unimplemented")
}
func (g *getIter) Key() *InternalKey {
return g.iterKey
}
func (g *getIter) Value() []byte {
return g.iterValue
}
func (g *getIter) Valid() bool {
return g.iterKey != nil && g.err == nil
}
func (g *getIter) Error() error {
return g.err
}
func (g *getIter) Close() error {
if g.iter != nil {
if err := g.iter.Close(); err != nil && g.err == nil {
g.err = err
}
g.iter = nil
}
return g.err
}
func (g *getIter) SetBounds(lower, upper []byte) {
panic("pebble: SetBounds unimplemented")
}

952
vendor/github.com/cockroachdb/pebble/ingest.go generated vendored Normal file
View File

@@ -0,0 +1,952 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"sort"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/internal/private"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
)
func sstableKeyCompare(userCmp Compare, a, b InternalKey) int {
c := userCmp(a.UserKey, b.UserKey)
if c != 0 {
return c
}
if a.Trailer == InternalKeyRangeDeleteSentinel {
if b.Trailer != InternalKeyRangeDeleteSentinel {
return -1
}
} else if b.Trailer == InternalKeyRangeDeleteSentinel {
return 1
}
return 0
}
func ingestValidateKey(opts *Options, key *InternalKey) error {
if key.Kind() == InternalKeyKindInvalid {
return base.CorruptionErrorf("pebble: external sstable has corrupted key: %s",
key.Pretty(opts.Comparer.FormatKey))
}
if key.SeqNum() != 0 {
return base.CorruptionErrorf("pebble: external sstable has non-zero seqnum: %s",
key.Pretty(opts.Comparer.FormatKey))
}
return nil
}
func ingestLoad1(
opts *Options, fmv FormatMajorVersion, path string, cacheID uint64, fileNum FileNum,
) (*fileMetadata, error) {
stat, err := opts.FS.Stat(path)
if err != nil {
return nil, err
}
f, err := opts.FS.Open(path)
if err != nil {
return nil, err
}
cacheOpts := private.SSTableCacheOpts(cacheID, fileNum).(sstable.ReaderOption)
r, err := sstable.NewReader(f, opts.MakeReaderOptions(), cacheOpts)
if err != nil {
return nil, err
}
defer r.Close()
// Avoid ingesting tables with format versions this DB doesn't support.
tf, err := r.TableFormat()
if err != nil {
return nil, err
}
if tf > fmv.MaxTableFormat() {
return nil, errors.Newf(
"pebble: table with format %s unsupported at DB format major version %d, %s",
tf, fmv, fmv.MaxTableFormat(),
)
}
meta := &fileMetadata{}
meta.FileNum = fileNum
meta.Size = uint64(stat.Size())
meta.CreationTime = time.Now().Unix()
// Avoid loading into the table cache for collecting stats if we
// don't need to. If there are no range deletions, we have all the
// information to compute the stats here.
//
// This is helpful in tests for avoiding awkwardness around deletion of
// ingested files from MemFS. MemFS implements the Windows semantics of
// disallowing removal of an open file. Under MemFS, if we don't populate
// meta.Stats here, the file will be loaded into the table cache for
// calculating stats before we can remove the original link.
maybeSetStatsFromProperties(meta, &r.Properties)
{
iter, err := r.NewIter(nil /* lower */, nil /* upper */)
if err != nil {
return nil, err
}
defer iter.Close()
var smallest InternalKey
if key, _ := iter.First(); key != nil {
if err := ingestValidateKey(opts, key); err != nil {
return nil, err
}
smallest = (*key).Clone()
}
if err := iter.Error(); err != nil {
return nil, err
}
if key, _ := iter.Last(); key != nil {
if err := ingestValidateKey(opts, key); err != nil {
return nil, err
}
meta.ExtendPointKeyBounds(opts.Comparer.Compare, smallest, key.Clone())
}
if err := iter.Error(); err != nil {
return nil, err
}
}
iter, err := r.NewRawRangeDelIter()
if err != nil {
return nil, err
}
if iter != nil {
defer iter.Close()
var smallest InternalKey
if s := iter.First(); s.Valid() {
key := s.SmallestKey()
if err := ingestValidateKey(opts, &key); err != nil {
return nil, err
}
smallest = key.Clone()
}
if err := iter.Error(); err != nil {
return nil, err
}
if s := iter.Last(); s.Valid() {
k := s.SmallestKey()
if err := ingestValidateKey(opts, &k); err != nil {
return nil, err
}
largest := s.LargestKey().Clone()
meta.ExtendPointKeyBounds(opts.Comparer.Compare, smallest, largest)
}
}
// Update the range-key bounds for the table.
{
iter, err := r.NewRawRangeKeyIter()
if err != nil {
return nil, err
}
if iter != nil {
defer iter.Close()
var smallest InternalKey
if s := iter.First(); s.Valid() {
key := s.SmallestKey()
if err := ingestValidateKey(opts, &key); err != nil {
return nil, err
}
smallest = key.Clone()
}
if err := iter.Error(); err != nil {
return nil, err
}
if s := iter.Last(); s.Valid() {
k := s.SmallestKey()
if err := ingestValidateKey(opts, &k); err != nil {
return nil, err
}
// As range keys are fragmented, the end key of the last range key in
// the table provides the upper bound for the table.
largest := s.LargestKey().Clone()
meta.ExtendRangeKeyBounds(opts.Comparer.Compare, smallest, largest)
}
if err := iter.Error(); err != nil {
return nil, err
}
}
}
if !meta.HasPointKeys && !meta.HasRangeKeys {
return nil, nil
}
// Sanity check that the various bounds on the file were set consistently.
if err := meta.Validate(opts.Comparer.Compare, opts.Comparer.FormatKey); err != nil {
return nil, err
}
return meta, nil
}
func ingestLoad(
opts *Options, fmv FormatMajorVersion, paths []string, cacheID uint64, pending []FileNum,
) ([]*fileMetadata, []string, error) {
meta := make([]*fileMetadata, 0, len(paths))
newPaths := make([]string, 0, len(paths))
for i := range paths {
m, err := ingestLoad1(opts, fmv, paths[i], cacheID, pending[i])
if err != nil {
return nil, nil, err
}
if m != nil {
meta = append(meta, m)
newPaths = append(newPaths, paths[i])
}
}
return meta, newPaths, nil
}
// Struct for sorting metadatas by smallest user keys, while ensuring the
// matching path also gets swapped to the same index. For use in
// ingestSortAndVerify.
type metaAndPaths struct {
meta []*fileMetadata
paths []string
cmp Compare
}
func (m metaAndPaths) Len() int {
return len(m.meta)
}
func (m metaAndPaths) Less(i, j int) bool {
return m.cmp(m.meta[i].Smallest.UserKey, m.meta[j].Smallest.UserKey) < 0
}
func (m metaAndPaths) Swap(i, j int) {
m.meta[i], m.meta[j] = m.meta[j], m.meta[i]
m.paths[i], m.paths[j] = m.paths[j], m.paths[i]
}
func ingestSortAndVerify(cmp Compare, meta []*fileMetadata, paths []string) error {
if len(meta) <= 1 {
return nil
}
sort.Sort(&metaAndPaths{
meta: meta,
paths: paths,
cmp: cmp,
})
for i := 1; i < len(meta); i++ {
if sstableKeyCompare(cmp, meta[i-1].Largest, meta[i].Smallest) >= 0 {
return errors.New("pebble: external sstables have overlapping ranges")
}
}
return nil
}
func ingestCleanup(fs vfs.FS, dirname string, meta []*fileMetadata) error {
var firstErr error
for i := range meta {
target := base.MakeFilepath(fs, dirname, fileTypeTable, meta[i].FileNum)
if err := fs.Remove(target); err != nil {
firstErr = firstError(firstErr, err)
}
}
return firstErr
}
func ingestLink(
jobID int, opts *Options, dirname string, paths []string, meta []*fileMetadata,
) error {
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := syncingFS{
FS: opts.FS,
syncOpts: vfs.SyncingFileOptions{
BytesPerSync: opts.BytesPerSync,
},
}
for i := range paths {
target := base.MakeFilepath(fs, dirname, fileTypeTable, meta[i].FileNum)
var err error
if _, ok := opts.FS.(*vfs.MemFS); ok && opts.DebugCheck != nil {
// The combination of MemFS+Ingest+DebugCheck produces awkwardness around
// the subsequent deletion of files. The problem is that MemFS implements
// the Windows semantics of disallowing removal of an open file. This is
// desirable because it helps catch bugs where we violate the
// requirements of the Windows semantics. The normal practice for Ingest
// is for the caller to remove the source files after the ingest
// completes successfully. Unfortunately, Options.DebugCheck causes
// ingest to run DB.CheckLevels() before the ingest finishes, and
// DB.CheckLevels() populates the table cache with the newly ingested
// files.
//
// The combination of MemFS+Ingest+DebugCheck is primarily used in
// tests. As a workaround, disable hard linking this combination
// occurs. See https://github.com/cockroachdb/pebble/issues/495.
err = vfs.Copy(fs, paths[i], target)
} else {
err = vfs.LinkOrCopy(fs, paths[i], target)
}
if err != nil {
if err2 := ingestCleanup(fs, dirname, meta[:i]); err2 != nil {
opts.Logger.Infof("ingest cleanup failed: %v", err2)
}
return err
}
if opts.EventListener.TableCreated != nil {
opts.EventListener.TableCreated(TableCreateInfo{
JobID: jobID,
Reason: "ingesting",
Path: target,
FileNum: meta[i].FileNum,
})
}
}
return nil
}
func ingestMemtableOverlaps(cmp Compare, mem flushable, meta []*fileMetadata) bool {
iter := mem.newIter(nil)
rangeDelIter := mem.newRangeDelIter(nil)
defer iter.Close()
if rangeDelIter != nil {
defer rangeDelIter.Close()
}
for _, m := range meta {
if overlapWithIterator(iter, &rangeDelIter, m, cmp) {
return true
}
}
return false
}
func ingestUpdateSeqNum(
cmp Compare, format base.FormatKey, seqNum uint64, meta []*fileMetadata,
) error {
setSeqFn := func(k base.InternalKey) base.InternalKey {
return base.MakeInternalKey(k.UserKey, seqNum, k.Kind())
}
for _, m := range meta {
// NB: we set the fields directly here, rather than via their Extend*
// methods, as we are updating sequence numbers.
if m.HasPointKeys {
m.SmallestPointKey = setSeqFn(m.SmallestPointKey)
}
if m.HasRangeKeys {
m.SmallestRangeKey = setSeqFn(m.SmallestRangeKey)
}
m.Smallest = setSeqFn(m.Smallest)
// Only update the seqnum for the largest key if that key is not an
// "exclusive sentinel" (i.e. a range deletion sentinel or a range key
// boundary), as doing so effectively drops the exclusive sentinel (by
// lowering the seqnum from the max value), and extends the bounds of the
// table.
// NB: as the largest range key is always an exclusive sentinel, it is never
// updated.
if m.HasPointKeys && !m.LargestPointKey.IsExclusiveSentinel() {
m.LargestPointKey = setSeqFn(m.LargestPointKey)
}
if !m.Largest.IsExclusiveSentinel() {
m.Largest = setSeqFn(m.Largest)
}
// Setting smallestSeqNum == largestSeqNum triggers the setting of
// Properties.GlobalSeqNum when an sstable is loaded.
m.SmallestSeqNum = seqNum
m.LargestSeqNum = seqNum
// Ensure the new bounds are consistent.
if err := m.Validate(cmp, format); err != nil {
return err
}
seqNum++
}
return nil
}
func overlapWithIterator(
iter internalIterator, rangeDelIter *keyspan.FragmentIterator, meta *fileMetadata, cmp Compare,
) bool {
// Check overlap with point operations.
//
// When using levelIter, it seeks to the SST whose boundaries
// contain meta.Smallest.UserKey(S).
// It then tries to find a point in that SST that is >= S.
// If there's no such point it means the SST ends in a tombstone in which case
// levelIter.SeekGE generates a boundary range del sentinel.
// The comparison of this boundary with meta.Largest(L) below
// is subtle but maintains correctness.
// 1) boundary < L,
// since boundary is also > S (initial seek),
// whatever the boundary's start key may be, we're always overlapping.
// 2) boundary > L,
// overlap with boundary cannot be determined since we don't know boundary's start key.
// We require checking for overlap with rangeDelIter.
// 3) boundary == L and L is not sentinel,
// means boundary < L and hence is similar to 1).
// 4) boundary == L and L is sentinel,
// we'll always overlap since for any values of i,j ranges [i, k) and [j, k) always overlap.
key, _ := iter.SeekGE(meta.Smallest.UserKey, false /* trySeekUsingNext */)
if key != nil {
c := sstableKeyCompare(cmp, *key, meta.Largest)
if c <= 0 {
return true
}
}
// Check overlap with range deletions.
if rangeDelIter == nil || *rangeDelIter == nil {
return false
}
rangeDelItr := *rangeDelIter
rangeDel := rangeDelItr.SeekLT(meta.Smallest.UserKey)
if !rangeDel.Valid() {
rangeDel = rangeDelItr.Next()
}
for ; rangeDel.Valid(); rangeDel = rangeDelItr.Next() {
key := rangeDel.SmallestKey()
c := sstableKeyCompare(cmp, key, meta.Largest)
if c > 0 {
// The start of the tombstone is after the largest key in the
// ingested table.
return false
}
if cmp(rangeDel.End, meta.Smallest.UserKey) > 0 {
// The end of the tombstone is greater than the smallest in the
// table. Note that the tombstone end key is exclusive, thus ">0"
// instead of ">=0".
return true
}
}
return false
}
func ingestTargetLevel(
newIters tableNewIters,
iterOps IterOptions,
cmp Compare,
v *version,
baseLevel int,
compactions map[*compaction]struct{},
meta *fileMetadata,
) (int, error) {
// Find the lowest level which does not have any files which overlap meta. We
// search from L0 to L6 looking for whether there are any files in the level
// which overlap meta. We want the "lowest" level (where lower means
// increasing level number) in order to reduce write amplification.
//
// There are 2 kinds of overlap we need to check for: file boundary overlap
// and data overlap. Data overlap implies file boundary overlap. Note that it
// is always possible to ingest into L0.
//
// To place meta at level i where i > 0:
// - there must not be any data overlap with levels <= i, since that will
// violate the sequence number invariant.
// - no file boundary overlap with level i, since that will violate the
// invariant that files do not overlap in levels i > 0.
//
// The file boundary overlap check is simpler to conceptualize. Consider the
// following example, in which the ingested file lies completely before or
// after the file being considered.
//
// |--| |--| ingested file: [a,b] or [f,g]
// |-----| existing file: [c,e]
// _____________________
// a b c d e f g
//
// In both cases the ingested file can move to considering the next level.
//
// File boundary overlap does not necessarily imply data overlap. The check
// for data overlap is a little more nuanced. Consider the following examples:
//
// 1. No data overlap:
//
// |-| |--| ingested file: [cc-d] or [ee-ff]
// |*--*--*----*------*| existing file: [a-g], points: [a, b, c, dd, g]
// _____________________
// a b c d e f g
//
// In this case the ingested files can "fall through" this level. The checks
// continue at the next level.
//
// 2. Data overlap:
//
// |--| ingested file: [d-e]
// |*--*--*----*------*| existing file: [a-g], points: [a, b, c, dd, g]
// _____________________
// a b c d e f g
//
// In this case the file cannot be ingested into this level as the point 'dd'
// is in the way.
//
// It is worth noting that the check for data overlap is only approximate. In
// the previous example, the ingested table [d-e] could contain only the
// points 'd' and 'e', in which case the table would be eligible for
// considering lower levels. However, such a fine-grained check would need to
// be exhaustive (comparing points and ranges in both the ingested existing
// tables) and such a check is prohibitively expensive. Thus Pebble treats any
// existing point that falls within the ingested table bounds as being "data
// overlap".
targetLevel := 0
// Do we overlap with keys in L0?
iter := v.Levels[0].Iter()
for meta0 := iter.First(); meta0 != nil; meta0 = iter.Next() {
c1 := sstableKeyCompare(cmp, meta.Smallest, meta0.Largest)
c2 := sstableKeyCompare(cmp, meta.Largest, meta0.Smallest)
if c1 > 0 || c2 < 0 {
continue
}
iter, rangeDelIter, err := newIters(iter.Current(), nil, nil)
if err != nil {
return 0, err
}
overlap := overlapWithIterator(iter, &rangeDelIter, meta, cmp)
iter.Close()
if rangeDelIter != nil {
rangeDelIter.Close()
}
if overlap {
return targetLevel, nil
}
}
level := baseLevel
for ; level < numLevels; level++ {
levelIter := newLevelIter(iterOps, cmp, nil /* split */, newIters,
v.Levels[level].Iter(), manifest.Level(level), nil)
var rangeDelIter keyspan.FragmentIterator
// Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE
// sets it up for the target file.
levelIter.initRangeDel(&rangeDelIter)
overlap := overlapWithIterator(levelIter, &rangeDelIter, meta, cmp)
levelIter.Close() // Closes range del iter as well.
if overlap {
return targetLevel, nil
}
// Check boundary overlap.
boundaryOverlaps := v.Overlaps(level, cmp, meta.Smallest.UserKey,
meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel())
if !boundaryOverlaps.Empty() {
continue
}
// Check boundary overlap with any ongoing compactions.
//
// We cannot check for data overlap with the new SSTs compaction will
// produce since compaction hasn't been done yet. However, there's no need
// to check since all keys in them will either be from c.startLevel or
// c.outputLevel, both levels having their data overlap already tested
// negative (else we'd have returned earlier).
overlaps := false
for c := range compactions {
if c.outputLevel == nil || level != c.outputLevel.level {
continue
}
if cmp(meta.Smallest.UserKey, c.largest.UserKey) <= 0 &&
cmp(meta.Largest.UserKey, c.smallest.UserKey) >= 0 {
overlaps = true
break
}
}
if !overlaps {
targetLevel = level
}
}
return targetLevel, nil
}
// Ingest ingests a set of sstables into the DB. Ingestion of the files is
// atomic and semantically equivalent to creating a single batch containing all
// of the mutations in the sstables. Ingestion may require the memtable to be
// flushed. The ingested sstable files are moved into the DB and must reside on
// the same filesystem as the DB. Sstables can be created for ingestion using
// sstable.Writer. On success, Ingest removes the input paths.
//
// All sstables *must* be Sync()'d by the caller after all bytes are written
// and before its file handle is closed; failure to do so could violate
// durability or lead to corrupted on-disk state. This method cannot, in a
// platform-and-FS-agnostic way, ensure that all sstables in the input are
// properly synced to disk. Opening new file handles and Sync()-ing them
// does not always guarantee durability; see the discussion here on that:
// https://github.com/cockroachdb/pebble/pull/835#issuecomment-663075379
//
// Ingestion loads each sstable into the lowest level of the LSM which it
// doesn't overlap (see ingestTargetLevel). If an sstable overlaps a memtable,
// ingestion forces the memtable to flush, and then waits for the flush to
// occur.
//
// The steps for ingestion are:
//
// 1. Allocate file numbers for every sstable being ingested.
// 2. Load the metadata for all sstables being ingest.
// 3. Sort the sstables by smallest key, verifying non overlap.
// 4. Hard link (or copy) the sstables into the DB directory.
// 5. Allocate a sequence number to use for all of the entries in the
// sstables. This is the step where overlap with memtables is
// determined. If there is overlap, we remember the most recent memtable
// that overlaps.
// 6. Update the sequence number in the ingested sstables.
// 7. Wait for the most recent memtable that overlaps to flush (if any).
// 8. Add the ingested sstables to the version (DB.ingestApply).
// 9. Publish the ingestion sequence number.
//
// Note that if the mutable memtable overlaps with ingestion, a flush of the
// memtable is forced equivalent to DB.Flush. Additionally, subsequent
// mutations that get sequence numbers larger than the ingestion sequence
// number get queued up behind the ingestion waiting for it to complete. This
// can produce a noticeable hiccup in performance. See
// https://github.com/cockroachdb/pebble/issues/25 for an idea for how to fix
// this hiccup.
func (d *DB) Ingest(paths []string) error {
if err := d.closed.Load(); err != nil {
panic(err)
}
if d.opts.ReadOnly {
return ErrReadOnly
}
_, err := d.ingest(paths, ingestTargetLevel)
return err
}
// IngestOperationStats provides some information about where in the LSM the
// bytes were ingested.
type IngestOperationStats struct {
// Bytes is the total bytes in the ingested sstables.
Bytes uint64
// ApproxIngestedIntoL0Bytes is the approximate number of bytes ingested
// into L0.
// Currently, this value is completely accurate, but we are allowing this to
// be approximate once https://github.com/cockroachdb/pebble/issues/25 is
// implemented.
ApproxIngestedIntoL0Bytes uint64
}
// IngestWithStats does the same as Ingest, and additionally returns
// IngestOperationStats.
func (d *DB) IngestWithStats(paths []string) (IngestOperationStats, error) {
if err := d.closed.Load(); err != nil {
panic(err)
}
if d.opts.ReadOnly {
return IngestOperationStats{}, ErrReadOnly
}
return d.ingest(paths, ingestTargetLevel)
}
func (d *DB) ingest(
paths []string, targetLevelFunc ingestTargetLevelFunc,
) (IngestOperationStats, error) {
// Allocate file numbers for all of the files being ingested and mark them as
// pending in order to prevent them from being deleted. Note that this causes
// the file number ordering to be out of alignment with sequence number
// ordering. The sorting of L0 tables by sequence number avoids relying on
// that (busted) invariant.
d.mu.Lock()
pendingOutputs := make([]FileNum, len(paths))
for i := range paths {
pendingOutputs[i] = d.mu.versions.getNextFileNum()
}
jobID := d.mu.nextJobID
d.mu.nextJobID++
d.mu.Unlock()
// Load the metadata for all of the files being ingested. This step detects
// and elides empty sstables.
meta, paths, err := ingestLoad(d.opts, d.FormatMajorVersion(), paths, d.cacheID, pendingOutputs)
if err != nil {
return IngestOperationStats{}, err
}
if len(meta) == 0 {
// All of the sstables to be ingested were empty. Nothing to do.
return IngestOperationStats{}, nil
}
// Verify the sstables do not overlap.
if err := ingestSortAndVerify(d.cmp, meta, paths); err != nil {
return IngestOperationStats{}, err
}
// Hard link the sstables into the DB directory. Since the sstables aren't
// referenced by a version, they won't be used. If the hard linking fails
// (e.g. because the files reside on a different filesystem), ingestLink will
// fall back to copying, and if that fails we undo our work and return an
// error.
if err := ingestLink(jobID, d.opts, d.dirname, paths, meta); err != nil {
return IngestOperationStats{}, err
}
// Fsync the directory we added the tables to. We need to do this at some
// point before we update the MANIFEST (via logAndApply), otherwise a crash
// can have the tables referenced in the MANIFEST, but not present in the
// directory.
if err := d.dataDir.Sync(); err != nil {
return IngestOperationStats{}, err
}
var mem *flushableEntry
prepare := func() {
// Note that d.commit.mu is held by commitPipeline when calling prepare.
d.mu.Lock()
defer d.mu.Unlock()
// Check to see if any files overlap with any of the memtables. The queue
// is ordered from oldest to newest with the mutable memtable being the
// last element in the slice. We want to wait for the newest table that
// overlaps.
for i := len(d.mu.mem.queue) - 1; i >= 0; i-- {
m := d.mu.mem.queue[i]
if ingestMemtableOverlaps(d.cmp, m, meta) {
mem = m
if mem.flushable == d.mu.mem.mutable {
err = d.makeRoomForWrite(nil)
}
mem.flushForced = true
d.maybeScheduleFlush()
return
}
}
}
var ve *versionEdit
apply := func(seqNum uint64) {
if err != nil {
// An error occurred during prepare.
return
}
// Update the sequence number for all of the sstables in the
// metadata. Writing the metadata to the manifest when the
// version edit is applied is the mechanism that persists the
// sequence number. The sstables themselves are left unmodified.
if err = ingestUpdateSeqNum(
d.cmp, d.opts.Comparer.FormatKey, seqNum, meta,
); err != nil {
return
}
// If we overlapped with a memtable in prepare wait for the flush to
// finish.
if mem != nil {
<-mem.flushed
}
// Assign the sstables to the correct level in the LSM and apply the
// version edit.
ve, err = d.ingestApply(jobID, meta, targetLevelFunc)
}
d.commit.AllocateSeqNum(len(meta), prepare, apply)
if err != nil {
if err2 := ingestCleanup(d.opts.FS, d.dirname, meta); err2 != nil {
d.opts.Logger.Infof("ingest cleanup failed: %v", err2)
}
} else {
for _, path := range paths {
if err2 := d.opts.FS.Remove(path); err2 != nil {
d.opts.Logger.Infof("ingest failed to remove original file: %s", err2)
}
}
}
info := TableIngestInfo{
JobID: jobID,
GlobalSeqNum: meta[0].SmallestSeqNum,
Err: err,
}
var stats IngestOperationStats
if ve != nil {
info.Tables = make([]struct {
TableInfo
Level int
}, len(ve.NewFiles))
for i := range ve.NewFiles {
e := &ve.NewFiles[i]
info.Tables[i].Level = e.Level
info.Tables[i].TableInfo = e.Meta.TableInfo()
stats.Bytes += e.Meta.Size
if e.Level == 0 {
stats.ApproxIngestedIntoL0Bytes += e.Meta.Size
}
}
}
d.opts.EventListener.TableIngested(info)
return stats, err
}
type ingestTargetLevelFunc func(
newIters tableNewIters,
iterOps IterOptions,
cmp Compare,
v *version,
baseLevel int,
compactions map[*compaction]struct{},
meta *fileMetadata,
) (int, error)
func (d *DB) ingestApply(
jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc,
) (*versionEdit, error) {
d.mu.Lock()
defer d.mu.Unlock()
ve := &versionEdit{
NewFiles: make([]newFileEntry, len(meta)),
}
metrics := make(map[int]*LevelMetrics)
// Lock the manifest for writing before we use the current version to
// determine the target level. This prevents two concurrent ingestion jobs
// from using the same version to determine the target level, and also
// provides serialization with concurrent compaction and flush jobs.
// logAndApply unconditionally releases the manifest lock, but any earlier
// returns must unlock the manifest.
d.mu.versions.logLock()
current := d.mu.versions.currentVersion()
baseLevel := d.mu.versions.picker.getBaseLevel()
iterOps := IterOptions{logger: d.opts.Logger}
for i := range meta {
// Determine the lowest level in the LSM for which the sstable doesn't
// overlap any existing files in the level.
m := meta[i]
f := &ve.NewFiles[i]
var err error
f.Level, err = findTargetLevel(d.newIters, iterOps, d.cmp, current, baseLevel, d.mu.compact.inProgress, m)
if err != nil {
d.mu.versions.logUnlock()
return nil, err
}
f.Meta = m
levelMetrics := metrics[f.Level]
if levelMetrics == nil {
levelMetrics = &LevelMetrics{}
metrics[f.Level] = levelMetrics
}
levelMetrics.NumFiles++
levelMetrics.Size += int64(m.Size)
levelMetrics.BytesIngested += m.Size
levelMetrics.TablesIngested++
}
if err := d.mu.versions.logAndApply(jobID, ve, metrics, false /* forceRotation */, func() []compactionInfo {
return d.getInProgressCompactionInfoLocked(nil)
}); err != nil {
return nil, err
}
d.updateReadStateLocked(d.opts.DebugCheck, nil)
d.updateTableStatsLocked(ve.NewFiles)
d.deleteObsoleteFiles(jobID, false /* waitForOngoing */)
// The ingestion may have pushed a level over the threshold for compaction,
// so check to see if one is necessary and schedule it.
d.maybeScheduleCompaction()
d.maybeValidateSSTablesLocked(ve.NewFiles)
return ve, nil
}
// maybeValidateSSTablesLocked adds the slice of newFileEntrys to the pending
// queue of files to be validated, when the feature is enabled.
// DB.mu must be locked when calling.
func (d *DB) maybeValidateSSTablesLocked(newFiles []newFileEntry) {
// Only add to the validation queue when the feature is enabled.
if !d.opts.Experimental.ValidateOnIngest {
return
}
d.mu.tableValidation.pending = append(d.mu.tableValidation.pending, newFiles...)
if d.shouldValidateSSTablesLocked() {
go d.validateSSTables()
}
}
// shouldValidateSSTablesLocked returns true if SSTable validation should run.
// DB.mu must be locked when calling.
func (d *DB) shouldValidateSSTablesLocked() bool {
return !d.mu.tableValidation.validating &&
d.closed.Load() == nil &&
d.opts.Experimental.ValidateOnIngest &&
len(d.mu.tableValidation.pending) > 0
}
// validateSSTables runs a round of validation on the tables in the pending
// queue.
func (d *DB) validateSSTables() {
d.mu.Lock()
if !d.shouldValidateSSTablesLocked() {
d.mu.Unlock()
return
}
pending := d.mu.tableValidation.pending
d.mu.tableValidation.pending = nil
d.mu.tableValidation.validating = true
jobID := d.mu.nextJobID
d.mu.nextJobID++
rs := d.loadReadState()
// Drop DB.mu before performing IO.
d.mu.Unlock()
// Validate all tables in the pending queue. This could lead to a situation
// where we are starving IO from other tasks due to having to page through
// all the blocks in all the sstables in the queue.
// TODO(travers): Add some form of pacing to avoid IO starvation.
for _, f := range pending {
// The file may have been moved or deleted since it was ingested, in
// which case we skip.
if !rs.current.Contains(f.Level, d.cmp, f.Meta) {
// Assume the file was moved to a lower level. It is rare enough
// that a table is moved or deleted between the time it was ingested
// and the time the validation routine runs that the overall cost of
// this inner loop is tolerably low, when amortized over all
// ingested tables.
found := false
for i := f.Level + 1; i < numLevels; i++ {
if rs.current.Contains(i, d.cmp, f.Meta) {
found = true
break
}
}
if !found {
continue
}
}
err := d.tableCache.withReader(f.Meta, func(r *sstable.Reader) error {
return r.ValidateBlockChecksums()
})
if err != nil {
// TODO(travers): Hook into the corruption reporting pipeline, once
// available. See pebble#1192.
d.opts.Logger.Fatalf("pebble: encountered corruption during ingestion: %s", err)
}
d.opts.EventListener.TableValidated(TableValidatedInfo{
JobID: jobID,
Meta: f.Meta,
})
}
rs.unref()
d.mu.Lock()
defer d.mu.Unlock()
d.mu.tableValidation.validating = false
d.mu.tableValidation.cond.Broadcast()
if d.shouldValidateSSTablesLocked() {
go d.validateSSTables()
}
}

36
vendor/github.com/cockroachdb/pebble/internal.go generated vendored Normal file
View File

@@ -0,0 +1,36 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import "github.com/cockroachdb/pebble/internal/base"
// InternalKeyKind exports the base.InternalKeyKind type.
type InternalKeyKind = base.InternalKeyKind
// These constants are part of the file format, and should not be changed.
const (
InternalKeyKindDelete = base.InternalKeyKindDelete
InternalKeyKindSet = base.InternalKeyKindSet
InternalKeyKindMerge = base.InternalKeyKindMerge
InternalKeyKindLogData = base.InternalKeyKindLogData
InternalKeyKindSingleDelete = base.InternalKeyKindSingleDelete
InternalKeyKindRangeDelete = base.InternalKeyKindRangeDelete
InternalKeyKindMax = base.InternalKeyKindMax
InternalKeyKindSetWithDelete = base.InternalKeyKindSetWithDelete
InternalKeyKindRangeKeySet = base.InternalKeyKindRangeKeySet
InternalKeyKindRangeKeyUnset = base.InternalKeyKindRangeKeyUnset
InternalKeyKindRangeKeyDelete = base.InternalKeyKindRangeKeyDelete
InternalKeyKindInvalid = base.InternalKeyKindInvalid
InternalKeySeqNumBatch = base.InternalKeySeqNumBatch
InternalKeySeqNumMax = base.InternalKeySeqNumMax
InternalKeyRangeDeleteSentinel = base.InternalKeyRangeDeleteSentinel
)
// InternalKey exports the base.InternalKey type.
type InternalKey = base.InternalKey
type internalIterator = base.InternalIterator
type internalIteratorWithStats = base.InternalIteratorWithStats

201
vendor/github.com/cockroachdb/pebble/internal/arenaskl/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

93
vendor/github.com/cockroachdb/pebble/internal/arenaskl/README.md generated vendored Normal file
View File

@@ -0,0 +1,93 @@
# arenaskl
Fast, lock-free, arena-based Skiplist implementation in Go that supports iteration
in both directions.
## Advantages
Arenaskl offers several advantages over other skiplist implementations:
* High performance that linearly scales with the number of cores. This is
achieved by allocating from a fixed-size arena and by avoiding locks.
* Iterators that can be allocated on the stack and easily cloned by value.
* Simple-to-use and low overhead model for detecting and handling race conditions
with other threads.
* Support for iterating in reverse (i.e. previous links).
## Limitations
The advantages come at a cost that prevents arenaskl from being a general-purpose
skiplist implementation:
* The size of the arena sets a hard upper bound on the combined size of skiplist
nodes, keys, and values. This limit includes even the size of deleted nodes,
keys, and values.
* Deletion is not supported. Instead, higher-level code is expected to
add deletion tombstones and needs to process those tombstones
appropriately.
## Pedigree
This code is based on Andy Kimball's arenaskl code:
https://github.com/andy-kimball/arenaskl
The arenaskl code is based on the skiplist found in Badger, a Go-based
KV store:
https://github.com/dgraph-io/badger/tree/master/skl
The skiplist in Badger is itself based on a C++ skiplist built for
Facebook's RocksDB:
https://github.com/facebook/rocksdb/tree/master/memtable
## Benchmarks
The benchmarks consist of a mix of reads and writes executed in parallel. The
fraction of reads is indicated in the run name: "frac_X" indicates a run where
X percent of the operations are reads.
The results are much better than `skiplist` and `slist`.
```
name time/op
ReadWrite/frac_0-8 470ns ±11%
ReadWrite/frac_10-8 462ns ± 3%
ReadWrite/frac_20-8 436ns ± 2%
ReadWrite/frac_30-8 410ns ± 2%
ReadWrite/frac_40-8 385ns ± 2%
ReadWrite/frac_50-8 360ns ± 4%
ReadWrite/frac_60-8 386ns ± 1%
ReadWrite/frac_70-8 352ns ± 2%
ReadWrite/frac_80-8 306ns ± 3%
ReadWrite/frac_90-8 253ns ± 4%
ReadWrite/frac_100-8 28.1ns ± 2%
```
Note that the above numbers are for concurrent operations using 8x
parallelism. The same benchmarks without concurrency (use these
numbers when comparing vs batchskl):
```
name time/op
ReadWrite/frac_0 1.53µs ± 1%
ReadWrite/frac_10 1.46µs ± 2%
ReadWrite/frac_20 1.39µs ± 3%
ReadWrite/frac_30 1.28µs ± 3%
ReadWrite/frac_40 1.21µs ± 2%
ReadWrite/frac_50 1.11µs ± 3%
ReadWrite/frac_60 1.23µs ±17%
ReadWrite/frac_70 1.16µs ± 4%
ReadWrite/frac_80 959ns ± 3%
ReadWrite/frac_90 738ns ± 5%
ReadWrite/frac_100 81.9ns ± 2%
```
Forward and backward iteration are also fast:
```
name time/op
IterNext 3.97ns ± 5%
IterPrev 3.88ns ± 3%
```

109
vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go generated vendored Normal file
View File

@@ -0,0 +1,109 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"math"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
)
// Arena is lock-free.
type Arena struct {
n uint64
buf []byte
}
const (
align4 = 3
)
var (
// ErrArenaFull indicates that the arena is full and cannot perform any more
// allocations.
ErrArenaFull = errors.New("allocation failed because arena is full")
)
// NewArena allocates a new arena using the specified buffer as the backing
// store.
func NewArena(buf []byte) *Arena {
// Don't store data at position 0 in order to reserve offset=0 as a kind
// of nil pointer.
return &Arena{
n: 1,
buf: buf,
}
}
// Size returns the number of bytes allocated by the arena.
func (a *Arena) Size() uint32 {
s := atomic.LoadUint64(&a.n)
if s > math.MaxUint32 {
// Saturate at MaxUint32.
return math.MaxUint32
}
return uint32(s)
}
// Capacity returns the capacity of the arena.
func (a *Arena) Capacity() uint32 {
return uint32(len(a.buf))
}
func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
// Verify that the arena isn't already full.
origSize := atomic.LoadUint64(&a.n)
if int(origSize) > len(a.buf) {
return 0, 0, ErrArenaFull
}
// Pad the allocation with enough bytes to ensure the requested alignment.
padded := uint32(size) + align
newSize := atomic.AddUint64(&a.n, uint64(padded))
if int(newSize)+int(overflow) > len(a.buf) {
return 0, 0, ErrArenaFull
}
// Return the aligned offset.
offset := (uint32(newSize) - padded + align) & ^align
return offset, padded, nil
}
func (a *Arena) getBytes(offset uint32, size uint32) []byte {
if offset == 0 {
return nil
}
return a.buf[offset : offset+size : offset+size]
}
func (a *Arena) getPointer(offset uint32) unsafe.Pointer {
if offset == 0 {
return nil
}
return unsafe.Pointer(&a.buf[offset])
}
func (a *Arena) getPointerOffset(ptr unsafe.Pointer) uint32 {
if ptr == nil {
return 0
}
return uint32(uintptr(ptr) - uintptr(unsafe.Pointer(&a.buf[0])))
}

80
vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go generated vendored Normal file
View File

@@ -0,0 +1,80 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import "github.com/cockroachdb/pebble/internal/base"
// flushIterator is an iterator over the skiplist object. Use Skiplist.NewFlushIter
// to construct an iterator. The current state of the iterator can be cloned by
// simply value copying the struct.
type flushIterator struct {
Iterator
bytesIterated *uint64
}
// flushIterator implements the base.InternalIterator interface.
var _ base.InternalIterator = (*flushIterator)(nil)
func (it *flushIterator) String() string {
return "memtable"
}
func (it *flushIterator) SeekGE(key []byte, tryNextUsingSeek bool) (*base.InternalKey, []byte) {
panic("pebble: SeekGE unimplemented")
}
func (it *flushIterator) SeekPrefixGE(
prefix, key []byte, trySeekUsingNext bool,
) (*base.InternalKey, []byte) {
panic("pebble: SeekPrefixGE unimplemented")
}
func (it *flushIterator) SeekLT(key []byte) (*base.InternalKey, []byte) {
panic("pebble: SeekLT unimplemented")
}
// First seeks position at the first entry in list. Returns the key and value
// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that First only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound.
func (it *flushIterator) First() (*base.InternalKey, []byte) {
key, val := it.Iterator.First()
if key == nil {
return nil, nil
}
*it.bytesIterated += uint64(it.nd.allocSize)
return key, val
}
// Next advances to the next position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
// Note: flushIterator.Next mirrors the implementation of Iterator.Next
// due to performance. Keep the two in sync.
func (it *flushIterator) Next() (*base.InternalKey, []byte) {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil, nil
}
it.decodeKey()
*it.bytesIterated += uint64(it.nd.allocSize)
return &it.key, it.value()
}
func (it *flushIterator) Prev() (*base.InternalKey, []byte) {
panic("pebble: Prev unimplemented")
}

274
vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,274 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"encoding/binary"
"sync"
"github.com/cockroachdb/pebble/internal/base"
)
type splice struct {
prev *node
next *node
}
func (s *splice) init(prev, next *node) {
s.prev = prev
s.next = next
}
// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
// to construct an iterator. The current state of the iterator can be cloned by
// simply value copying the struct. All iterator methods are thread-safe.
type Iterator struct {
list *Skiplist
nd *node
key base.InternalKey
lower []byte
upper []byte
}
// Iterator implements the base.InternalIterator interface.
var _ base.InternalIterator = (*Iterator)(nil)
var iterPool = sync.Pool{
New: func() interface{} {
return &Iterator{}
},
}
// Close resets the iterator.
func (it *Iterator) Close() error {
it.list = nil
it.nd = nil
it.lower = nil
it.upper = nil
iterPool.Put(it)
return nil
}
func (it *Iterator) String() string {
return "memtable"
}
// Error returns any accumulated error.
func (it *Iterator) Error() error {
return nil
}
// SeekGE moves the iterator to the first entry whose key is greater than or
// equal to the given key. Returns the key and value if the iterator is
// pointing at a valid entry, and (nil, nil) otherwise. Note that SeekGE only
// checks the upper bound. It is up to the caller to ensure that key is greater
// than or equal to the lower bound.
func (it *Iterator) SeekGE(key []byte, trySeekUsingNext bool) (*base.InternalKey, []byte) {
if trySeekUsingNext {
if it.nd == it.list.tail {
// Iterator is done.
return nil, nil
}
less := it.list.cmp(it.key.UserKey, key) < 0
// Arbitrary constant. By measuring the seek cost as a function of the
// number of elements in the skip list, and fitting to a model, we
// could adjust the number of nexts based on the current size of the
// skip list.
const numNexts = 5
for i := 0; less && i < numNexts; i++ {
k, _ := it.Next()
if k == nil {
// Iterator is done.
return nil, nil
}
less = it.list.cmp(it.key.UserKey, key) < 0
}
if !less {
return &it.key, it.value()
}
}
_, it.nd, _ = it.seekForBaseSplice(key)
if it.nd == it.list.tail {
return nil, nil
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, nil
}
return &it.key, it.value()
}
// SeekPrefixGE moves the iterator to the first entry whose key is greater than
// or equal to the given key. This method is equivalent to SeekGE and is
// provided so that an arenaskl.Iterator implements the
// internal/base.InternalIterator interface.
func (it *Iterator) SeekPrefixGE(
prefix, key []byte, trySeekUsingNext bool,
) (*base.InternalKey, []byte) {
return it.SeekGE(key, trySeekUsingNext)
}
// SeekLT moves the iterator to the last entry whose key is less than the given
// key. Returns the key and value if the iterator is pointing at a valid entry,
// and (nil, nil) otherwise. Note that SeekLT only checks the lower bound. It
// is up to the caller to ensure that key is less than the upper bound.
func (it *Iterator) SeekLT(key []byte) (*base.InternalKey, []byte) {
// NB: the top-level Iterator has already adjusted key based on
// the upper-bound.
it.nd, _, _ = it.seekForBaseSplice(key)
if it.nd == it.list.head {
return nil, nil
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, nil
}
return &it.key, it.value()
}
// First seeks position at the first entry in list. Returns the key and value
// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that First only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound (e.g. via a call to SeekGE(lower)).
func (it *Iterator) First() (*base.InternalKey, []byte) {
it.nd = it.list.getNext(it.list.head, 0)
if it.nd == it.list.tail {
return nil, nil
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, nil
}
return &it.key, it.value()
}
// Last seeks position at the last entry in list. Returns the key and value if
// the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that Last only checks the lower bound. It is up to the caller to ensure that
// key is less than the upper bound (e.g. via a call to SeekLT(upper)).
func (it *Iterator) Last() (*base.InternalKey, []byte) {
it.nd = it.list.getPrev(it.list.tail, 0)
if it.nd == it.list.head {
return nil, nil
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, nil
}
return &it.key, it.value()
}
// Next advances to the next position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
// Note: flushIterator.Next mirrors the implementation of Iterator.Next
// due to performance. Keep the two in sync.
func (it *Iterator) Next() (*base.InternalKey, []byte) {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil, nil
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, nil
}
return &it.key, it.value()
}
// Prev moves to the previous position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
func (it *Iterator) Prev() (*base.InternalKey, []byte) {
it.nd = it.list.getPrev(it.nd, 0)
if it.nd == it.list.head {
return nil, nil
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, nil
}
return &it.key, it.value()
}
// value returns the value at the current position.
func (it *Iterator) value() []byte {
return it.nd.getValue(it.list.arena)
}
// Head true iff the iterator is positioned at the sentinel head node.
func (it *Iterator) Head() bool {
return it.nd == it.list.head
}
// Tail true iff the iterator is positioned at the sentinel tail node.
func (it *Iterator) Tail() bool {
return it.nd == it.list.tail
}
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
func (it *Iterator) SetBounds(lower, upper []byte) {
it.lower = lower
it.upper = upper
}
func (it *Iterator) decodeKey() {
b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
// This is a manual inline of base.DecodeInternalKey, because the Go compiler
// seems to refuse to automatically inline it currently.
l := len(b) - 8
if l >= 0 {
it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
it.key.UserKey = b[:l:l]
} else {
it.key.Trailer = uint64(base.InternalKeyKindInvalid)
it.key.UserKey = nil
}
}
func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {
ikey := base.MakeSearchKey(key)
level := int(it.list.Height() - 1)
prev = it.list.head
for {
prev, next, found = it.list.findSpliceForLevel(ikey, level, prev)
if found {
if level != 0 {
// next is pointing at the target node, but we need to find previous on
// the bottom level.
prev = it.list.getPrev(next, 0)
}
break
}
if level == 0 {
break
}
level--
}
return
}

131
vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go generated vendored Normal file
View File

@@ -0,0 +1,131 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"math"
"sync/atomic"
"github.com/cockroachdb/pebble/internal/base"
)
// MaxNodeSize returns the maximum space needed for a node with the specified
// key and value sizes. This could overflow a uint32, which is why a uint64
// is used here. If a key/value overflows a uint32, it should not be added to
// the skiplist.
func MaxNodeSize(keySize, valueSize uint32) uint64 {
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
}
type links struct {
nextOffset uint32
prevOffset uint32
}
func (l *links) init(prevOffset, nextOffset uint32) {
l.nextOffset = nextOffset
l.prevOffset = prevOffset
}
type node struct {
// Immutable fields, so no need to lock to access key.
keyOffset uint32
keySize uint32
valueSize uint32
allocSize uint32
// Most nodes do not need to use the full height of the tower, since the
// probability of each successive level decreases exponentially. Because
// these elements are never accessed, they do not need to be allocated.
// Therefore, when a node is allocated in the arena, its memory footprint
// is deliberately truncated to not include unneeded tower elements.
//
// All accesses to elements should use CAS operations, with no need to lock.
tower [maxHeight]links
}
func newNode(
arena *Arena, height uint32, key base.InternalKey, value []byte,
) (nd *node, err error) {
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
keySize := key.Size()
if int64(keySize) > math.MaxUint32 {
panic("key is too large")
}
valueSize := len(value)
if int64(len(value)) > math.MaxUint32 {
panic("value is too large")
}
if int64(len(value))+int64(keySize)+int64(maxNodeSize) > math.MaxUint32 {
panic("combined key and value size is too large")
}
nd, err = newRawNode(arena, height, uint32(keySize), uint32(valueSize))
if err != nil {
return
}
key.Encode(nd.getKeyBytes(arena))
copy(nd.getValue(arena), value)
return
}
func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *node, err error) {
// Compute the amount of the tower that will never be used, since the height
// is less than maxHeight.
unusedSize := uint32((maxHeight - int(height)) * linksSize)
nodeSize := uint32(maxNodeSize) - unusedSize
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
if err != nil {
return
}
nd = (*node)(arena.getPointer(nodeOffset))
nd.keyOffset = nodeOffset + nodeSize
nd.keySize = keySize
nd.valueSize = valueSize
nd.allocSize = allocSize
return
}
func (n *node) getKeyBytes(arena *Arena) []byte {
return arena.getBytes(n.keyOffset, n.keySize)
}
func (n *node) getValue(arena *Arena) []byte {
return arena.getBytes(n.keyOffset+n.keySize, uint32(n.valueSize))
}
func (n *node) nextOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].nextOffset)
}
func (n *node) prevOffset(h int) uint32 {
return atomic.LoadUint32(&n.tower[h].prevOffset)
}
func (n *node) casNextOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
}
func (n *node) casPrevOffset(h int, old, val uint32) bool {
return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
}

479
vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go generated vendored Normal file
View File

@@ -0,0 +1,479 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Adapted from RocksDB inline skiplist.
Key differences:
- No optimization for sequential inserts (no "prev").
- No custom comparator.
- Support overwrites. This requires care when we see the same key when inserting.
For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
there is no need for values. We don't intend to support versioning. In-place updates of values
would be more efficient.
- We discard all non-concurrent code.
- We do not support Splices. This simplifies the code a lot.
- No AllocateNode or other pointer arithmetic.
- We combine the findLessThan, findGreaterOrEqual, etc into one function.
*/
/*
Further adapted from Badger: https://github.com/dgraph-io/badger.
Key differences:
- Support for previous pointers - doubly linked lists. Note that it's up to higher
level code to deal with the intermediate state that occurs during insertion,
where node A is linked to node B, but node B is not yet linked back to node A.
- Iterator includes mutator functions.
*/
package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
import (
"encoding/binary"
"math"
"runtime"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/fastrand"
)
const (
maxHeight = 20
maxNodeSize = int(unsafe.Sizeof(node{}))
linksSize = int(unsafe.Sizeof(links{}))
pValue = 1 / math.E
)
// ErrRecordExists indicates that an entry with the specified key already
// exists in the skiplist. Duplicate entries are not directly supported and
// instead must be handled by the user by appending a unique version suffix to
// keys.
var ErrRecordExists = errors.New("record with this key already exists")
// Skiplist is a fast, cocnurrent skiplist implementation that supports forward
// and backward iteration. See batchskl.Skiplist for a non-concurrent
// skiplist. Keys and values are immutable once added to the skiplist and
// deletion is not supported. Instead, higher-level code is expected to add new
// entries that shadow existing entries and perform deletion via tombstones. It
// is up to the user to process these shadow entries and tombstones
// appropriately during retrieval.
type Skiplist struct {
arena *Arena
cmp base.Compare
head *node
tail *node
height uint32 // Current height. 1 <= height <= maxHeight. CAS.
// If set to true by tests, then extra delays are added to make it easier to
// detect unusual race conditions.
testing bool
}
// Inserter TODO(peter)
type Inserter struct {
spl [maxHeight]splice
height uint32
}
// Add TODO(peter)
func (ins *Inserter) Add(list *Skiplist, key base.InternalKey, value []byte) error {
return list.addInternal(key, value, ins)
}
var (
probabilities [maxHeight]uint32
)
func init() {
// Precompute the skiplist probabilities so that only a single random number
// needs to be generated and so that the optimal pvalue can be used (inverse
// of Euler's number).
p := float64(1.0)
for i := 0; i < maxHeight; i++ {
probabilities[i] = uint32(float64(math.MaxUint32) * p)
p *= pValue
}
}
// NewSkiplist constructs and initializes a new, empty skiplist. All nodes, keys,
// and values in the skiplist will be allocated from the given arena.
func NewSkiplist(arena *Arena, cmp base.Compare) *Skiplist {
skl := &Skiplist{}
skl.Reset(arena, cmp)
return skl
}
// Reset the skiplist to empty and re-initialize.
func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
// Allocate head and tail nodes.
head, err := newRawNode(arena, maxHeight, 0, 0)
if err != nil {
panic("arenaSize is not large enough to hold the head node")
}
head.keyOffset = 0
tail, err := newRawNode(arena, maxHeight, 0, 0)
if err != nil {
panic("arenaSize is not large enough to hold the tail node")
}
tail.keyOffset = 0
// Link all head/tail levels together.
headOffset := arena.getPointerOffset(unsafe.Pointer(head))
tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
for i := 0; i < maxHeight; i++ {
head.tower[i].nextOffset = tailOffset
tail.tower[i].prevOffset = headOffset
}
*s = Skiplist{
arena: arena,
cmp: cmp,
head: head,
tail: tail,
height: 1,
}
}
// Height returns the height of the highest tower within any of the nodes that
// have ever been allocated as part of this skiplist.
func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
// Arena returns the arena backing this skiplist.
func (s *Skiplist) Arena() *Arena { return s.arena }
// Size returns the number of bytes that have allocated from the arena.
func (s *Skiplist) Size() uint32 { return s.arena.Size() }
// Add adds a new key if it does not yet exist. If the key already exists, then
// Add returns ErrRecordExists. If there isn't enough room in the arena, then
// Add returns ErrArenaFull.
func (s *Skiplist) Add(key base.InternalKey, value []byte) error {
var ins Inserter
return s.addInternal(key, value, &ins)
}
func (s *Skiplist) addInternal(key base.InternalKey, value []byte, ins *Inserter) error {
if s.findSplice(key, ins) {
// Found a matching node, but handle case where it's been deleted.
return ErrRecordExists
}
if s.testing {
// Add delay to make it easier to test race between this thread
// and another thread that sees the intermediate state between
// finding the splice and using it.
runtime.Gosched()
}
nd, height, err := s.newNode(key, value)
if err != nil {
return err
}
ndOffset := s.arena.getPointerOffset(unsafe.Pointer(nd))
// We always insert from the base level and up. After you add a node in base
// level, we cannot create a node in the level above because it would have
// discovered the node in the base level.
var found bool
var invalidateSplice bool
for i := 0; i < int(height); i++ {
prev := ins.spl[i].prev
next := ins.spl[i].next
if prev == nil {
// New node increased the height of the skiplist, so assume that the
// new level has not yet been populated.
if next != nil {
panic("next is expected to be nil, since prev is nil")
}
prev = s.head
next = s.tail
}
// +----------------+ +------------+ +----------------+
// | prev | | nd | | next |
// | prevNextOffset |---->| | | |
// | |<----| prevOffset | | |
// | | | nextOffset |---->| |
// | | | |<----| nextPrevOffset |
// +----------------+ +------------+ +----------------+
//
// 1. Initialize prevOffset and nextOffset to point to prev and next.
// 2. CAS prevNextOffset to repoint from next to nd.
// 3. CAS nextPrevOffset to repoint from prev to nd.
for {
prevOffset := s.arena.getPointerOffset(unsafe.Pointer(prev))
nextOffset := s.arena.getPointerOffset(unsafe.Pointer(next))
nd.tower[i].init(prevOffset, nextOffset)
// Check whether next has an updated link to prev. If it does not,
// that can mean one of two things:
// 1. The thread that added the next node hasn't yet had a chance
// to add the prev link (but will shortly).
// 2. Another thread has added a new node between prev and next.
nextPrevOffset := next.prevOffset(i)
if nextPrevOffset != prevOffset {
// Determine whether #1 or #2 is true by checking whether prev
// is still pointing to next. As long as the atomic operations
// have at least acquire/release semantics (no need for
// sequential consistency), this works, as it is equivalent to
// the "publication safety" pattern.
prevNextOffset := prev.nextOffset(i)
if prevNextOffset == nextOffset {
// Ok, case #1 is true, so help the other thread along by
// updating the next node's prev link.
next.casPrevOffset(i, nextPrevOffset, prevOffset)
}
}
if prev.casNextOffset(i, nextOffset, ndOffset) {
// Managed to insert nd between prev and next, so update the next
// node's prev link and go to the next level.
if s.testing {
// Add delay to make it easier to test race between this thread
// and another thread that sees the intermediate state between
// setting next and setting prev.
runtime.Gosched()
}
next.casPrevOffset(i, prevOffset, ndOffset)
break
}
// CAS failed. We need to recompute prev and next. It is unlikely to
// be helpful to try to use a different level as we redo the search,
// because it is unlikely that lots of nodes are inserted between prev
// and next.
prev, next, found = s.findSpliceForLevel(key, i, prev)
if found {
if i != 0 {
panic("how can another thread have inserted a node at a non-base level?")
}
return ErrRecordExists
}
invalidateSplice = true
}
}
// If we had to recompute the splice for a level, invalidate the entire
// cached splice.
if invalidateSplice {
ins.height = 0
} else {
// The splice was valid. We inserted a node between spl[i].prev and
// spl[i].next. Optimistically update spl[i].prev for use in a subsequent
// call to add.
for i := uint32(0); i < height; i++ {
ins.spl[i].prev = nd
}
}
return nil
}
// NewIter returns a new Iterator object. The lower and upper bound parameters
// control the range of keys the iterator will return. Specifying for nil for
// lower or upper bound disables the check for that boundary. Note that lower
// bound is not checked on {SeekGE,First} and upper bound is not check on
// {SeekLT,Last}. The user is expected to perform that check. Note that it is
// safe for an iterator to be copied by value.
func (s *Skiplist) NewIter(lower, upper []byte) *Iterator {
it := iterPool.Get().(*Iterator)
*it = Iterator{list: s, nd: s.head, lower: lower, upper: upper}
return it
}
// NewFlushIter returns a new flushIterator, which is similar to an Iterator
// but also sets the current number of the bytes that have been iterated
// through.
func (s *Skiplist) NewFlushIter(bytesFlushed *uint64) base.InternalIterator {
return &flushIterator{
Iterator: Iterator{list: s, nd: s.head},
bytesIterated: bytesFlushed,
}
}
func (s *Skiplist) newNode(
key base.InternalKey, value []byte,
) (nd *node, height uint32, err error) {
height = s.randomHeight()
nd, err = newNode(s.arena, height, key, value)
if err != nil {
return
}
// Try to increase s.height via CAS.
listHeight := s.Height()
for height > listHeight {
if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
// Successfully increased skiplist.height.
break
}
listHeight = s.Height()
}
return
}
func (s *Skiplist) randomHeight() uint32 {
rnd := fastrand.Uint32()
h := uint32(1)
for h < maxHeight && rnd <= probabilities[h] {
h++
}
return h
}
func (s *Skiplist) findSplice(key base.InternalKey, ins *Inserter) (found bool) {
listHeight := s.Height()
var level int
prev := s.head
if ins.height < listHeight {
// Our cached height is less than the list height, which means there were
// inserts that increased the height of the list. Recompute the splice from
// scratch.
ins.height = listHeight
level = int(ins.height)
} else {
// Our cached height is equal to the list height.
for ; level < int(listHeight); level++ {
spl := &ins.spl[level]
if s.getNext(spl.prev, level) != spl.next {
// One or more nodes have been inserted between the splice at this
// level.
continue
}
if spl.prev != s.head && !s.keyIsAfterNode(spl.prev, key) {
// Key lies before splice.
level = int(listHeight)
break
}
if spl.next != s.tail && s.keyIsAfterNode(spl.next, key) {
// Key lies after splice.
level = int(listHeight)
break
}
// The splice brackets the key!
prev = spl.prev
break
}
}
for level = level - 1; level >= 0; level-- {
var next *node
prev, next, found = s.findSpliceForLevel(key, level, prev)
if next == nil {
next = s.tail
}
ins.spl[level].init(prev, next)
}
return
}
func (s *Skiplist) findSpliceForLevel(
key base.InternalKey, level int, start *node,
) (prev, next *node, found bool) {
prev = start
for {
// Assume prev.key < key.
next = s.getNext(prev, level)
if next == s.tail {
// Tail node, so done.
break
}
offset, size := next.keyOffset, next.keySize
nextKey := s.arena.buf[offset : offset+size]
n := int32(size) - 8
cmp := s.cmp(key.UserKey, nextKey[:n])
if cmp < 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
if cmp == 0 {
// User-key equality.
var nextTrailer uint64
if n >= 0 {
nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
} else {
nextTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == nextTrailer {
// Internal key equality.
found = true
break
}
if key.Trailer > nextTrailer {
// We are done for this level, since prev.key < key < next.key.
break
}
}
// Keep moving right on this level.
prev = next
}
return
}
func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
n := int32(nd.keySize) - 8
cmp := s.cmp(ndKey[:n], key.UserKey)
if cmp < 0 {
return true
}
if cmp > 0 {
return false
}
// User-key equality.
var ndTrailer uint64
if n >= 0 {
ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
} else {
ndTrailer = uint64(base.InternalKeyKindInvalid)
}
if key.Trailer == ndTrailer {
// Internal key equality.
return false
}
return key.Trailer < ndTrailer
}
func (s *Skiplist) getNext(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
return (*node)(s.arena.getPointer(offset))
}
func (s *Skiplist) getPrev(nd *node, h int) *node {
offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
return (*node)(s.arena.getPointer(offset))
}

60
vendor/github.com/cockroachdb/pebble/internal/base/cleaner.go generated vendored Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/pebble/vfs"
// Cleaner cleans obsolete files.
type Cleaner interface {
Clean(fs vfs.FS, fileType FileType, path string) error
}
// NeedsFileContents is implemented by a cleaner that needs the contents of the
// files that it is being asked to clean.
type NeedsFileContents interface {
needsFileContents()
}
// DeleteCleaner deletes file.
type DeleteCleaner struct{}
// Clean removes file.
func (DeleteCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
return fs.Remove(path)
}
func (DeleteCleaner) String() string {
return "delete"
}
// ArchiveCleaner archives file instead delete.
type ArchiveCleaner struct{}
var _ NeedsFileContents = ArchiveCleaner{}
// Clean archives file.
func (ArchiveCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
switch fileType {
case FileTypeLog, FileTypeManifest, FileTypeTable:
destDir := fs.PathJoin(fs.PathDir(path), "archive")
if err := fs.MkdirAll(destDir, 0755); err != nil {
return err
}
destPath := fs.PathJoin(destDir, fs.PathBase(path))
return fs.Rename(path, destPath)
default:
return fs.Remove(path)
}
}
func (ArchiveCleaner) String() string {
return "archive"
}
func (ArchiveCleaner) needsFileContents() {
}

239
vendor/github.com/cockroachdb/pebble/internal/base/comparer.go generated vendored Normal file
View File

@@ -0,0 +1,239 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"bytes"
"encoding/binary"
"fmt"
"strconv"
"unicode/utf8"
)
// Compare returns -1, 0, or +1 depending on whether a is 'less than', 'equal
// to' or 'greater than' b. The two arguments can only be 'equal' if their
// contents are exactly equal. Furthermore, the empty slice must be 'less than'
// any non-empty slice. Compare is used to compare user keys, such as those
// passed as arguments to the various DB methods, as well as those returned
// from Separator, Successor, and Split.
type Compare func(a, b []byte) int
// Equal returns true if a and b are equivalent. For a given Compare,
// Equal(a,b) must return true iff Compare(a,b) returns zero, that is,
// Equal is a (potentially faster) specialization of Compare.
type Equal func(a, b []byte) bool
// AbbreviatedKey returns a fixed length prefix of a user key such that AbbreviatedKey(a)
// < AbbreviatedKey(b) iff a < b and AbbreviatedKey(a) > AbbreviatedKey(b) iff a > b. If
// AbbreviatedKey(a) == AbbreviatedKey(b) an additional comparison is required to
// determine if the two keys are actually equal.
//
// This helps optimize indexed batch comparisons for cache locality. If a Split
// function is specified, AbbreviatedKey usually returns the first eight bytes
// of the user key prefix in the order that gives the correct ordering.
type AbbreviatedKey func(key []byte) uint64
// FormatKey returns a formatter for the user key.
type FormatKey func(key []byte) fmt.Formatter
// FormatValue returns a formatter for the user value. The key is also
// specified for the value formatter in order to support value formatting that
// is dependent on the key.
type FormatValue func(key, value []byte) fmt.Formatter
// Separator is used to construct SSTable index blocks. A trivial implementation
// is `return a`, but appending fewer bytes leads to smaller SSTables.
//
// Given keys a, b for which Compare(a, b) < 0, Separator returns a key k such
// that:
//
// 1. Compare(a, k) <= 0, and
// 2. Compare(k, b) < 0.
//
// As a special case, b may be nil in which case the second condition is dropped.
//
// For example, if dst, a and b are the []byte equivalents of the strings
// "aqua", "black" and "blue", then the result may be "aquablb".
// Similarly, if the arguments were "aqua", "green" and "", then the result
// may be "aquah".
type Separator func(dst, a, b []byte) []byte
// Successor returns a shortened key given a key a, such that Compare(k, a) >=
// 0. A simple implementation may return a unchanged. The dst parameter may be
// used to store the returned key, though it is valid to pass nil. The returned
// key must be valid to pass to Compare.
type Successor func(dst, a []byte) []byte
// Split returns the length of the prefix of the user key that corresponds to
// the key portion of an MVCC encoding scheme to enable the use of prefix bloom
// filters.
//
// The method will only ever be called with valid MVCC keys, that is, keys that
// the user could potentially store in the database. Pebble does not know which
// keys are MVCC keys and which are not, and may call Split on both MVCC keys
// and non-MVCC keys.
//
// A trivial MVCC scheme is one in which Split() returns len(a). This
// corresponds to assigning a constant version to each key in the database. For
// performance reasons, it is preferable to use a `nil` split in this case.
//
// The returned prefix must have the following properties:
//
// 1) The prefix must be a byte prefix:
//
// bytes.HasPrefix(a, prefix(a))
//
// 2) A key consisting of just a prefix must sort before all other keys with
// that prefix:
//
// Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
//
// 3) Prefixes must be used to order keys before suffixes:
//
// If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
//
// 4) Suffixes themselves must be valid keys and comparable, respecting the same
// ordering as within a key.
//
// If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
//
type Split func(a []byte) int
// Comparer defines a total ordering over the space of []byte keys: a 'less
// than' relationship.
type Comparer struct {
Compare Compare
Equal Equal
AbbreviatedKey AbbreviatedKey
FormatKey FormatKey
FormatValue FormatValue
Separator Separator
Split Split
Successor Successor
// Name is the name of the comparer.
//
// The Level-DB on-disk format stores the comparer name, and opening a
// database with a different comparer from the one it was created with
// will result in an error.
Name string
}
// DefaultFormatter is the default implementation of user key formatting:
// non-ASCII data is formatted as escaped hexadecimal values.
var DefaultFormatter = func(key []byte) fmt.Formatter {
return FormatBytes(key)
}
// DefaultComparer is the default implementation of the Comparer interface.
// It uses the natural ordering, consistent with bytes.Compare.
var DefaultComparer = &Comparer{
Compare: bytes.Compare,
Equal: bytes.Equal,
AbbreviatedKey: func(key []byte) uint64 {
if len(key) >= 8 {
return binary.BigEndian.Uint64(key)
}
var v uint64
for _, b := range key {
v <<= 8
v |= uint64(b)
}
return v << uint(8*(8-len(key)))
},
FormatKey: DefaultFormatter,
Separator: func(dst, a, b []byte) []byte {
i, n := SharedPrefixLen(a, b), len(dst)
dst = append(dst, a...)
min := len(a)
if min > len(b) {
min = len(b)
}
if i >= min {
// Do not shorten if one string is a prefix of the other.
return dst
}
if a[i] >= b[i] {
// b is smaller than a or a is already the shortest possible.
return dst
}
if i < len(b)-1 || a[i]+1 < b[i] {
i += n
dst[i]++
return dst[:i+1]
}
i += n + 1
for ; i < len(dst); i++ {
if dst[i] != 0xff {
dst[i]++
return dst[:i+1]
}
}
return dst
},
Successor: func(dst, a []byte) (ret []byte) {
for i := 0; i < len(a); i++ {
if a[i] != 0xff {
dst = append(dst, a[:i+1]...)
dst[len(dst)-1]++
return dst
}
}
// a is a run of 0xffs, leave it alone.
return append(dst, a...)
},
// This name is part of the C++ Level-DB implementation's default file
// format, and should not be changed.
Name: "leveldb.BytewiseComparator",
}
// SharedPrefixLen returns the largest i such that a[:i] equals b[:i].
// This function can be useful in implementing the Comparer interface.
func SharedPrefixLen(a, b []byte) int {
i, n := 0, len(a)
if n > len(b) {
n = len(b)
}
asUint64 := func(c []byte, i int) uint64 {
return binary.LittleEndian.Uint64(c[i:])
}
for i < n-7 && asUint64(a, i) == asUint64(b, i) {
i += 8
}
for i < n && a[i] == b[i] {
i++
}
return i
}
// FormatBytes formats a byte slice using hexadecimal escapes for non-ASCII
// data.
type FormatBytes []byte
const lowerhex = "0123456789abcdef"
// Format implements the fmt.Formatter interface.
func (p FormatBytes) Format(s fmt.State, c rune) {
buf := make([]byte, 0, len(p))
for _, b := range p {
if b < utf8.RuneSelf && strconv.IsPrint(rune(b)) {
buf = append(buf, b)
continue
}
buf = append(buf, `\x`...)
buf = append(buf, lowerhex[b>>4])
buf = append(buf, lowerhex[b&0xF])
}
s.Write(buf)
}

28
vendor/github.com/cockroachdb/pebble/internal/base/error.go generated vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/errors"
// ErrNotFound means that a get or delete call did not find the requested key.
var ErrNotFound = errors.New("pebble: not found")
// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST,
// sstable) isn't in the expected format.
var ErrCorruption = errors.New("pebble: corruption")
// MarkCorruptionError marks given error as a corruption error.
func MarkCorruptionError(err error) error {
if errors.Is(err, ErrCorruption) {
return err
}
return errors.Mark(err, ErrCorruption)
}
// CorruptionErrorf formats according to a format specifier and returns
// the string as an error value that is marked as a corruption error.
func CorruptionErrorf(format string, args ...interface{}) error {
return errors.Mark(errors.Newf(format, args...), ErrCorruption)
}

169
vendor/github.com/cockroachdb/pebble/internal/base/filenames.go generated vendored Normal file
View File

@@ -0,0 +1,169 @@
// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"fmt"
"strconv"
"strings"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/redact"
)
// FileNum is an internal DB identifier for a file.
type FileNum uint64
// String returns a string representation of the file number.
func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
// FileType enumerates the types of files found in a DB.
type FileType int
// The FileType enumeration.
const (
FileTypeLog FileType = iota
FileTypeLock
FileTypeTable
FileTypeManifest
FileTypeCurrent
FileTypeOptions
FileTypeOldTemp
FileTypeTemp
)
// MakeFilename builds a filename from components.
func MakeFilename(fileType FileType, fileNum FileNum) string {
switch fileType {
case FileTypeLog:
return fmt.Sprintf("%s.log", fileNum)
case FileTypeLock:
return "LOCK"
case FileTypeTable:
return fmt.Sprintf("%s.sst", fileNum)
case FileTypeManifest:
return fmt.Sprintf("MANIFEST-%s", fileNum)
case FileTypeCurrent:
return "CURRENT"
case FileTypeOptions:
return fmt.Sprintf("OPTIONS-%s", fileNum)
case FileTypeOldTemp:
return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum)
case FileTypeTemp:
return fmt.Sprintf("temporary.%s.dbtmp", fileNum)
}
panic("unreachable")
}
// MakeFilepath builds a filepath from components.
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, fileNum))
}
// ParseFilename parses the components from a filename.
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) {
filename = fs.PathBase(filename)
switch {
case filename == "CURRENT":
return FileTypeCurrent, 0, true
case filename == "LOCK":
return FileTypeLock, 0, true
case strings.HasPrefix(filename, "MANIFEST-"):
fileNum, ok = parseFileNum(filename[len("MANIFEST-"):])
if !ok {
break
}
return FileTypeManifest, fileNum, true
case strings.HasPrefix(filename, "OPTIONS-"):
fileNum, ok = parseFileNum(filename[len("OPTIONS-"):])
if !ok {
break
}
return FileTypeOptions, fileNum, ok
case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
if !ok {
break
}
return FileTypeOldTemp, fileNum, ok
case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
fileNum, ok = parseFileNum(s)
if !ok {
break
}
return FileTypeTemp, fileNum, ok
default:
i := strings.IndexByte(filename, '.')
if i < 0 {
break
}
fileNum, ok = parseFileNum(filename[:i])
if !ok {
break
}
switch filename[i+1:] {
case "sst":
return FileTypeTable, fileNum, true
case "log":
return FileTypeLog, fileNum, true
}
}
return 0, fileNum, false
}
func parseFileNum(s string) (fileNum FileNum, ok bool) {
u, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return fileNum, false
}
return FileNum(u), true
}
// A Fataler fatals a process with a message when called.
type Fataler interface {
Fatalf(format string, args ...interface{})
}
// MustExist checks if err is an error indicating a file does not exist.
// If it is, it lists the containing directory's files to annotate the error
// with counts of the various types of files and invokes the provided fataler.
// See cockroachdb/cockroach#56490.
func MustExist(fs vfs.FS, filename string, fataler Fataler, err error) {
if err == nil || !oserror.IsNotExist(err) {
return
}
ls, lsErr := fs.List(fs.PathDir(filename))
if lsErr != nil {
// TODO(jackson): if oserror.IsNotExist(lsErr), the the data directory
// doesn't exist anymore. Another process likely deleted it before
// killing the process. We want to fatal the process, but without
// triggering error reporting like Sentry.
fataler.Fatalf("%s:\norig err: %s\nlist err: %s", redact.Safe(fs.PathBase(filename)), err, lsErr)
}
var total, unknown, tables, logs, manifests int
total = len(ls)
for _, f := range ls {
typ, _, ok := ParseFilename(fs, f)
if !ok {
unknown++
continue
}
switch typ {
case FileTypeTable:
tables++
case FileTypeLog:
logs++
case FileTypeManifest:
manifests++
}
}
fataler.Fatalf("%s:\n%s\ndirectory contains %d files, %d unknown, %d tables, %d logs, %d manifests",
fs.PathBase(filename), err, total, unknown, tables, logs, manifests)
}

412
vendor/github.com/cockroachdb/pebble/internal/base/internal.go generated vendored Normal file
View File

@@ -0,0 +1,412 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base // import "github.com/cockroachdb/pebble/internal/base"
import (
"encoding/binary"
"fmt"
"strconv"
"strings"
)
// InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
// value, a merged value, etc.
type InternalKeyKind uint8
// These constants are part of the file format, and should not be changed.
const (
InternalKeyKindDelete InternalKeyKind = 0
InternalKeyKindSet InternalKeyKind = 1
InternalKeyKindMerge InternalKeyKind = 2
InternalKeyKindLogData InternalKeyKind = 3
//InternalKeyKindColumnFamilyDeletion InternalKeyKind = 4
//InternalKeyKindColumnFamilyValue InternalKeyKind = 5
//InternalKeyKindColumnFamilyMerge InternalKeyKind = 6
InternalKeyKindSingleDelete InternalKeyKind = 7
//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
//InternalKeyKindBeginPrepareXID InternalKeyKind = 9
//InternalKeyKindEndPrepareXID InternalKeyKind = 10
//InternalKeyKindCommitXID InternalKeyKind = 11
//InternalKeyKindRollbackXID InternalKeyKind = 12
//InternalKeyKindNoop InternalKeyKind = 13
//InternalKeyKindColumnFamilyRangeDelete InternalKeyKind = 14
InternalKeyKindRangeDelete InternalKeyKind = 15
//InternalKeyKindColumnFamilyBlobIndex InternalKeyKind = 16
//InternalKeyKindBlobIndex InternalKeyKind = 17
// InternalKeyKindSeparator is a key used for separator / successor keys
// written to sstable block indexes.
//
// NOTE: the RocksDB value has been repurposed. This was done to ensure that
// keys written to block indexes with value "17" (when 17 happened to be the
// max value, and InternalKeyKindMax was therefore set to 17), remain stable
// when new key kinds are supported in Pebble.
InternalKeyKindSeparator InternalKeyKind = 17
// InternalKeyKindSetWithDelete keys are SET keys that have met with a
// DELETE or SINGLEDEL key in a prior compaction. This key kind is
// specific to Pebble. See
// https://github.com/cockroachdb/pebble/issues/1255.
InternalKeyKindSetWithDelete InternalKeyKind = 18
// InternalKeyKindRangeKeyDelete removes all range keys within a key range.
// See the internal/rangekey package for more details.
InternalKeyKindRangeKeyDelete InternalKeyKind = 19
// InternalKeyKindRangeKeySet and InternalKeyKindRangeUnset represent
// keys that set and unset values associated with ranges of key
// space. See the internal/rangekey package for more details.
InternalKeyKindRangeKeyUnset InternalKeyKind = 20
InternalKeyKindRangeKeySet InternalKeyKind = 21
// This maximum value isn't part of the file format. It's unlikely,
// but future extensions may increase this value.
//
// When constructing an internal key to pass to DB.Seek{GE,LE},
// internalKeyComparer sorts decreasing by kind (after sorting increasing by
// user key and decreasing by sequence number). Thus, use InternalKeyKindMax,
// which sorts 'less than or equal to' any other valid internalKeyKind, when
// searching for any kind of internal key formed by a certain user key and
// seqNum.
InternalKeyKindMax InternalKeyKind = 21
// A marker for an invalid key.
InternalKeyKindInvalid InternalKeyKind = 255
// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
// which prevents those entries from being excluded from iteration.
InternalKeySeqNumBatch = uint64(1 << 55)
// InternalKeySeqNumMax is the largest valid sequence number.
InternalKeySeqNumMax = uint64(1<<56 - 1)
// InternalKeyRangeDeleteSentinel is the marker for a range delete sentinel
// key. This sequence number and kind are used for the upper stable boundary
// when a range deletion tombstone is the largest key in an sstable. This is
// necessary because sstable boundaries are inclusive, while the end key of a
// range deletion tombstone is exclusive.
InternalKeyRangeDeleteSentinel = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeDelete)
// InternalKeyBoundaryRangeKey is the marker for a range key boundary. This
// sequence number and kind are used during interleaved range key and point
// iteration to allow an iterator to stop at range key start keys where
// there exists no point key.
InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
)
var internalKeyKindNames = []string{
InternalKeyKindDelete: "DEL",
InternalKeyKindSet: "SET",
InternalKeyKindMerge: "MERGE",
InternalKeyKindLogData: "LOGDATA",
InternalKeyKindSingleDelete: "SINGLEDEL",
InternalKeyKindRangeDelete: "RANGEDEL",
InternalKeyKindSeparator: "SEPARATOR",
InternalKeyKindSetWithDelete: "SETWITHDEL",
InternalKeyKindRangeKeySet: "RANGEKEYSET",
InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET",
InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
InternalKeyKindInvalid: "INVALID",
}
func (k InternalKeyKind) String() string {
if int(k) < len(internalKeyKindNames) {
return internalKeyKindNames[k]
}
return fmt.Sprintf("UNKNOWN:%d", k)
}
// InternalKey is a key used for the in-memory and on-disk partial DBs that
// make up a pebble DB.
//
// It consists of the user key (as given by the code that uses package pebble)
// followed by 8-bytes of metadata:
// - 1 byte for the type of internal key: delete or set,
// - 7 bytes for a uint56 sequence number, in little-endian format.
type InternalKey struct {
UserKey []byte
Trailer uint64
}
// InvalidInternalKey is an invalid internal key for which Valid() will return
// false.
var InvalidInternalKey = MakeInternalKey(nil, 0, InternalKeyKindInvalid)
// MakeInternalKey constructs an internal key from a specified user key,
// sequence number and kind.
func MakeInternalKey(userKey []byte, seqNum uint64, kind InternalKeyKind) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (seqNum << 8) | uint64(kind),
}
}
// MakeTrailer constructs an internal key trailer from the specified sequence
// number and kind.
func MakeTrailer(seqNum uint64, kind InternalKeyKind) uint64 {
return (seqNum << 8) | uint64(kind)
}
// MakeSearchKey constructs an internal key that is appropriate for searching
// for a the specified user key. The search key contain the maximal sequence
// number and kind ensuring that it sorts before any other internal keys for
// the same user key.
func MakeSearchKey(userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindMax),
}
}
// MakeRangeDeleteSentinelKey constructs an internal key that is a range
// deletion sentinel key, used as the upper boundary for an sstable when a
// range deletion is the largest key in an sstable.
func MakeRangeDeleteSentinelKey(userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: InternalKeyRangeDeleteSentinel,
}
}
// MakeExclusiveSentinelKey constructs an internal key that is an
// exclusive sentinel key, used as the upper boundary for an sstable
// when a ranged key is the largest key in an sstable.
func MakeExclusiveSentinelKey(kind InternalKeyKind, userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (InternalKeySeqNumMax << 8) | uint64(kind),
}
}
var kindsMap = map[string]InternalKeyKind{
"DEL": InternalKeyKindDelete,
"SINGLEDEL": InternalKeyKindSingleDelete,
"RANGEDEL": InternalKeyKindRangeDelete,
"SET": InternalKeyKindSet,
"MERGE": InternalKeyKindMerge,
"INVALID": InternalKeyKindInvalid,
"SEPARATOR": InternalKeyKindSeparator,
"SETWITHDEL": InternalKeyKindSetWithDelete,
"RANGEKEYSET": InternalKeyKindRangeKeySet,
"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
"RANGEKEYDEL": InternalKeyKindRangeKeyDelete,
}
// ParseInternalKey parses the string representation of an internal key. The
// format is <user-key>.<kind>.<seq-num>. If the seq-num starts with a "b" it
// is marked as a batch-seq-num (i.e. the InternalKeySeqNumBatch bit is set).
func ParseInternalKey(s string) InternalKey {
x := strings.Split(s, ".")
ukey := x[0]
kind, ok := kindsMap[x[1]]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", x[1]))
}
j := 0
if x[2][0] == 'b' {
j = 1
}
seqNum, _ := strconv.ParseUint(x[2][j:], 10, 64)
if x[2][0] == 'b' {
seqNum |= InternalKeySeqNumBatch
}
return MakeInternalKey([]byte(ukey), seqNum, kind)
}
// ParseKind parses the string representation of an internal key kind.
func ParseKind(s string) InternalKeyKind {
kind, ok := kindsMap[s]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", s))
}
return kind
}
// InternalTrailerLen is the number of bytes used to encode InternalKey.Trailer.
const InternalTrailerLen = 8
// DecodeInternalKey decodes an encoded internal key. See InternalKey.Encode().
func DecodeInternalKey(encodedKey []byte) InternalKey {
n := len(encodedKey) - InternalTrailerLen
var trailer uint64
if n >= 0 {
trailer = binary.LittleEndian.Uint64(encodedKey[n:])
encodedKey = encodedKey[:n:n]
} else {
trailer = uint64(InternalKeyKindInvalid)
encodedKey = nil
}
return InternalKey{
UserKey: encodedKey,
Trailer: trailer,
}
}
// InternalCompare compares two internal keys using the specified comparison
// function. For equal user keys, internal keys compare in descending sequence
// number order. For equal user keys and sequence numbers, internal keys
// compare in descending kind order (this may happen in practice among range
// keys).
func InternalCompare(userCmp Compare, a, b InternalKey) int {
if x := userCmp(a.UserKey, b.UserKey); x != 0 {
return x
}
if a.Trailer > b.Trailer {
return -1
}
if a.Trailer < b.Trailer {
return 1
}
return 0
}
// Encode encodes the receiver into the buffer. The buffer must be large enough
// to hold the encoded data. See InternalKey.Size().
func (k InternalKey) Encode(buf []byte) {
i := copy(buf, k.UserKey)
binary.LittleEndian.PutUint64(buf[i:], k.Trailer)
}
// EncodeTrailer returns the trailer encoded to an 8-byte array.
func (k InternalKey) EncodeTrailer() [8]byte {
var buf [8]byte
binary.LittleEndian.PutUint64(buf[:], k.Trailer)
return buf
}
// Separator returns a separator key such that k <= x && x < other, where less
// than is consistent with the Compare function. The buf parameter may be used
// to store the returned InternalKey.UserKey, though it is valid to pass a
// nil. See the Separator type for details on separator keys.
func (k InternalKey) Separator(
cmp Compare, sep Separator, buf []byte, other InternalKey,
) InternalKey {
buf = sep(buf, k.UserKey, other.UserKey)
if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
// The separator user key is physically shorter than k.UserKey (if it is
// longer, we'll continue to use "k"), but logically after. Tack on the max
// sequence number to the shortened user key. Note that we could tack on
// any sequence number and kind here to create a valid separator key. We
// use the max sequence number to match the behavior of LevelDB and
// RocksDB.
return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
}
return k
}
// Successor returns a successor key such that k <= x. A simple implementation
// may return k unchanged. The buf parameter may be used to store the returned
// InternalKey.UserKey, though it is valid to pass a nil.
func (k InternalKey) Successor(cmp Compare, succ Successor, buf []byte) InternalKey {
buf = succ(buf, k.UserKey)
if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
// The successor user key is physically shorter that k.UserKey (if it is
// longer, we'll continue to use "k"), but logically after. Tack on the max
// sequence number to the shortened user key. Note that we could tack on
// any sequence number and kind here to create a valid separator key. We
// use the max sequence number to match the behavior of LevelDB and
// RocksDB.
return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
}
return k
}
// Size returns the encoded size of the key.
func (k InternalKey) Size() int {
return len(k.UserKey) + 8
}
// SetSeqNum sets the sequence number component of the key.
func (k *InternalKey) SetSeqNum(seqNum uint64) {
k.Trailer = (seqNum << 8) | (k.Trailer & 0xff)
}
// SeqNum returns the sequence number component of the key.
func (k InternalKey) SeqNum() uint64 {
return k.Trailer >> 8
}
// Visible returns true if the key is visible at the specified snapshot
// sequence number.
func (k InternalKey) Visible(snapshot uint64) bool {
return Visible(k.SeqNum(), snapshot)
}
// Visible returns true if a key with the provided sequence number is visible at
// the specified snapshot sequence number.
func Visible(seqNum uint64, snapshot uint64) bool {
return seqNum < snapshot || (seqNum&InternalKeySeqNumBatch) != 0
}
// SetKind sets the kind component of the key.
func (k *InternalKey) SetKind(kind InternalKeyKind) {
k.Trailer = (k.Trailer &^ 0xff) | uint64(kind)
}
// Kind returns the kind compoment of the key.
func (k InternalKey) Kind() InternalKeyKind {
return InternalKeyKind(k.Trailer & 0xff)
}
// Valid returns true if the key has a valid kind.
func (k InternalKey) Valid() bool {
return k.Kind() <= InternalKeyKindMax
}
// Clone clones the storage for the UserKey component of the key.
func (k InternalKey) Clone() InternalKey {
if len(k.UserKey) == 0 {
return k
}
return InternalKey{
UserKey: append([]byte(nil), k.UserKey...),
Trailer: k.Trailer,
}
}
// String returns a string representation of the key.
func (k InternalKey) String() string {
return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())
}
// Pretty returns a formatter for the key.
func (k InternalKey) Pretty(f FormatKey) fmt.Formatter {
return prettyInternalKey{k, f}
}
// IsExclusiveSentinel returns whether this internal key excludes point keys
// with the same user key if used as an end boundary. See the comment on
// InternalKeyRangeDeletionSentinel.
func (k InternalKey) IsExclusiveSentinel() bool {
switch kind := k.Kind(); kind {
case InternalKeyKindRangeDelete:
return k.Trailer == InternalKeyRangeDeleteSentinel
case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
return (k.Trailer >> 8) == InternalKeySeqNumMax
default:
return false
}
}
type prettyInternalKey struct {
InternalKey
formatKey FormatKey
}
func (k prettyInternalKey) Format(s fmt.State, c rune) {
fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
}
// ParsePrettyInternalKey parses the pretty string representation of an
// internal key. The format is <user-key>#<seq-num>,<kind>.
func ParsePrettyInternalKey(s string) InternalKey {
x := strings.FieldsFunc(s, func(c rune) bool { return c == '#' || c == ',' })
ukey := x[0]
kind, ok := kindsMap[x[2]]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", x[2]))
}
seqNum, _ := strconv.ParseUint(x[1], 10, 64)
return MakeInternalKey([]byte(ukey), seqNum, kind)
}

300
vendor/github.com/cockroachdb/pebble/internal/base/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,300 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "fmt"
// InternalIterator iterates over a DB's key/value pairs in key order. Unlike
// the Iterator interface, the returned keys are InternalKeys composed of the
// user-key, a sequence number and a key kind. In forward iteration, key/value
// pairs for identical user-keys are returned in descending sequence order. In
// reverse iteration, key/value pairs for identical user-keys are returned in
// ascending sequence order.
//
// InternalIterators provide 5 absolute positioning methods and 2 relative
// positioning methods. The absolute positioning methods are:
//
// - SeekGE
// - SeekPrefixGE
// - SeekLT
// - First
// - Last
//
// The relative positioning methods are:
//
// - Next
// - Prev
//
// The relative positioning methods can be used in conjunction with any of the
// absolute positioning methods with one exception: SeekPrefixGE does not
// support reverse iteration via Prev. It is undefined to call relative
// positioning methods without ever calling an absolute positioning method.
//
// InternalIterators can optionally implement a prefix iteration mode. This
// mode is entered by calling SeekPrefixGE and exited by any other absolute
// positioning method (SeekGE, SeekLT, First, Last). When in prefix iteration
// mode, a call to Next will advance to the next key which has the same
// "prefix" as the one supplied to SeekPrefixGE. Note that "prefix" in this
// context is not a strict byte prefix, but defined by byte equality for the
// result of the Comparer.Split method. An InternalIterator is not required to
// support prefix iteration mode, and can implement SeekPrefixGE by forwarding
// to SeekGE.
//
// Bounds, [lower, upper), can be set on iterators, either using the SetBounds()
// function in the interface, or in implementation specific ways during iterator
// creation. The forward positioning routines (SeekGE, First, and Next) only
// check the upper bound. The reverse positioning routines (SeekLT, Last, and
// Prev) only check the lower bound. It is up to the caller to ensure that the
// forward positioning routines respect the lower bound and the reverse
// positioning routines respect the upper bound (i.e. calling SeekGE instead of
// First if there is a lower bound, and SeekLT instead of Last if there is an
// upper bound). This imposition is done in order to elevate that enforcement to
// the caller (generally pebble.Iterator or pebble.mergingIter) rather than
// having it duplicated in every InternalIterator implementation.
//
// Additionally, the caller needs to ensure that SeekGE/SeekPrefixGE are not
// called with a key > the upper bound, and SeekLT is not called with a key <
// the lower bound. InternalIterator implementations are required to respect
// the iterator bounds, never returning records outside of the bounds with one
// exception: an iterator may generate synthetic RANGEDEL marker records. See
// levelIter.syntheticBoundary for the sole existing example of this behavior.
// Specifically, levelIter can return synthetic keys whose user key is equal to
// the lower/upper bound.
//
// The bounds provided to an internal iterator must remain valid until a
// subsequent call to SetBounds has returned. This requirement exists so that
// iterator implementations may compare old and new bounds to apply low-level
// optimizations. The pebble.Iterator satisfies this requirement by maintaining
// two bound buffers and switching between them.
//
// An iterator must be closed after use, but it is not necessary to read an
// iterator until exhaustion.
//
// An iterator is not goroutine-safe, but it is safe to use multiple iterators
// concurrently, either in separate goroutines or switching between the
// iterators in a single goroutine.
//
// It is also safe to use an iterator concurrently with modifying its
// underlying DB, if that DB permits modification. However, the resultant
// key/value pairs are not guaranteed to be a consistent snapshot of that DB
// at a particular point in time.
//
// InternalIterators accumulate errors encountered during operation, exposing
// them through the Error method. All of the absolute positioning methods
// reset any accumulated error before positioning. Relative positioning
// methods return without advancing if the iterator has accumulated an error.
type InternalIterator interface {
// SeekGE moves the iterator to the first key/value pair whose key is greater
// than or equal to the given key. Returns the key and value if the iterator
// is pointing at a valid entry, and (nil, nil) otherwise. Note that SeekGE
// only checks the upper bound. It is up to the caller to ensure that key
// is greater than or equal to the lower bound.
//
// trySeekUsingNext is a performance optimization that callers can use to
// indicate that they have not done any action to move this iterator beyond
// the first key that would be found if this iterator were to honestly do
// the intended seek. For example, say the caller did a
// SeekGE(k1...), followed by SeekGE(k2...) where k1 <= k2, without any
// intermediate positioning calls. The caller can safely specify true for this
// parameter in the second call. As another example, say the caller did do one
// call to Next between the two Seek calls, and k1 < k2. Again, the caller can
// safely specify a true value for this parameter. Note that a false value is
// always safe. The callee is free to ignore the true value if its
// implementation does not permit this optimization.
// - We make the caller do this determination since a string comparison of
// k1, k2 is not necessarily cheap, and there may be many iterators in
// the iterator stack. Doing it once at the root of the iterator stack
// is cheaper.
SeekGE(key []byte, trySeekUsingNext bool) (*InternalKey, []byte)
// SeekPrefixGE moves the iterator to the first key/value pair whose key is
// greater than or equal to the given key. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise. Note that
// SeekPrefixGE only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound.
//
// The prefix argument is used by some InternalIterator implementations (e.g.
// sstable.Reader) to avoid expensive operations. A user-defined Split
// function must be supplied to the Comparer for the DB. The supplied prefix
// will be the prefix of the given key returned by that Split function. If
// the iterator is able to determine that no key with the prefix exists, it
// can return (nil,nil). Unlike SeekGE, this is not an indication that
// iteration is exhausted.
//
// Note that the iterator may return keys not matching the prefix. It is up
// to the caller to check if the prefix matches.
//
// Calling SeekPrefixGE places the receiver into prefix iteration mode. Once
// in this mode, reverse iteration may not be supported and will return an
// error. Note that pebble/Iterator.SeekPrefixGE has this same restriction on
// not supporting reverse iteration in prefix iteration mode until a
// different positioning routine (SeekGE, SeekLT, First or Last) switches the
// iterator out of prefix iteration.
//
// trySeekUsingNext is a performance optimization that callers can use to
// indicate that they have not done any action to move this iterator
// beyond the first key that would be found if this iterator were to
// honestly do the indicated seek. For example, say the caller did a
// SeekPrefixGE(k1...), followed by SeekPrefixGE(k2...) where k1 <= k2,
// without any intermediate positioning calls. The caller can safely
// specify true for this parameter in the second call. As another example,
// say the caller did do one call to Next between the two SeekPrefixGE
// calls, and k1 < k2. Again, the caller can safely specify a true value
// for this parameter. Note that a false value is always safe. The callee
// is free to ignore the true value if its implementation does not permit
// this optimization.
// - We make the caller do this determination since a string comparison of
// k1, k2 is not necessarily cheap, and there may be many iterators in
// the iterator stack. Doing it once at the root of the iterator stack
// is cheaper.
// - This optimization could also be applied to SeekLT (where it would be
// trySeekUsingPrev). We currently only do it for SeekPrefixGE and SeekGE
// because this is where this optimization helps the performance of
// CockroachDB. The SeekLT cases in CockroachDB are typically accompanied
// with bounds that change between seek calls, and is optimized inside
// certain iterator implementations, like singleLevelIterator, without any
// extra parameter passing (though the same amortization of string
// comparisons could be done to improve that optimization, by making the
// root of the iterator stack do it).
SeekPrefixGE(prefix, key []byte, trySeekUsingNext bool) (*InternalKey, []byte)
// SeekLT moves the iterator to the last key/value pair whose key is less
// than the given key. Returns the key and value if the iterator is pointing
// at a valid entry, and (nil, nil) otherwise. Note that SeekLT only checks
// the lower bound. It is up to the caller to ensure that key is less than
// the upper bound.
SeekLT(key []byte) (*InternalKey, []byte)
// First moves the iterator the the first key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise. Note that First only checks the upper bound. It is up to the
// caller to ensure that First() is not called when there is a lower bound,
// and instead call SeekGE(lower).
First() (*InternalKey, []byte)
// Last moves the iterator the the last key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise. Note that Last only checks the lower bound. It is up to the
// caller to ensure that Last() is not called when there is an upper bound,
// and instead call SeekLT(upper).
Last() (*InternalKey, []byte)
// Next moves the iterator to the next key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise. Note that Next only checks the upper bound. It is up to the
// caller to ensure that key is greater than or equal to the lower bound.
//
// It is valid to call Next when the iterator is positioned before the first
// key/value pair due to either a prior call to SeekLT or Prev which returned
// (nil, nil). It is not allowed to call Next when the previous call to SeekGE,
// SeekPrefixGE or Next returned (nil, nil).
Next() (*InternalKey, []byte)
// Prev moves the iterator to the previous key/value pair. Returns the key
// and value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise. Note that Prev only checks the lower bound. It is up to the
// caller to ensure that key is less than the upper bound.
//
// It is valid to call Prev when the iterator is positioned after the last
// key/value pair due to either a prior call to SeekGE or Next which returned
// (nil, nil). It is not allowed to call Prev when the previous call to SeekLT
// or Prev returned (nil, nil).
Prev() (*InternalKey, []byte)
// Error returns any accumulated error.
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
// all the key/value pairs in a table is not considered to be an error.
// It is valid to call Close multiple times. Other methods should not be
// called after the iterator has been closed.
Close() error
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
//
// The bounds provided must remain valid until a subsequent call to
// SetBounds has returned. This requirement exists so that iterator
// implementations may compare old and new bounds to apply low-level
// optimizations.
SetBounds(lower, upper []byte)
fmt.Stringer
}
// InternalIteratorWithStats extends InternalIterator to expose stats.
type InternalIteratorWithStats interface {
InternalIterator
Stats() InternalIteratorStats
ResetStats()
}
// InternalIteratorStats contains miscellaneous stats produced by
// InternalIterators that are part of the InternalIterator tree. Not every
// field is relevant for an InternalIterator implementation. The field values
// are aggregated as one goes up the InternalIterator tree.
type InternalIteratorStats struct {
// Bytes in the loaded blocks. If the block was compressed, this is the
// compressed bytes. Currently, only the second-level index and data blocks
// containing points are included.
BlockBytes uint64
// Subset of BlockBytes that were in the block cache.
BlockBytesInCache uint64
// The following can repeatedly count the same points if they are iterated
// over multiple times. Additionally, they may count a point twice when
// switching directions. The latter could be improved if needed.
// Bytes in keys that were iterated over. Currently, only point keys are
// included.
KeyBytes uint64
// Bytes in values that were iterated over. Currently, only point values are
// included.
ValueBytes uint64
// The count of points iterated over.
PointCount uint64
// Points that were iterated over that were covered by range tombstones. It
// can be useful for discovering instances of
// https://github.com/cockroachdb/pebble/issues/1070.
PointsCoveredByRangeTombstones uint64
}
// Merge merges the stats in from into the given stats.
func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
s.BlockBytes += from.BlockBytes
s.BlockBytesInCache += from.BlockBytesInCache
s.KeyBytes += from.KeyBytes
s.ValueBytes += from.ValueBytes
s.PointCount += from.PointCount
s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones
}
type internalIteratorWithEmptyStats struct {
InternalIterator
}
var _ InternalIteratorWithStats = internalIteratorWithEmptyStats{}
// Stats implements InternalIteratorWithStats.
func (i internalIteratorWithEmptyStats) Stats() InternalIteratorStats {
return InternalIteratorStats{}
}
// ResetStats implements InternalIteratorWithStats.
func (i internalIteratorWithEmptyStats) ResetStats() {}
// WrapIterWithStats ensures that either iter implements the stats methods or
// wraps it, such that the return value implements InternalIteratorWithStats.
func WrapIterWithStats(iter InternalIterator) InternalIteratorWithStats {
if iter == nil {
return nil
}
i, ok := iter.(InternalIteratorWithStats)
if ok {
return i
}
return internalIteratorWithEmptyStats{InternalIterator: iter}
}

133
vendor/github.com/cockroachdb/pebble/internal/base/merger.go generated vendored Normal file
View File

@@ -0,0 +1,133 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "io"
// Merge creates a ValueMerger for the specified key initialized with the value
// of one merge operand.
type Merge func(key, value []byte) (ValueMerger, error)
// ValueMerger receives merge operands one by one. The operand received is either
// newer or older than all operands received so far as indicated by the function
// names, `MergeNewer()` and `MergeOlder()`. Once all operands have been received,
// the client will invoke `Finish()` to obtain the final result. The order of
// a merge is not changed after the first call to `MergeNewer()` or
// `MergeOlder()`, i.e. the same method is used to submit all operands.
//
// The implementation may choose to merge values into the result immediately upon
// receiving each operand, or buffer operands until Finish() is called. For example,
// buffering may be useful to avoid (de)serializing partial merge results.
//
// The merge operation must be associative. That is, for the values A, B, C:
//
// Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
//
// Examples of merge operators are integer addition, list append, and string
// concatenation.
type ValueMerger interface {
// MergeNewer adds an operand that is newer than all existing operands.
// The caller retains ownership of value.
//
// If an error is returned the merge is aborted and no other methods must
// be called.
MergeNewer(value []byte) error
// MergeOlder adds an operand that is older than all existing operands.
// The caller retains ownership of value.
//
// If an error is returned the merge is aborted and no other methods must
// be called.
MergeOlder(value []byte) error
// Finish does any final processing of the added operands and returns a
// result. The caller can assume the returned byte slice will not be mutated.
//
// Finish must be the last function called on the ValueMerger. The caller
// must not call any other ValueMerger functions after calling Finish.
//
// If `includesBase` is true, the oldest merge operand was part of the
// merge. This will always be the true during normal iteration, but may be
// false during compaction when only a subset of operands may be
// available. Note that `includesBase` is set to true conservatively: a false
// value means that we could not definitely determine that the base merge
// operand was included.
//
// If a Closer is returned, the returned slice will remain valid until it is
// closed. The caller must arrange for the closer to be eventually closed.
Finish(includesBase bool) ([]byte, io.Closer, error)
}
// DeletableValueMerger is an extension to ValueMerger which allows indicating that the
// result of a merge operation is non-existent. Such non-existent entries will eventually
// be deleted during compaction. Note that during compaction, non-existence of the result
// of a merge means that the merge operands will not result in any record being output.
// This is not the same as transforming the merge operands into a deletion tombstone, as
// older merge operands will still be visible during iteration. Deletion of the merge operands
// in this way is akin to the way a SingleDelete+Set combine into non-existence while leaving
// older records for the same key unaffected.
type DeletableValueMerger interface {
ValueMerger
// DeletableFinish enables a value merger to indicate that the result of a merge operation
// is non-existent. See Finish for a description of includesBase.
DeletableFinish(includesBase bool) (value []byte, delete bool, closer io.Closer, err error)
}
// Merger defines an associative merge operation. The merge operation merges
// two or more values for a single key. A merge operation is requested by
// writing a value using {Batch,DB}.Merge(). The value at that key is merged
// with any existing value. It is valid to Set a value at a key and then Merge
// a new value. Similar to non-merged values, a merged value can be deleted by
// either Delete or DeleteRange.
//
// The merge operation is invoked when a merge value is encountered during a
// read, either during a compaction or during iteration.
type Merger struct {
Merge Merge
// Name is the name of the merger.
//
// Pebble stores the merger name on disk, and opening a database with a
// different merger from the one it was created with will result in an error.
Name string
}
// AppendValueMerger concatenates merge operands in order from oldest to newest.
type AppendValueMerger struct {
buf []byte
}
// MergeNewer appends value to the result.
func (a *AppendValueMerger) MergeNewer(value []byte) error {
a.buf = append(a.buf, value...)
return nil
}
// MergeOlder prepends value to the result, which involves allocating a new buffer.
func (a *AppendValueMerger) MergeOlder(value []byte) error {
buf := make([]byte, len(a.buf)+len(value))
copy(buf, value)
copy(buf[len(value):], a.buf)
a.buf = buf
return nil
}
// Finish returns the buffer that was constructed on-demand in `Merge{OlderNewer}()` calls.
func (a *AppendValueMerger) Finish(includesBase bool) ([]byte, io.Closer, error) {
return a.buf, nil, nil
}
// DefaultMerger is the default implementation of the Merger interface. It
// concatenates the two values to merge.
var DefaultMerger = &Merger{
Merge: func(key, value []byte) (ValueMerger, error) {
res := &AppendValueMerger{}
res.buf = append(res.buf, value...)
return res, nil
},
Name: "pebble.concatenate",
}

76
vendor/github.com/cockroachdb/pebble/internal/base/metrics.go generated vendored Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "time"
// ThroughputMetric is used to measure the byte throughput of some component
// that performs work in a single-threaded manner. The throughput can be
// approximated by Bytes/(WorkDuration+IdleTime). The idle time is represented
// separately, so that the user of this metric could approximate the peak
// throughput as Bytes/WorkTime. The metric is designed to be cumulative (see
// Merge).
type ThroughputMetric struct {
// Bytes is the processes bytes by the component.
Bytes int64
// WorkDuration is the duration that the component spent doing work.
WorkDuration time.Duration
// IdleDuration is the duration that the component was idling, waiting for
// work.
IdleDuration time.Duration
}
// Merge accumulates the information from another throughput metric.
func (tm *ThroughputMetric) Merge(x ThroughputMetric) {
tm.Bytes += x.Bytes
tm.WorkDuration += x.WorkDuration
tm.IdleDuration += x.IdleDuration
}
// PeakRate returns the approximate peak rate if there was no idling.
func (tm *ThroughputMetric) PeakRate() int64 {
if tm.Bytes == 0 {
return 0
}
return int64((float64(tm.Bytes) / float64(tm.WorkDuration)) * float64(time.Second))
}
// Rate returns the observed rate.
func (tm *ThroughputMetric) Rate() int64 {
if tm.Bytes == 0 {
return 0
}
return int64((float64(tm.Bytes) / float64(tm.WorkDuration+tm.IdleDuration)) *
float64(time.Second))
}
// GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
// accumulating samples of that gauge.
type GaugeSampleMetric struct {
// The sum of all the samples.
sampleSum int64
// The number of samples.
count int64
}
// AddSample adds the given sample.
func (gsm *GaugeSampleMetric) AddSample(sample int64) {
gsm.sampleSum += sample
gsm.count++
}
// Merge accumulates the information from another gauge metric.
func (gsm *GaugeSampleMetric) Merge(x GaugeSampleMetric) {
gsm.sampleSum += x.sampleSum
gsm.count += x.count
}
// Mean returns the mean value.
func (gsm *GaugeSampleMetric) Mean() float64 {
if gsm.count == 0 {
return 0
}
return float64(gsm.sampleSum) / float64(gsm.count)
}

76
vendor/github.com/cockroachdb/pebble/internal/base/options.go generated vendored Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
// SSTable block defaults.
const (
DefaultBlockRestartInterval = 16
DefaultBlockSize = 4096
DefaultBlockSizeThreshold = 90
)
// FilterType is the level at which to apply a filter: block or table.
type FilterType int
// The available filter types.
const (
TableFilter FilterType = iota
)
func (t FilterType) String() string {
switch t {
case TableFilter:
return "table"
}
return "unknown"
}
// FilterWriter provides an interface for creating filter blocks. See
// FilterPolicy for more details about filters.
type FilterWriter interface {
// AddKey adds a key to the current filter block.
AddKey(key []byte)
// Finish appends to dst an encoded filter tha holds the current set of
// keys. The writer state is reset after the call to Finish allowing the
// writer to be reused for the creation of additional filters.
Finish(dst []byte) []byte
}
// FilterPolicy is an algorithm for probabilistically encoding a set of keys.
// The canonical implementation is a Bloom filter.
//
// Every FilterPolicy has a name. This names the algorithm itself, not any one
// particular instance. Aspects specific to a particular instance, such as the
// set of keys or any other parameters, will be encoded in the []byte filter
// returned by NewWriter.
//
// The name may be written to files on disk, along with the filter data. To use
// these filters, the FilterPolicy name at the time of writing must equal the
// name at the time of reading. If they do not match, the filters will be
// ignored, which will not affect correctness but may affect performance.
type FilterPolicy interface {
// Name names the filter policy.
Name() string
// MayContain returns whether the encoded filter may contain given key.
// False positives are possible, where it returns true for keys not in the
// original set.
MayContain(ftype FilterType, filter, key []byte) bool
// NewWriter creates a new FilterWriter.
NewWriter(ftype FilterType) FilterWriter
}
// BlockPropertyFilter is used in an Iterator to filter sstables and blocks
// within the sstable. It should not maintain any per-sstable state, and must
// be thread-safe.
type BlockPropertyFilter interface {
// Name returns the name of the block property collector.
Name() string
// Intersects returns true if the set represented by prop intersects with
// the set in the filter.
Intersects(prop []byte) (bool, error)
}

56
vendor/github.com/cockroachdb/pebble/internal/batchskl/README.md generated vendored Normal file
View File

@@ -0,0 +1,56 @@
# batchskl
Fast, non-concurrent skiplist implementation in Go that supports
forward and backward iteration.
## Limitations
* The interface is tailored for use in indexing pebble batches. Keys
and values are stored outside of the skiplist making the skiplist
awkward for general purpose use.
* Deletion is not supported. Instead, higher-level code is expected to
add deletion tombstones and needs to process those tombstones
appropriately.
## Pedigree
This code is based on Andy Kimball's arenaskl code.
The arenaskl code is based on the skiplist found in Badger, a Go-based
KV store:
https://github.com/dgraph-io/badger/tree/master/skl
The skiplist in Badger is itself based on a C++ skiplist built for
Facebook's RocksDB:
https://github.com/facebook/rocksdb/tree/master/memtable
## Benchmarks
The benchmarks consist of a mix of reads and writes executed in parallel. The
fraction of reads is indicated in the run name: "frac_X" indicates a run where
X percent of the operations are reads.
```
name time/op
ReadWrite/frac_0 1.03µs ± 2%
ReadWrite/frac_10 1.32µs ± 1%
ReadWrite/frac_20 1.26µs ± 1%
ReadWrite/frac_30 1.18µs ± 1%
ReadWrite/frac_40 1.09µs ± 1%
ReadWrite/frac_50 987ns ± 2%
ReadWrite/frac_60 1.07µs ± 1%
ReadWrite/frac_70 909ns ± 1%
ReadWrite/frac_80 693ns ± 2%
ReadWrite/frac_90 599ns ± 2%
ReadWrite/frac_100 45.3ns ± 3%
```
Forward and backward iteration are also fast:
```
name time/op
IterNext 4.49ns ± 3%
IterPrev 4.48ns ± 3%
```

199
vendor/github.com/cockroachdb/pebble/internal/batchskl/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,199 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package batchskl
import "github.com/cockroachdb/pebble/internal/base"
type splice struct {
prev uint32
next uint32
}
// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
// to construct an iterator. The current state of the iterator can be cloned
// by simply value copying the struct.
type Iterator struct {
list *Skiplist
nd uint32
key base.InternalKey
lower []byte
upper []byte
}
// Close resets the iterator.
func (it *Iterator) Close() error {
it.list = nil
it.nd = 0
return nil
}
// SeekGE moves the iterator to the first entry whose key is greater than or
// equal to the given key. Returns true if the iterator is pointing at a valid
// entry and false otherwise. Note that SeekGE only checks the upper bound. It
// is up to the caller to ensure that key is greater than or equal to the lower
// bound.
func (it *Iterator) SeekGE(key []byte) *base.InternalKey {
_, it.nd = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// SeekLT moves the iterator to the last entry whose key is less the given
// key. Returns true if the iterator is pointing at a valid entry and false
// otherwise. Note that SeekLT only checks the lower bound. It is up to the
// caller to ensure that key is less than the upper bound.
func (it *Iterator) SeekLT(key []byte) *base.InternalKey {
it.nd, _ = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// First seeks position at the first entry in list. Final state of iterator is
// Valid() iff list is not empty. Note that First only checks the upper
// bound. It is up to the caller to ensure that key is greater than or equal to
// the lower bound (e.g. via a call to SeekGE(lower)).
func (it *Iterator) First() *base.InternalKey {
it.nd = it.list.getNext(it.list.head, 0)
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// Last seeks position at the last entry in list. Final state of iterator is
// Valid() iff list is not empty. Note that Last only checks the lower
// bound. It is up to the caller to ensure that key is less than the upper
// bound (e.g. via a call to SeekLT(upper)).
func (it *Iterator) Last() *base.InternalKey {
it.nd = it.list.getPrev(it.list.tail, 0)
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// Next advances to the next position. If there are no following nodes, then
// Valid() will be false after this call.
func (it *Iterator) Next() *base.InternalKey {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// Prev moves to the previous position. If there are no previous nodes, then
// Valid() will be false after this call.
func (it *Iterator) Prev() *base.InternalKey {
it.nd = it.list.getPrev(it.nd, 0)
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// Key returns the key at the current position.
func (it *Iterator) Key() *base.InternalKey {
return &it.key
}
// KeyInfo returns the offset of the start of the record, the start of the key,
// and the end of the key.
func (it *Iterator) KeyInfo() (offset, keyStart, keyEnd uint32) {
n := it.list.node(it.nd)
return n.offset, n.keyStart, n.keyEnd
}
// Head true iff the iterator is positioned at the sentinel head node.
func (it *Iterator) Head() bool {
return it.nd == it.list.head
}
// Tail true iff the iterator is positioned at the sentinel tail node.
func (it *Iterator) Tail() bool {
return it.nd == it.list.tail
}
// Valid returns nil iff the iterator is positioned at a valid node.
func (it *Iterator) Valid() bool {
return it.list != nil && it.nd != it.list.head && it.nd != it.list.tail
}
func (it *Iterator) String() string {
return "batch"
}
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekLT, First, or Last.
func (it *Iterator) SetBounds(lower, upper []byte) {
it.lower = lower
it.upper = upper
}
func (it *Iterator) seekForBaseSplice(key []byte, abbreviatedKey uint64) (prev, next uint32) {
prev = it.list.head
for level := it.list.height - 1; ; level-- {
prev, next = it.list.findSpliceForLevel(key, abbreviatedKey, level, prev)
if level == 0 {
break
}
}
return
}

441
vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go generated vendored Normal file
View File

@@ -0,0 +1,441 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Adapted from RocksDB inline skiplist.
Key differences:
- No optimization for sequential inserts (no "prev").
- No custom comparator.
- Support overwrites. This requires care when we see the same key when inserting.
For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
there is no need for values. We don't intend to support versioning. In-place updates of values
would be more efficient.
- We discard all non-concurrent code.
- We do not support Splices. This simplifies the code a lot.
- No AllocateNode or other pointer arithmetic.
- We combine the findLessThan, findGreaterOrEqual, etc into one function.
*/
/*
Further adapted from Badger: https://github.com/dgraph-io/badger.
Key differences:
- Support for previous pointers - doubly linked lists. Note that it's up to higher
level code to deal with the intermediate state that occurs during insertion,
where node A is linked to node B, but node B is not yet linked back to node A.
- Iterator includes mutator functions.
*/
/*
Further adapted from arenaskl: https://github.com/andy-kimball/arenaskl
Key differences:
- Removed support for deletion.
- Removed support for concurrency.
- External storage of keys.
- Node storage grows to an arbitrary size.
*/
package batchskl // import "github.com/cockroachdb/pebble/internal/batchskl"
import (
"bytes"
"encoding/binary"
"fmt"
"math"
"time"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"golang.org/x/exp/rand"
)
const (
maxHeight = 20
maxNodeSize = int(unsafe.Sizeof(node{}))
linksSize = int(unsafe.Sizeof(links{}))
maxNodesSize = math.MaxUint32
)
var (
// ErrExists indicates that a duplicate record was inserted. This should never
// happen for normal usage of batchskl as every key should have a unique
// sequence number.
ErrExists = errors.New("record with this key already exists")
// ErrTooManyRecords is a sentinel error returned when the size of the raw
// nodes slice exceeds the maximum allowed size (currently 1 << 32 - 1). This
// corresponds to ~117 M skiplist entries.
ErrTooManyRecords = errors.New("too many records")
)
type links struct {
next uint32
prev uint32
}
type node struct {
// The offset of the start of the record in the storage.
offset uint32
// The offset of the start and end of the key in storage.
keyStart uint32
keyEnd uint32
// A fixed 8-byte abbreviation of the key, used to avoid retrieval of the key
// during seek operations. The key retrieval can be expensive purely due to
// cache misses while the abbreviatedKey stored here will be in the same
// cache line as the key and the links making accessing and comparing against
// it almost free.
abbreviatedKey uint64
// Most nodes do not need to use the full height of the link tower, since the
// probability of each successive level decreases exponentially. Because
// these elements are never accessed, they do not need to be allocated.
// Therefore, when a node is allocated, its memory footprint is deliberately
// truncated to not include unneeded link elements.
links [maxHeight]links
}
// Skiplist is a fast, non-cocnurrent skiplist implementation that supports
// forward and backward iteration. See arenaskl.Skiplist for a concurrent
// skiplist. Keys and values are stored externally from the skiplist via the
// Storage interface. Deletion is not supported. Instead, higher-level code is
// expected to perform deletion via tombstones and needs to process those
// tombstones appropriately during retrieval operations.
type Skiplist struct {
storage *[]byte
cmp base.Compare
abbreviatedKey base.AbbreviatedKey
nodes []byte
head uint32
tail uint32
height uint32 // Current height: 1 <= height <= maxHeight
rand rand.PCGSource
}
var (
probabilities [maxHeight]uint32
)
func init() {
const pValue = 1 / math.E
// Precompute the skiplist probabilities so that only a single random number
// needs to be generated and so that the optimal pvalue can be used (inverse
// of Euler's number).
p := float64(1.0)
for i := 0; i < maxHeight; i++ {
probabilities[i] = uint32(float64(math.MaxUint32) * p)
p *= pValue
}
}
// NewSkiplist constructs and initializes a new, empty skiplist.
func NewSkiplist(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) *Skiplist {
s := &Skiplist{}
s.Init(storage, cmp, abbreviatedKey)
return s
}
// Reset the fields in the skiplist for reuse.
func (s *Skiplist) Reset() {
*s = Skiplist{
nodes: s.nodes[:0],
height: 1,
}
const batchMaxRetainedSize = 1 << 20 // 1 MB
if cap(s.nodes) > batchMaxRetainedSize {
s.nodes = nil
}
}
// Init the skiplist to empty and re-initialize.
func (s *Skiplist) Init(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) {
*s = Skiplist{
storage: storage,
cmp: cmp,
abbreviatedKey: abbreviatedKey,
nodes: s.nodes[:0],
height: 1,
}
s.rand.Seed(uint64(time.Now().UnixNano()))
const initBufSize = 256
if cap(s.nodes) < initBufSize {
s.nodes = make([]byte, 0, initBufSize)
}
// Allocate head and tail nodes. While allocating a new node can fail, in the
// context of initializing the skiplist we consider it unrecoverable.
var err error
s.head, err = s.newNode(maxHeight, 0, 0, 0, 0)
if err != nil {
panic(err)
}
s.tail, err = s.newNode(maxHeight, 0, 0, 0, 0)
if err != nil {
panic(err)
}
// Link all head/tail levels together.
headNode := s.node(s.head)
tailNode := s.node(s.tail)
for i := uint32(0); i < maxHeight; i++ {
headNode.links[i].next = s.tail
tailNode.links[i].prev = s.head
}
}
// Add adds a new key to the skiplist if it does not yet exist. If the record
// already exists, then Add returns ErrRecordExists.
func (s *Skiplist) Add(keyOffset uint32) error {
data := (*s.storage)[keyOffset+1:]
v, n := binary.Uvarint(data)
if n <= 0 {
return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
}
data = data[n:]
if v > uint64(len(data)) {
return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
}
keyStart := 1 + keyOffset + uint32(n)
keyEnd := keyStart + uint32(v)
key := data[:v]
abbreviatedKey := s.abbreviatedKey(key)
// spl holds the list of next and previous links for each level in the
// skiplist indicating where the new node will be inserted.
var spl [maxHeight]splice
// Fast-path for in-order insertion of keys: compare the new key against the
// last key.
prev := s.getPrev(s.tail, 0)
if prevNode := s.node(prev); prev == s.head ||
abbreviatedKey > prevNode.abbreviatedKey ||
(abbreviatedKey == prevNode.abbreviatedKey &&
s.cmp(key, (*s.storage)[prevNode.keyStart:prevNode.keyEnd]) > 0) {
for level := uint32(0); level < s.height; level++ {
spl[level].prev = s.getPrev(s.tail, level)
spl[level].next = s.tail
}
} else {
s.findSplice(key, abbreviatedKey, &spl)
}
height := s.randomHeight()
// Increase s.height as necessary.
for ; s.height < height; s.height++ {
spl[s.height].next = s.tail
spl[s.height].prev = s.head
}
// We always insert from the base level and up. After you add a node in base
// level, we cannot create a node in the level above because it would have
// discovered the node in the base level.
nd, err := s.newNode(height, keyOffset, keyStart, keyEnd, abbreviatedKey)
if err != nil {
return err
}
newNode := s.node(nd)
for level := uint32(0); level < height; level++ {
next := spl[level].next
prev := spl[level].prev
newNode.links[level].next = next
newNode.links[level].prev = prev
s.node(next).links[level].prev = nd
s.node(prev).links[level].next = nd
}
return nil
}
// NewIter returns a new Iterator object. The lower and upper bound parameters
// control the range of keys the iterator will return. Specifying for nil for
// lower or upper bound disables the check for that boundary. Note that lower
// bound is not checked on {SeekGE,First} and upper bound is not check on
// {SeekLT,Last}. The user is expected to perform that check. Note that it is
// safe for an iterator to be copied by value.
func (s *Skiplist) NewIter(lower, upper []byte) Iterator {
return Iterator{list: s, lower: lower, upper: upper}
}
func (s *Skiplist) newNode(
height,
offset, keyStart, keyEnd uint32, abbreviatedKey uint64,
) (uint32, error) {
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
unusedSize := (maxHeight - int(height)) * linksSize
nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
if err != nil {
return 0, err
}
nd := s.node(nodeOffset)
nd.offset = offset
nd.keyStart = keyStart
nd.keyEnd = keyEnd
nd.abbreviatedKey = abbreviatedKey
return nodeOffset, nil
}
func (s *Skiplist) alloc(size uint32) (uint32, error) {
offset := len(s.nodes)
// We only have a need for memory up to offset + size, but we never want
// to allocate a node whose tail points into unallocated memory.
minAllocSize := offset + maxNodeSize
if cap(s.nodes) < minAllocSize {
allocSize := cap(s.nodes) * 2
if allocSize < minAllocSize {
allocSize = minAllocSize
}
// Cap the allocation at the max allowed size to avoid wasted capacity.
if allocSize > maxNodesSize {
// The new record may still not fit within the allocation, in which case
// we return early with an error. This avoids the panic below when we
// resize the slice. It also avoids the allocation and copy.
if uint64(offset)+uint64(size) > maxNodesSize {
return 0, errors.Wrapf(ErrTooManyRecords,
"alloc of new record (size=%d) would overflow uint32 (current size=%d)",
uint64(offset)+uint64(size), offset,
)
}
allocSize = maxNodesSize
}
tmp := make([]byte, len(s.nodes), allocSize)
copy(tmp, s.nodes)
s.nodes = tmp
}
newSize := uint32(offset) + size
s.nodes = s.nodes[:newSize]
return uint32(offset), nil
}
func (s *Skiplist) node(offset uint32) *node {
return (*node)(unsafe.Pointer(&s.nodes[offset]))
}
func (s *Skiplist) randomHeight() uint32 {
rnd := uint32(s.rand.Uint64())
h := uint32(1)
for h < maxHeight && rnd <= probabilities[h] {
h++
}
return h
}
func (s *Skiplist) findSplice(key []byte, abbreviatedKey uint64, spl *[maxHeight]splice) {
prev := s.head
for level := s.height - 1; ; level-- {
// The code in this loop is the same as findSpliceForLevel(). For some
// reason, calling findSpliceForLevel() here is much much slower than the
// inlined code below. The excess time is also caught up in the final
// return statement which makes little sense. Revisit when in go1.14 or
// later if inlining improves.
next := s.getNext(prev, level)
for next != s.tail {
// Assume prev.key < key.
nextNode := s.node(next)
nextAbbreviatedKey := nextNode.abbreviatedKey
if abbreviatedKey < nextAbbreviatedKey {
// We are done for this level, since prev.key < key < next.key.
break
}
if abbreviatedKey == nextAbbreviatedKey {
if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
// We are done for this level, since prev.key < key <= next.key.
break
}
}
// Keep moving right on this level.
prev = next
next = nextNode.links[level].next
}
spl[level].prev = prev
spl[level].next = next
if level == 0 {
break
}
}
}
func (s *Skiplist) findSpliceForLevel(
key []byte, abbreviatedKey uint64, level, start uint32,
) (prev, next uint32) {
prev = start
next = s.getNext(prev, level)
for next != s.tail {
// Assume prev.key < key.
nextNode := s.node(next)
nextAbbreviatedKey := nextNode.abbreviatedKey
if abbreviatedKey < nextAbbreviatedKey {
// We are done for this level, since prev.key < key < next.key.
break
}
if abbreviatedKey == nextAbbreviatedKey {
if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
}
// Keep moving right on this level.
prev = next
next = nextNode.links[level].next
}
return
}
func (s *Skiplist) getKey(nd uint32) base.InternalKey {
n := s.node(nd)
kind := base.InternalKeyKind((*s.storage)[n.offset])
key := (*s.storage)[n.keyStart:n.keyEnd]
return base.MakeInternalKey(key, uint64(n.offset)|base.InternalKeySeqNumBatch, kind)
}
func (s *Skiplist) getNext(nd, h uint32) uint32 {
return s.node(nd).links[h].next
}
func (s *Skiplist) getPrev(nd, h uint32) uint32 {
return s.node(nd).links[h].prev
}
func (s *Skiplist) debug() string {
var buf bytes.Buffer
for level := uint32(0); level < s.height; level++ {
var count int
for nd := s.head; nd != s.tail; nd = s.getNext(nd, level) {
count++
}
fmt.Fprintf(&buf, "%d: %d\n", level, count)
}
return buf.String()
}
// Silence unused warning.
var _ = (*Skiplist).debug

61
vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go generated vendored Normal file
View File

@@ -0,0 +1,61 @@
// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package bytealloc
import "github.com/cockroachdb/pebble/internal/rawalloc"
// An A provides chunk allocation of []byte, amortizing the overhead of each
// allocation. Because the underlying storage for the slices is shared, they
// should share a similar lifetime in order to avoid pinning large amounts of
// memory unnecessarily. The allocator itself is a []byte where cap() indicates
// the total amount of memory and len() is the amount already allocated. The
// size of the buffer to allocate from is grown exponentially when it runs out
// of room up to a maximum size (chunkAllocMaxSize).
type A []byte
const chunkAllocMinSize = 512
const chunkAllocMaxSize = 16384
func (a A) reserve(n int) A {
allocSize := cap(a) * 2
if allocSize < chunkAllocMinSize {
allocSize = chunkAllocMinSize
} else if allocSize > chunkAllocMaxSize {
allocSize = chunkAllocMaxSize
}
if allocSize < n {
allocSize = n
}
return rawalloc.New(0, allocSize)
}
// Alloc allocates a new chunk of memory with the specified length.
func (a A) Alloc(n int) (A, []byte) {
if cap(a)-len(a) < n {
a = a.reserve(n)
}
p := len(a)
r := a[p : p+n : p+n]
a = a[:p+n]
return a, r
}
// Copy allocates a new chunk of memory, initializing it from src.
func (a A) Copy(src []byte) (A, []byte) {
var alloc []byte
a, alloc = a.Alloc(len(src))
copy(alloc, src)
return a, alloc
}

21
vendor/github.com/cockroachdb/pebble/internal/cache/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
The MIT License
Copyright (c) 2018 Damian Gryski
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

10
vendor/github.com/cockroachdb/pebble/internal/cache/cgo_disabled.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !cgo
// +build !cgo
package cache
const cgoEnabled = false

10
vendor/github.com/cockroachdb/pebble/internal/cache/cgo_enabled.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build cgo
// +build cgo
package cache
const cgoEnabled = true

841
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go generated vendored Normal file
View File

@@ -0,0 +1,841 @@
// Copyright 2018. All rights reserved. Use of this source code is governed by
// an MIT-style license that can be found in the LICENSE file.
// Package cache implements the CLOCK-Pro caching algorithm.
//
// CLOCK-Pro is a patent-free alternative to the Adaptive Replacement Cache,
// https://en.wikipedia.org/wiki/Adaptive_replacement_cache.
// It is an approximation of LIRS ( https://en.wikipedia.org/wiki/LIRS_caching_algorithm ),
// much like the CLOCK page replacement algorithm is an approximation of LRU.
//
// This implementation is based on the python code from https://bitbucket.org/SamiLehtinen/pyclockpro .
//
// Slides describing the algorithm: http://fr.slideshare.net/huliang64/clockpro
//
// The original paper: http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html
//
// It is MIT licensed, like the original.
package cache // import "github.com/cockroachdb/pebble/internal/cache"
import (
"fmt"
"os"
"runtime"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
type fileKey struct {
// id is the namespace for fileNums.
id uint64
fileNum base.FileNum
}
type key struct {
fileKey
offset uint64
}
// file returns the "file key" for the receiver. This is the key used for the
// shard.files map.
func (k key) file() key {
k.offset = 0
return k
}
func (k key) String() string {
return fmt.Sprintf("%d/%d/%d", k.id, k.fileNum, k.offset)
}
// Handle provides a strong reference to a value in the cache. The reference
// does not pin the value in the cache, but it does prevent the underlying byte
// slice from being reused.
type Handle struct {
value *Value
}
// Get returns the value stored in handle.
func (h Handle) Get() []byte {
if h.value != nil {
// NB: We don't increment shard.hits in this code path because we only want
// to record a hit when the handle is retrieved from the cache.
return h.value.buf
}
return nil
}
// Release releases the reference to the cache entry.
func (h Handle) Release() {
if h.value != nil {
h.value.release()
}
}
type shard struct {
hits int64
misses int64
mu sync.RWMutex
reservedSize int64
maxSize int64
coldTarget int64
blocks robinHoodMap // fileNum+offset -> block
files robinHoodMap // fileNum -> list of blocks
// The blocks and files maps store values in manually managed memory that is
// invisible to the Go GC. This is fine for Value and entry objects that are
// stored in manually managed memory, but when the "invariants" build tag is
// set, all Value and entry objects are Go allocated and the entries map will
// contain a reference to every entry.
entries map[*entry]struct{}
handHot *entry
handCold *entry
handTest *entry
sizeHot int64
sizeCold int64
sizeTest int64
// The count fields are used exclusively for asserting expectations.
// We've seen infinite looping (cockroachdb/cockroach#70154) that
// could be explained by a corrupted sizeCold. Through asserting on
// these fields, we hope to gain more insight from any future
// reproductions.
countHot int64
countCold int64
countTest int64
}
func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
c.mu.RLock()
var value *Value
if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
value = e.acquireValue()
if value != nil {
atomic.StoreInt32(&e.referenced, 1)
}
}
c.mu.RUnlock()
if value == nil {
atomic.AddInt64(&c.misses, 1)
return Handle{}
}
atomic.AddInt64(&c.hits, 1)
return Handle{value: value}
}
func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
if n := value.refs(); n != 1 {
panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
}
c.mu.Lock()
defer c.mu.Unlock()
k := key{fileKey{id, fileNum}, offset}
e := c.blocks.Get(k)
switch {
case e == nil:
// no cache entry? add it
e = newEntry(c, k, int64(len(value.buf)))
e.setValue(value)
if c.metaAdd(k, e) {
value.ref.trace("add-cold")
c.sizeCold += e.size
c.countCold++
} else {
value.ref.trace("skip-cold")
e.free()
e = nil
}
case e.peekValue() != nil:
// cache entry was a hot or cold page
e.setValue(value)
atomic.StoreInt32(&e.referenced, 1)
delta := int64(len(value.buf)) - e.size
e.size = int64(len(value.buf))
if e.ptype == etHot {
value.ref.trace("add-hot")
c.sizeHot += delta
} else {
value.ref.trace("add-cold")
c.sizeCold += delta
}
c.evict()
default:
// cache entry was a test page
c.sizeTest -= e.size
c.countTest--
c.metaDel(e)
c.metaCheck(e)
e.size = int64(len(value.buf))
c.coldTarget += e.size
if c.coldTarget > c.targetSize() {
c.coldTarget = c.targetSize()
}
atomic.StoreInt32(&e.referenced, 0)
e.setValue(value)
e.ptype = etHot
if c.metaAdd(k, e) {
value.ref.trace("add-hot")
c.sizeHot += e.size
c.countHot++
} else {
value.ref.trace("skip-hot")
e.free()
e = nil
}
}
c.checkConsistency()
// Values are initialized with a reference count of 1. That reference count
// is being transferred to the returned Handle.
return Handle{value: value}
}
func (c *shard) checkConsistency() {
// See the comment above the count{Hot,Cold,Test} fields.
switch {
case c.sizeHot < 0 || c.sizeCold < 0 || c.sizeTest < 0 || c.countHot < 0 || c.countCold < 0 || c.countTest < 0:
panic(fmt.Sprintf("pebble: unexpected negative: %d (%d bytes) hot, %d (%d bytes) cold, %d (%d bytes) test",
c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest))
case c.sizeHot > 0 && c.countHot == 0:
panic(fmt.Sprintf("pebble: mismatch %d hot size, %d hot count", c.sizeHot, c.countHot))
case c.sizeCold > 0 && c.countCold == 0:
panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
case c.sizeTest > 0 && c.countTest == 0:
panic(fmt.Sprintf("pebble: mismatch %d test size, %d test count", c.sizeTest, c.countTest))
}
}
// Delete deletes the cached value for the specified file and offset.
func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
// The common case is there is nothing to delete, so do a quick check with
// shared lock.
k := key{fileKey{id, fileNum}, offset}
c.mu.RLock()
exists := c.blocks.Get(k) != nil
c.mu.RUnlock()
if !exists {
return
}
c.mu.Lock()
defer c.mu.Unlock()
e := c.blocks.Get(k)
if e == nil {
return
}
c.metaEvict(e)
c.checkConsistency()
}
// EvictFile evicts all of the cache values for the specified file.
func (c *shard) EvictFile(id uint64, fileNum base.FileNum) {
c.mu.Lock()
defer c.mu.Unlock()
fkey := key{fileKey{id, fileNum}, 0}
blocks := c.files.Get(fkey)
if blocks == nil {
return
}
for b, n := blocks, (*entry)(nil); ; b = n {
n = b.fileLink.next
c.metaEvict(b)
if b == n {
break
}
}
c.checkConsistency()
}
func (c *shard) Free() {
c.mu.Lock()
defer c.mu.Unlock()
// NB: we use metaDel rather than metaEvict in order to avoid the expensive
// metaCheck call when the "invariants" build tag is specified.
for c.handHot != nil {
e := c.handHot
c.metaDel(c.handHot)
e.free()
}
c.blocks.free()
c.files.free()
}
func (c *shard) Reserve(n int) {
c.mu.Lock()
defer c.mu.Unlock()
c.reservedSize += int64(n)
// Changing c.reservedSize will either increase or decrease
// the targetSize. But we want coldTarget to be in the range
// [0, targetSize]. So, if c.targetSize decreases, make sure
// that the coldTarget fits within the limits.
targetSize := c.targetSize()
if c.coldTarget > targetSize {
c.coldTarget = targetSize
}
c.evict()
c.checkConsistency()
}
// Size returns the current space used by the cache.
func (c *shard) Size() int64 {
c.mu.RLock()
size := c.sizeHot + c.sizeCold
c.mu.RUnlock()
return size
}
func (c *shard) targetSize() int64 {
target := c.maxSize - c.reservedSize
// Always return a positive integer for targetSize. This is so that we don't
// end up in an infinite loop in evict(), in cases where reservedSize is
// greater than or equal to maxSize.
if target < 1 {
return 1
}
return target
}
// Add the entry to the cache, returning true if the entry was added and false
// if it would not fit in the cache.
func (c *shard) metaAdd(key key, e *entry) bool {
c.evict()
if e.size > c.targetSize() {
// The entry is larger than the target cache size.
return false
}
c.blocks.Put(key, e)
if entriesGoAllocated {
// Go allocated entries need to be referenced from Go memory. The entries
// map provides that reference.
c.entries[e] = struct{}{}
}
if c.handHot == nil {
// first element
c.handHot = e
c.handCold = e
c.handTest = e
} else {
c.handHot.link(e)
}
if c.handCold == c.handHot {
c.handCold = c.handCold.prev()
}
fkey := key.file()
if fileBlocks := c.files.Get(fkey); fileBlocks == nil {
c.files.Put(fkey, e)
} else {
fileBlocks.linkFile(e)
}
return true
}
// Remove the entry from the cache. This removes the entry from the blocks map,
// the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
// entry.
func (c *shard) metaDel(e *entry) {
if value := e.peekValue(); value != nil {
value.ref.trace("metaDel")
}
e.setValue(nil)
c.blocks.Delete(e.key)
if entriesGoAllocated {
// Go allocated entries need to be referenced from Go memory. The entries
// map provides that reference.
delete(c.entries, e)
}
if e == c.handHot {
c.handHot = c.handHot.prev()
}
if e == c.handCold {
c.handCold = c.handCold.prev()
}
if e == c.handTest {
c.handTest = c.handTest.prev()
}
if e.unlink() == e {
// This was the last entry in the cache.
c.handHot = nil
c.handCold = nil
c.handTest = nil
}
fkey := e.key.file()
if next := e.unlinkFile(); e == next {
c.files.Delete(fkey)
} else {
c.files.Put(fkey, next)
}
}
// Check that the specified entry is not referenced by the cache.
func (c *shard) metaCheck(e *entry) {
if invariants.Enabled {
if _, ok := c.entries[e]; ok {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in entries map\n%s",
e, e.key, debug.Stack())
os.Exit(1)
}
if c.blocks.findByValue(e) != nil {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks map\n%s\n%s",
e, e.key, &c.blocks, debug.Stack())
os.Exit(1)
}
if c.files.findByValue(e) != nil {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in files map\n%s\n%s",
e, e.key, &c.files, debug.Stack())
os.Exit(1)
}
// NB: c.hand{Hot,Cold,Test} are pointers into a single linked list. We
// only have to traverse one of them to check all of them.
var countHot, countCold, countTest int64
var sizeHot, sizeCold, sizeTest int64
for t := c.handHot.next(); t != nil; t = t.next() {
// Recompute count{Hot,Cold,Test} and size{Hot,Cold,Test}.
switch t.ptype {
case etHot:
countHot++
sizeHot += t.size
case etCold:
countCold++
sizeCold += t.size
case etTest:
countTest++
sizeTest += t.size
}
if e == t {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks list\n%s",
e, e.key, debug.Stack())
os.Exit(1)
}
if t == c.handHot {
break
}
}
if countHot != c.countHot || countCold != c.countCold || countTest != c.countTest ||
sizeHot != c.sizeHot || sizeCold != c.sizeCold || sizeTest != c.sizeTest {
fmt.Fprintf(os.Stderr, `divergence of Hot,Cold,Test statistics
cache's statistics: hot %d, %d, cold %d, %d, test %d, %d
recalculated statistics: hot %d, %d, cold %d, %d, test %d, %d\n%s`,
c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest,
countHot, sizeHot, countCold, sizeCold, countTest, sizeTest,
debug.Stack())
os.Exit(1)
}
}
}
func (c *shard) metaEvict(e *entry) {
switch e.ptype {
case etHot:
c.sizeHot -= e.size
c.countHot--
case etCold:
c.sizeCold -= e.size
c.countCold--
case etTest:
c.sizeTest -= e.size
c.countTest--
}
c.metaDel(e)
c.metaCheck(e)
e.free()
}
func (c *shard) evict() {
for c.targetSize() <= c.sizeHot+c.sizeCold && c.handCold != nil {
c.runHandCold(c.countCold, c.sizeCold)
}
}
func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
// countColdDebug and sizeColdDebug should equal c.countCold and
// c.sizeCold. They're parameters only to aid in debugging of
// cockroachdb/cockroach#70154. Since they're parameters, their
// arguments will appear within stack traces should we encounter
// a reproduction.
if c.countCold != countColdDebug || c.sizeCold != sizeColdDebug {
panic(fmt.Sprintf("runHandCold: cold count and size are %d, %d, arguments are %d and %d",
c.countCold, c.sizeCold, countColdDebug, sizeColdDebug))
}
e := c.handCold
if e.ptype == etCold {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
e.ptype = etHot
c.sizeCold -= e.size
c.countCold--
c.sizeHot += e.size
c.countHot++
} else {
e.setValue(nil)
e.ptype = etTest
c.sizeCold -= e.size
c.countCold--
c.sizeTest += e.size
c.countTest++
for c.targetSize() < c.sizeTest && c.handTest != nil {
c.runHandTest()
}
}
}
c.handCold = c.handCold.next()
for c.targetSize()-c.coldTarget <= c.sizeHot && c.handHot != nil {
c.runHandHot()
}
}
func (c *shard) runHandHot() {
if c.handHot == c.handTest && c.handTest != nil {
c.runHandTest()
if c.handHot == nil {
return
}
}
e := c.handHot
if e.ptype == etHot {
if atomic.LoadInt32(&e.referenced) == 1 {
atomic.StoreInt32(&e.referenced, 0)
} else {
e.ptype = etCold
c.sizeHot -= e.size
c.countHot--
c.sizeCold += e.size
c.countCold++
}
}
c.handHot = c.handHot.next()
}
func (c *shard) runHandTest() {
if c.sizeCold > 0 && c.handTest == c.handCold && c.handCold != nil {
// sizeCold is > 0, so assert that countCold == 0. See the
// comment above count{Hot,Cold,Test}.
if c.countCold == 0 {
panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
}
c.runHandCold(c.countCold, c.sizeCold)
if c.handTest == nil {
return
}
}
e := c.handTest
if e.ptype == etTest {
c.sizeTest -= e.size
c.countTest--
c.coldTarget -= e.size
if c.coldTarget < 0 {
c.coldTarget = 0
}
c.metaDel(e)
c.metaCheck(e)
e.free()
}
c.handTest = c.handTest.next()
}
// Metrics holds metrics for the cache.
type Metrics struct {
// The number of bytes inuse by the cache.
Size int64
// The count of objects (blocks or tables) in the cache.
Count int64
// The number of cache hits.
Hits int64
// The number of cache misses.
Misses int64
}
// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
// used for page replacement
// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
// order to provide better concurrency, 2 x NumCPUs shards are created, with
// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
// is run independently on each shard.
//
// Blocks are keyed by an (id, fileNum, offset) triple. The ID is a namespace
// for file numbers and allows a single Cache to be shared between multiple
// Pebble instances. The fileNum and offset refer to an sstable file number and
// the offset of the block within the file. Because sstables are immutable and
// file numbers are never reused, (fileNum,offset) are unique for the lifetime
// of a Pebble instance.
//
// In addition to maintaining a map from (fileNum,offset) to data, each shard
// maintains a map of the cached blocks for a particular fileNum. This allows
// efficient eviction of all of the blocks for a file which is used when an
// sstable is deleted from disk.
//
// Memory Management
//
// In order to reduce pressure on the Go GC, manual memory management is
// performed for the data stored in the cache. Manual memory management is
// performed by calling into C.{malloc,free} to allocate memory. Cache.Values
// are reference counted and the memory backing a manual value is freed when
// the reference count drops to 0.
//
// Manual memory management brings the possibility of memory leaks. It is
// imperative that every Handle returned by Cache.{Get,Set} is eventually
// released. The "invariants" build tag enables a leak detection facility that
// places a GC finalizer on cache.Value. When the cache.Value finalizer is run,
// if the underlying buffer is still present a leak has occurred. The "tracing"
// build tag enables tracing of cache.Value reference count manipulation and
// eases finding where a leak has occurred. These two facilities are usually
// used in combination by specifying `-tags invariants,tracing`. Note that
// "tracing" produces a significant slowdown, while "invariants" does not.
type Cache struct {
refs int64
maxSize int64
idAlloc uint64
shards []shard
// Traces recorded by Cache.trace. Used for debugging.
tr struct {
sync.Mutex
msgs []string
}
}
// New creates a new cache of the specified size. Memory for the cache is
// allocated on demand, not during initialization. The cache is created with a
// reference count of 1. Each DB it is associated with adds a reference, so the
// creator of the cache should usually release their reference after the DB is
// created.
//
// c := cache.New(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func New(size int64) *Cache {
return newShards(size, 2*runtime.GOMAXPROCS(0))
}
func newShards(size int64, shards int) *Cache {
c := &Cache{
refs: 1,
maxSize: size,
idAlloc: 1,
shards: make([]shard, shards),
}
c.trace("alloc", c.refs)
for i := range c.shards {
c.shards[i] = shard{
maxSize: size / int64(len(c.shards)),
coldTarget: size / int64(len(c.shards)),
}
if entriesGoAllocated {
c.shards[i].entries = make(map[*entry]struct{})
}
c.shards[i].blocks.init(16)
c.shards[i].files.init(16)
}
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(c, func(obj interface{}) {
c := obj.(*Cache)
if v := atomic.LoadInt64(&c.refs); v != 0 {
c.tr.Lock()
fmt.Fprintf(os.Stderr,
"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
if len(c.tr.msgs) > 0 {
fmt.Fprintf(os.Stderr, "%s\n", strings.Join(c.tr.msgs, "\n"))
}
c.tr.Unlock()
os.Exit(1)
}
})
return c
}
func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
// Inlined version of fnv.New64 + Write.
const offset64 = 14695981039346656037
const prime64 = 1099511628211
h := uint64(offset64)
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(id & 0xff)
id >>= 8
}
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(fileNum & 0xff)
fileNum >>= 8
}
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(offset & 0xff)
offset >>= 8
}
return &c.shards[h%uint64(len(c.shards))]
}
// Ref adds a reference to the cache. The cache only remains valid as long a
// reference is maintained to it.
func (c *Cache) Ref() {
v := atomic.AddInt64(&c.refs, 1)
if v <= 1 {
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
c.trace("ref", v)
}
// Unref releases a reference on the cache.
func (c *Cache) Unref() {
v := atomic.AddInt64(&c.refs, -1)
c.trace("unref", v)
switch {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
for i := range c.shards {
c.shards[i].Free()
}
}
}
// Get retrieves the cache value for the specified file and offset, returning
// nil if no value is present.
func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
}
// Set sets the cache value for the specified file and offset, overwriting an
// existing value if present. A Handle is returned which provides faster
// retrieval of the cached value than Get (lock-free and avoidance of the map
// lookup). The value must have been allocated by Cache.Alloc.
func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
}
// Delete deletes the cached value for the specified file and offset.
func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) {
c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
}
// EvictFile evicts all of the cache values for the specified file.
func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
for i := range c.shards {
c.shards[i].EvictFile(id, fileNum)
}
}
// MaxSize returns the max size of the cache.
func (c *Cache) MaxSize() int64 {
return c.maxSize
}
// Size returns the current space used by the cache.
func (c *Cache) Size() int64 {
var size int64
for i := range c.shards {
size += c.shards[i].Size()
}
return size
}
// Alloc allocates a byte slice of the specified size, possibly reusing
// previously allocated but unused memory. The memory backing the value is
// manually managed. The caller MUST either add the value to the cache (via
// Cache.Set), or release the value (via Cache.Free). Failure to do so will
// result in a memory leak.
func (c *Cache) Alloc(n int) *Value {
return newValue(n)
}
// Free frees the specified value. The buffer associated with the value will
// possibly be reused, making it invalid to use the buffer after calling
// Free. Do not call Free on a value that has been added to the cache.
func (c *Cache) Free(v *Value) {
if n := v.refs(); n > 1 {
panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
}
v.release()
}
// Reserve N bytes in the cache. This effectively shrinks the size of the cache
// by N bytes, without actually consuming any memory. The returned closure
// should be invoked to release the reservation.
func (c *Cache) Reserve(n int) func() {
// Round-up the per-shard reservation. Most reservations should be large, so
// this probably doesn't matter in practice.
shardN := (n + len(c.shards) - 1) / len(c.shards)
for i := range c.shards {
c.shards[i].Reserve(shardN)
}
return func() {
if shardN == -1 {
panic("pebble: cache reservation already released")
}
for i := range c.shards {
c.shards[i].Reserve(-shardN)
}
shardN = -1
}
}
// Metrics returns the metrics for the cache.
func (c *Cache) Metrics() Metrics {
var m Metrics
for i := range c.shards {
s := &c.shards[i]
s.mu.RLock()
m.Count += int64(s.blocks.Count())
m.Size += s.sizeHot + s.sizeCold
s.mu.RUnlock()
m.Hits += atomic.LoadInt64(&s.hits)
m.Misses += atomic.LoadInt64(&s.misses)
}
return m
}
// NewID returns a new ID to be used as a namespace for cached file
// blocks.
func (c *Cache) NewID() uint64 {
return atomic.AddUint64(&c.idAlloc, 1)
}

10
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_normal.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !tracing
// +build !tracing
package cache
func (c *Cache) trace(_ string, _ int64) {}

20
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build tracing
// +build tracing
package cache
import (
"fmt"
"runtime/debug"
)
func (c *Cache) trace(msg string, refs int64) {
s := fmt.Sprintf("%s: refs=%d\n%s", msg, refs, debug.Stack())
c.tr.Lock()
c.tr.msgs = append(c.tr.msgs, s)
c.tr.Unlock()
}

155
vendor/github.com/cockroachdb/pebble/internal/cache/entry.go generated vendored Normal file
View File

@@ -0,0 +1,155 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
type entryType int8
const (
etTest entryType = iota
etCold
etHot
)
func (p entryType) String() string {
switch p {
case etTest:
return "test"
case etCold:
return "cold"
case etHot:
return "hot"
}
return "unknown"
}
// entry holds the metadata for a cache entry. The memory for an entry is
// allocated from manually managed memory.
//
// Using manual memory management for entries is technically a volation of the
// Cgo pointer rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// Specifically, Go pointers should not be stored in C allocated memory. The
// reason for this rule is that the Go GC will not look at C allocated memory
// to find pointers to Go objects. If the only reference to a Go object is
// stored in C allocated memory, the object will be reclaimed. The shard field
// of the entry struct points to a Go allocated object, thus the
// violation. What makes this "safe" is that the Cache guarantees that there
// are other pointers to the shard which will keep it alive.
type entry struct {
key key
// The value associated with the entry. The entry holds a reference on the
// value which is maintained by entry.setValue().
val *Value
blockLink struct {
next *entry
prev *entry
}
fileLink struct {
next *entry
prev *entry
}
size int64
ptype entryType
// referenced is atomically set to indicate that this entry has been accessed
// since the last time one of the clock hands swept it.
referenced int32
shard *shard
// Reference count for the entry. The entry is freed when the reference count
// drops to zero.
ref refcnt
}
func newEntry(s *shard, key key, size int64) *entry {
e := entryAllocNew()
*e = entry{
key: key,
size: size,
ptype: etCold,
shard: s,
}
e.blockLink.next = e
e.blockLink.prev = e
e.fileLink.next = e
e.fileLink.prev = e
e.ref.init(1)
return e
}
func (e *entry) free() {
e.setValue(nil)
*e = entry{}
entryAllocFree(e)
}
func (e *entry) next() *entry {
if e == nil {
return nil
}
return e.blockLink.next
}
func (e *entry) prev() *entry {
if e == nil {
return nil
}
return e.blockLink.prev
}
func (e *entry) link(s *entry) {
s.blockLink.prev = e.blockLink.prev
s.blockLink.prev.blockLink.next = s
s.blockLink.next = e
s.blockLink.next.blockLink.prev = s
}
func (e *entry) unlink() *entry {
next := e.blockLink.next
e.blockLink.prev.blockLink.next = e.blockLink.next
e.blockLink.next.blockLink.prev = e.blockLink.prev
e.blockLink.prev = e
e.blockLink.next = e
return next
}
func (e *entry) linkFile(s *entry) {
s.fileLink.prev = e.fileLink.prev
s.fileLink.prev.fileLink.next = s
s.fileLink.next = e
s.fileLink.next.fileLink.prev = s
}
func (e *entry) unlinkFile() *entry {
next := e.fileLink.next
e.fileLink.prev.fileLink.next = e.fileLink.next
e.fileLink.next.fileLink.prev = e.fileLink.prev
e.fileLink.prev = e
e.fileLink.next = e
return next
}
func (e *entry) setValue(v *Value) {
if v != nil {
v.acquire()
}
old := e.val
e.val = v
if old != nil {
old.release()
}
}
func (e *entry) peekValue() *Value {
return e.val
}
func (e *entry) acquireValue() *Value {
v := e.val
if v != nil {
v.acquire()
}
return v
}

38
vendor/github.com/cockroachdb/pebble/internal/cache/entry_invariants.go generated vendored Normal file
View File

@@ -0,0 +1,38 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package cache
import (
"fmt"
"os"
"github.com/cockroachdb/pebble/internal/invariants"
)
// When the "invariants" or "tracing" build tags are enabled, we need to
// allocate entries using the Go allocator so entry.val properly maintains a
// reference to the Value.
const entriesGoAllocated = true
func entryAllocNew() *entry {
e := &entry{}
// Note: this is a no-op if invariants and tracing are disabled or race is
// enabled.
invariants.SetFinalizer(e, func(obj interface{}) {
e := obj.(*entry)
if v := e.ref.refs(); v != 0 {
fmt.Fprintf(os.Stderr, "%p: cache entry has non-zero reference count: %d\n%s",
e, v, e.ref.traces())
os.Exit(1)
}
})
return e
}
func entryAllocFree(e *entry) {
}

103
vendor/github.com/cockroachdb/pebble/internal/cache/entry_normal.go generated vendored Normal file
View File

@@ -0,0 +1,103 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package cache
import (
"runtime"
"sync"
"unsafe"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
const (
entrySize = int(unsafe.Sizeof(entry{}))
entryAllocCacheLimit = 128
// Avoid using runtime.SetFinalizer in race builds as finalizers tickle a bug
// in the Go race detector in go1.15 and earlier versions. This requires that
// entries are Go allocated rather than manually allocated.
//
// If cgo is disabled we need to allocate the entries using the Go allocator
// and is violates the Go GC rules to put Go pointers (such as the entry
// pointer fields) into untyped memory (i.e. a []byte).
entriesGoAllocated = invariants.RaceEnabled || !cgoEnabled
)
var entryAllocPool = sync.Pool{
New: func() interface{} {
return newEntryAllocCache()
},
}
func entryAllocNew() *entry {
a := entryAllocPool.Get().(*entryAllocCache)
e := a.alloc()
entryAllocPool.Put(a)
return e
}
func entryAllocFree(e *entry) {
a := entryAllocPool.Get().(*entryAllocCache)
a.free(e)
entryAllocPool.Put(a)
}
type entryAllocCache struct {
entries []*entry
}
func newEntryAllocCache() *entryAllocCache {
c := &entryAllocCache{}
if !entriesGoAllocated {
// Note the use of a "real" finalizer here (as opposed to a build tag-gated
// no-op finalizer). Without the finalizer, objects released from the pool
// and subsequently GC'd by the Go runtime would fail to have their manually
// allocated memory freed, which results in a memory leak.
// lint:ignore SetFinalizer
runtime.SetFinalizer(c, freeEntryAllocCache)
}
return c
}
func freeEntryAllocCache(obj interface{}) {
c := obj.(*entryAllocCache)
for i, e := range c.entries {
c.dealloc(e)
c.entries[i] = nil
}
}
func (c *entryAllocCache) alloc() *entry {
n := len(c.entries)
if n == 0 {
if entriesGoAllocated {
return &entry{}
}
b := manual.New(entrySize)
return (*entry)(unsafe.Pointer(&b[0]))
}
e := c.entries[n-1]
c.entries = c.entries[:n-1]
return e
}
func (c *entryAllocCache) dealloc(e *entry) {
if !entriesGoAllocated {
buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(e))[:entrySize:entrySize]
manual.Free(buf)
}
}
func (c *entryAllocCache) free(e *entry) {
if len(c.entries) == entryAllocCacheLimit {
c.dealloc(e)
return
}
c.entries = append(c.entries, e)
}

55
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_normal.go generated vendored Normal file
View File

@@ -0,0 +1,55 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !tracing
// +build !tracing
package cache
import (
"fmt"
"sync/atomic"
)
// refcnt provides an atomic reference count. This version is used when the
// "tracing" build tag is not enabled. See refcnt_tracing.go for the "tracing"
// enabled version.
type refcnt int32
// initialize the reference count to the specified value.
func (v *refcnt) init(val int32) {
*v = refcnt(val)
}
func (v *refcnt) refs() int32 {
return atomic.LoadInt32((*int32)(v))
}
func (v *refcnt) acquire() {
switch v := atomic.AddInt32((*int32)(v), 1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
}
func (v *refcnt) release() bool {
switch v := atomic.AddInt32((*int32)(v), -1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
return true
default:
return false
}
}
func (v *refcnt) trace(msg string) {
}
func (v *refcnt) traces() string {
return ""
}
// Silence unused warning.
var _ = (*refcnt)(nil).traces

66
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,66 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build tracing
// +build tracing
package cache
import (
"fmt"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
)
// refcnt provides an atomic reference count, along with a tracing facility for
// debugging logic errors in manipulating the reference count. This version is
// used when the "tracing" build tag is enabled.
type refcnt struct {
val int32
sync.Mutex
msgs []string
}
func (v *refcnt) init(val int32) {
v.val = val
v.trace("init")
}
func (v *refcnt) refs() int32 {
return atomic.LoadInt32(&v.val)
}
func (v *refcnt) acquire() {
switch n := atomic.AddInt32(&v.val, 1); {
case n <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
}
v.trace("acquire")
}
func (v *refcnt) release() bool {
n := atomic.AddInt32(&v.val, -1)
switch {
case n < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
}
v.trace("release")
return n == 0
}
func (v *refcnt) trace(msg string) {
s := fmt.Sprintf("%s: refs=%d\n%s", msg, v.refs(), debug.Stack())
v.Lock()
v.msgs = append(v.msgs, s)
v.Unlock()
}
func (v *refcnt) traces() string {
v.Lock()
s := strings.Join(v.msgs, "\n")
v.Unlock()
return s
}

320
vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go generated vendored Normal file
View File

@@ -0,0 +1,320 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
import (
"fmt"
"math/bits"
"os"
"runtime/debug"
"strings"
"time"
"unsafe"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
var hashSeed = uint64(time.Now().UnixNano())
// Fibonacci hash: https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/
func robinHoodHash(k key, shift uint32) uint32 {
const m = 11400714819323198485
h := hashSeed
h ^= k.id * m
h ^= uint64(k.fileNum) * m
h ^= k.offset * m
return uint32(h >> shift)
}
type robinHoodEntry struct {
key key
// Note that value may point to a Go allocated object (if the "invariants"
// build tag was specified), even though the memory for the entry itself is
// manually managed. This is technically a volation of the Cgo pointer rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// Specifically, Go pointers should not be stored in C allocated memory. The
// reason for this rule is that the Go GC will not look at C allocated memory
// to find pointers to Go objects. If the only reference to a Go object is
// stored in C allocated memory, the object will be reclaimed. What makes
// this "safe" is that the Cache guarantees that there are other pointers to
// the entry and shard which will keep them alive. In particular, every Go
// allocated entry in the cache is referenced by the shard.entries map. And
// every shard is referenced by the Cache.shards map.
value *entry
// The distance the entry is from its desired position.
dist uint32
}
type robinHoodEntries struct {
ptr unsafe.Pointer
len uint32
}
func newRobinHoodEntries(n uint32) robinHoodEntries {
size := uintptr(n) * unsafe.Sizeof(robinHoodEntry{})
return robinHoodEntries{
ptr: unsafe.Pointer(&(manual.New(int(size)))[0]),
len: n,
}
}
func (e robinHoodEntries) at(i uint32) *robinHoodEntry {
return (*robinHoodEntry)(unsafe.Pointer(uintptr(e.ptr) +
uintptr(i)*unsafe.Sizeof(robinHoodEntry{})))
}
func (e robinHoodEntries) free() {
size := uintptr(e.len) * unsafe.Sizeof(robinHoodEntry{})
buf := (*[manual.MaxArrayLen]byte)(e.ptr)[:size:size]
manual.Free(buf)
}
// robinHoodMap is an implementation of Robin Hood hashing. Robin Hood hashing
// is an open-address hash table using linear probing. The twist is that the
// linear probe distance is reduced by moving existing entries when inserting
// and deleting. This is accomplished by keeping track of how far an entry is
// from its "desired" slot (hash of key modulo number of slots). During
// insertion, if the new entry being inserted is farther from its desired slot
// than the target entry, we swap the target and new entry. This effectively
// steals from the "rich" target entry and gives to the "poor" new entry (thus
// the origin of the name).
//
// An extension over the base Robin Hood hashing idea comes from
// https://probablydance.com/2017/02/26/i-wrote-the-fastest-hashtable/. A cap
// is placed on the max distance an entry can be from its desired slot. When
// this threshold is reached during insertion, the size of the table is doubled
// and insertion is restarted. Additionally, the entries slice is given "max
// dist" extra entries on the end. The very last entry in the entries slice is
// never used and acts as a sentinel which terminates loops. The previous
// maxDist-1 entries act as the extra entries. For example, if the size of the
// table is 2, maxDist is computed as 4 and the actual size of the entry slice
// is 6.
//
// +---+---+---+---+---+---+
// | 0 | 1 | 2 | 3 | 4 | 5 |
// +---+---+---+---+---+---+
// ^
// size
//
// In this scenario, the target entry for a key will always be in the range
// [0,1]. Valid entries may reside in the range [0,4] due to the linear probing
// of up to maxDist entries. The entry at index 5 will never contain a value,
// and instead acts as a sentinel (its distance is always 0). The max distance
// threshold is set to log2(num-entries). This ensures that retrieval is O(log
// N), though note that N is the number of total entries, not the count of
// valid entries.
//
// Deletion is implemented via the backward shift delete mechanism instead of
// tombstones. This preserves the performance of the table in the presence of
// deletions. See
// http://codecapsule.com/2013/11/17/robin-hood-hashing-backward-shift-deletion
// for details.
type robinHoodMap struct {
entries robinHoodEntries
size uint32
shift uint32
count uint32
maxDist uint32
}
func maxDistForSize(size uint32) uint32 {
desired := uint32(bits.Len32(size))
if desired < 4 {
desired = 4
}
return desired
}
func newRobinHoodMap(initialCapacity int) *robinHoodMap {
m := &robinHoodMap{}
m.init(initialCapacity)
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(m, func(obj interface{}) {
m := obj.(*robinHoodMap)
if m.entries.ptr != nil {
fmt.Fprintf(os.Stderr, "%p: robin-hood map not freed\n", m)
os.Exit(1)
}
})
return m
}
func (m *robinHoodMap) init(initialCapacity int) {
if initialCapacity < 1 {
initialCapacity = 1
}
targetSize := 1 << (uint(bits.Len(uint(2*initialCapacity-1))) - 1)
m.rehash(uint32(targetSize))
}
func (m *robinHoodMap) free() {
if m.entries.ptr != nil {
m.entries.free()
m.entries.ptr = nil
}
}
func (m *robinHoodMap) rehash(size uint32) {
oldEntries := m.entries
m.size = size
m.shift = uint32(64 - bits.Len32(m.size-1))
m.maxDist = maxDistForSize(size)
m.entries = newRobinHoodEntries(size + m.maxDist)
m.count = 0
for i := uint32(0); i < oldEntries.len; i++ {
e := oldEntries.at(i)
if e.value != nil {
m.Put(e.key, e.value)
}
}
if oldEntries.ptr != nil {
oldEntries.free()
}
}
// Find an entry containing the specified value. This is intended to be used
// from debug and test code.
func (m *robinHoodMap) findByValue(v *entry) *robinHoodEntry {
for i := uint32(0); i < m.entries.len; i++ {
e := m.entries.at(i)
if e.value == v {
return e
}
}
return nil
}
func (m *robinHoodMap) Count() int {
return int(m.count)
}
func (m *robinHoodMap) Put(k key, v *entry) {
maybeExists := true
n := robinHoodEntry{key: k, value: v, dist: 0}
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if maybeExists && k == e.key {
// Entry already exists: overwrite.
e.value = n.value
m.checkEntry(i)
return
}
if e.value == nil {
// Found an empty entry: insert here.
*e = n
m.count++
m.checkEntry(i)
return
}
if e.dist < n.dist {
// Swap the new entry with the current entry because the current is
// rich. We then continue to loop, looking for a new location for the
// current entry. Note that this is also the not-found condition for
// retrieval, which means that "k" is not present in the map. See Get().
n, *e = *e, n
m.checkEntry(i)
maybeExists = false
}
// The new entry gradually moves away from its ideal position.
n.dist++
// If we've reached the max distance threshold, grow the table and restart
// the insertion.
if n.dist == m.maxDist {
m.rehash(2 * m.size)
i = robinHoodHash(n.key, m.shift) - 1
n.dist = 0
maybeExists = false
}
}
}
func (m *robinHoodMap) Get(k key) *entry {
var dist uint32
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if k == e.key {
// Found.
return e.value
}
if e.dist < dist {
// Not found.
return nil
}
dist++
}
}
func (m *robinHoodMap) Delete(k key) {
var dist uint32
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if k == e.key {
m.checkEntry(i)
// We found the entry to delete. Shift the following entries backwards
// until the next empty value or entry with a zero distance. Note that
// empty values are guaranteed to have "dist == 0".
m.count--
for j := i + 1; ; j++ {
t := m.entries.at(j)
if t.dist == 0 {
*e = robinHoodEntry{}
return
}
e.key = t.key
e.value = t.value
e.dist = t.dist - 1
e = t
m.checkEntry(j)
}
}
if dist > e.dist {
// Not found.
return
}
dist++
}
}
func (m *robinHoodMap) checkEntry(i uint32) {
if invariants.Enabled {
e := m.entries.at(i)
if e.value != nil {
pos := robinHoodHash(e.key, m.shift)
if (uint32(i) - pos) != e.dist {
fmt.Fprintf(os.Stderr, "%d: invalid dist=%d, expected %d: %s\n%s",
i, e.dist, uint32(i)-pos, e.key, debug.Stack())
os.Exit(1)
}
if e.dist > m.maxDist {
fmt.Fprintf(os.Stderr, "%d: invalid dist=%d > maxDist=%d: %s\n%s",
i, e.dist, m.maxDist, e.key, debug.Stack())
os.Exit(1)
}
}
}
}
func (m *robinHoodMap) String() string {
var buf strings.Builder
fmt.Fprintf(&buf, "count: %d\n", m.count)
for i := uint32(0); i < m.entries.len; i++ {
e := m.entries.at(i)
if e.value != nil {
fmt.Fprintf(&buf, "%d: [%s,%p,%d]\n", i, e.key, e.value, e.dist)
}
}
return buf.String()
}

46
vendor/github.com/cockroachdb/pebble/internal/cache/value.go generated vendored Normal file
View File

@@ -0,0 +1,46 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
// Value holds a reference counted immutable value.
type Value struct {
buf []byte
// Reference count for the value. The value is freed when the reference count
// drops to zero.
ref refcnt
}
// Buf returns the buffer associated with the value. The contents of the buffer
// should not be changed once the value has been added to the cache. Instead, a
// new Value should be created and added to the cache to replace the existing
// value.
func (v *Value) Buf() []byte {
if v == nil {
return nil
}
return v.buf
}
// Truncate the buffer to the specified length. The buffer length should not be
// changed once the value has been added to the cache as there may be
// concurrent readers of the Value. Instead, a new Value should be created and
// added to the cache to replace the existing value.
func (v *Value) Truncate(n int) {
v.buf = v.buf[:n]
}
func (v *Value) refs() int32 {
return v.ref.refs()
}
func (v *Value) acquire() {
v.ref.acquire()
}
func (v *Value) release() {
if v.ref.release() {
v.free()
}
}

55
vendor/github.com/cockroachdb/pebble/internal/cache/value_invariants.go generated vendored Normal file
View File

@@ -0,0 +1,55 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package cache
import (
"fmt"
"os"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
// newValue creates a Value with a manually managed buffer of size n.
//
// This definition of newValue is used when either the "invariants" or
// "tracing" build tags are specified. It hooks up a finalizer to the returned
// Value that checks for memory leaks when the GC determines the Value is no
// longer reachable.
func newValue(n int) *Value {
if n == 0 {
return nil
}
b := manual.New(n)
v := &Value{buf: b}
v.ref.init(1)
// Note: this is a no-op if invariants and tracing are disabled or race is
// enabled.
invariants.SetFinalizer(v, func(obj interface{}) {
v := obj.(*Value)
if v.buf != nil {
fmt.Fprintf(os.Stderr, "%p: cache value was not freed: refs=%d\n%s",
v, v.refs(), v.ref.traces())
os.Exit(1)
}
})
return v
}
func (v *Value) free() {
// When "invariants" are enabled set the value contents to 0xff in order to
// cache use-after-free bugs.
for i := range v.buf {
v.buf[i] = 0xff
}
manual.Free(v.buf)
// Setting Value.buf to nil is needed for correctness of the leak checking
// that is performed when the "invariants" or "tracing" build tags are
// enabled.
v.buf = nil
}

57
vendor/github.com/cockroachdb/pebble/internal/cache/value_normal.go generated vendored Normal file
View File

@@ -0,0 +1,57 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package cache
import (
"unsafe"
"github.com/cockroachdb/pebble/internal/manual"
)
const valueSize = int(unsafe.Sizeof(Value{}))
func newValue(n int) *Value {
if n == 0 {
return nil
}
if !cgoEnabled {
// If Cgo is disabled then all memory is allocated from the Go heap and we
// can't play the trick below to combine the Value and buffer allocation.
v := &Value{buf: make([]byte, n)}
v.ref.init(1)
return v
}
// When we're not performing leak detection, the lifetime of the returned
// Value is exactly the lifetime of the backing buffer and we can manually
// allocate both.
//
// TODO(peter): It may be better to separate the allocation of the value and
// the buffer in order to reduce internal fragmentation in malloc. If the
// buffer is right at a power of 2, adding valueSize might push the
// allocation over into the next larger size.
b := manual.New(valueSize + n)
v := (*Value)(unsafe.Pointer(&b[0]))
v.buf = b[valueSize:]
v.ref.init(1)
return v
}
func (v *Value) free() {
if !cgoEnabled {
return
}
// When we're not performing leak detection, the Value and buffer were
// allocated contiguously.
n := valueSize + cap(v.buf)
buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(v))[:n:n]
v.buf = nil
manual.Free(buf)
}

40
vendor/github.com/cockroachdb/pebble/internal/crc/crc.go generated vendored Normal file
View File

@@ -0,0 +1,40 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package crc implements the checksum algorithm used throughout pebble.
//
// The algorithm is CRC-32 with Castagnoli's polynomial, followed by a bit
// rotation and an additional delta. The additional processing is to lessen the
// probability of arbitrary key/value data coincidentally containing bytes that
// look like a checksum.
//
// To calculate the uint32 checksum of some data:
// var u uint32 = crc.New(data).Value()
// In pebble, the uint32 value is then stored in little-endian format.
package crc // import "github.com/cockroachdb/pebble/internal/crc"
import "hash/crc32"
var table = crc32.MakeTable(crc32.Castagnoli)
// CRC is a small convenience wrapper for computing the CRC32 checksum used by
// pebble. This is the same algorithm as used by RocksDB.
type CRC uint32
// New returns the result of adding the bytes to the zero-value CRC.
func New(b []byte) CRC {
return CRC(0).Update(b)
}
// Update returns the result of adding the bytes to the CRC.
func (c CRC) Update(b []byte) CRC {
return CRC(crc32.Update(uint32(c), table, b))
}
// Value returns the cooked CRC value. The additional processing is to lessen
// the probability of arbitrary key/value data coincidentally containing bytes
// that look like a checksum.
func (c CRC) Value() uint32 {
return uint32(c>>15|c<<17) + 0xa282ead8
}

15
vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go generated vendored Normal file
View File

@@ -0,0 +1,15 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package fastrand
import _ "unsafe" // required by go:linkname
// Uint32 returns a lock free uint32 value.
//go:linkname Uint32 runtime.fastrand
func Uint32() uint32
// Uint32n returns a lock free uint32 value in the interval [0, n).
//go:linkname Uint32n runtime.fastrandn
func Uint32n(n uint32) uint32

80
vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go generated vendored Normal file
View File

@@ -0,0 +1,80 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package humanize
import (
"fmt"
"math"
"github.com/cockroachdb/redact"
)
func logn(n, b float64) float64 {
return math.Log(n) / math.Log(b)
}
func humanate(s uint64, base float64, suffixes []string) string {
if s < 10 {
return fmt.Sprintf("%d%s", s, suffixes[0])
}
e := math.Floor(logn(float64(s), base))
suffix := suffixes[int(e)]
val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10
f := "%.0f%s"
if val < 10 {
f = "%.1f%s"
}
return fmt.Sprintf(f, val, suffix)
}
type config struct {
base float64
suffix []string
}
// IEC produces human readable representations of integer values in IEC units.
var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}}
// SI produces human readable representations of integer values in SI units.
var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}}
// Int64 produces a human readable representation of the value.
func (c *config) Int64(s int64) FormattedString {
if s < 0 {
return FormattedString("-" + humanate(uint64(-s), c.base, c.suffix))
}
return FormattedString(humanate(uint64(s), c.base, c.suffix))
}
// Uint64 produces a human readable representation of the value.
func (c *config) Uint64(s uint64) FormattedString {
return FormattedString(humanate(s, c.base, c.suffix))
}
// Int64 produces a human readable representation of the value in IEC units
// (base 1024).
func Int64(s int64) FormattedString {
return IEC.Int64(s)
}
// Uint64 produces a human readable representation of the value in IEC units
// (base 1024).
func Uint64(s uint64) FormattedString {
return IEC.Uint64(s)
}
// FormattedString represents a human readable representation of a value. It
// implements the redact.SafeValue interface to signal that it represents a
// a string that does not need to be redacted.
type FormattedString string
var _ redact.SafeValue = FormattedString("")
// SafeValue implements redact.SafeValue.
func (fs FormattedString) SafeValue() {}
// String implements fmt.Stringer.
func (fs FormattedString) String() string { return string(fs) }

27
vendor/github.com/cockroachdb/pebble/internal/intern/intern.go generated vendored Normal file
View File

@@ -0,0 +1,27 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package intern
import "sync"
var pool = sync.Pool{
New: func() interface{} {
return make(map[string]string)
},
}
// Bytes returns b converted to a string, interned.
func Bytes(b []byte) string {
m := pool.Get().(map[string]string)
c, ok := m[string(b)]
if ok {
pool.Put(m)
return c
}
s := string(b)
m[s] = s
pool.Put(m)
return s
}

14
vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_off.go generated vendored Normal file
View File

@@ -0,0 +1,14 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package invariants
// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
// race builds or if neither the invariants or tracing build tags are
// specified.
func SetFinalizer(obj, finalizer interface{}) {
}

17
vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_on.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package invariants
import "runtime"
// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
// race builds or if neither the invariants or tracing build tags are
// specified.
func SetFinalizer(obj, finalizer interface{}) {
runtime.SetFinalizer(obj, finalizer)
}

11
vendor/github.com/cockroachdb/pebble/internal/invariants/off.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !invariants && !race
// +build !invariants,!race
package invariants
// Enabled is true if we were built with the "invariants" or "race" build tags.
const Enabled = false

11
vendor/github.com/cockroachdb/pebble/internal/invariants/on.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build invariants || race
// +build invariants race
package invariants
// Enabled is true if we were built with the "invariants" or "race" build tags.
const Enabled = true

11
vendor/github.com/cockroachdb/pebble/internal/invariants/race_off.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !race
// +build !race
package invariants
// RaceEnabled is true if we were built with the "race" build tag.
const RaceEnabled = false

11
vendor/github.com/cockroachdb/pebble/internal/invariants/race_on.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build race
// +build race
package invariants
// RaceEnabled is true if we were built with the "race" build tag.
const RaceEnabled = true

416
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored Normal file
View File

@@ -0,0 +1,416 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"bytes"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
// that DefragmentingIter will reuse. Buffers greater than this will be
// discarded and reallocated as necessary.
const bufferReuseMaxCapacity = 10 << 10 // 10 KB
// DefragmentMethod configures the defragmentation performed by the
// DefragmentingIter.
type DefragmentMethod func(base.Compare, Span, Span) bool
// DefragmentInternal configures a DefragmentingIter to defragment spans
// only if they have identical keys.
//
// This defragmenting method is intended for use in compactions that may see
// internal range keys fragments that may now be joined, because the state that
// required their fragmentation has been dropped.
var DefragmentInternal DefragmentMethod = func(cmp base.Compare, a, b Span) bool {
if len(a.Keys) != len(b.Keys) {
return false
}
for i := range a.Keys {
if a.Keys[i].Trailer != b.Keys[i].Trailer {
return false
}
if cmp(a.Keys[i].Suffix, b.Keys[i].Suffix) != 0 {
return false
}
if !bytes.Equal(a.Keys[i].Value, b.Keys[i].Value) {
return false
}
}
return true
}
// DefragmentReducer merges the current and next Key slices, returning a new Key
// slice.
//
// Implementations should modify and return `cur` to save on allocations, or
// consider allocating a new slice, as the `cur` slice may be retained by the
// DefragmentingIter and mutated. The `next` slice must not be mutated.
//
// The incoming slices are sorted by (SeqNum, Kind) descending. The output slice
// must also have this sort order.
type DefragmentReducer func(cur, next []Key) []Key
// StaticDefragmentReducer is a no-op DefragmentReducer that simply returns the
// current key slice, effectively retaining the first set of keys encountered
// for a defragmented span.
//
// This reducer can be used, for example, when the set of Keys for each Span
// being reduced is not expected to change, and therefore the keys from the
// first span encountered can be used without considering keys in subsequent
// spans.
var StaticDefragmentReducer DefragmentReducer = func(cur, _ []Key) []Key {
return cur
}
// iterPos is an enum indicating the position of the defragmenting iter's
// wrapped iter. The defragmenting iter must look ahead or behind when
// defragmenting forward or backwards respectively, and this enum records that
// current position.
type iterPos int8
const (
iterPosPrev iterPos = -1
iterPosCurr iterPos = 0
iterPosNext iterPos = +1
)
// DefragmentingIter wraps a key span iterator, defragmenting physical
// fragmentation during iteration.
//
// During flushes and compactions, keys applied over a span may be split at
// sstable boundaries. This fragmentation can produce internal key bounds that
// do not match any of the bounds ever supplied to a user operation. This
// physical fragmentation is necessary to avoid excessively wide sstables.
//
// The defragmenting iterator undoes this physical fragmentation, joining spans
// with abutting bounds and equal state. The defragmenting iterator takes a
// DefragmentMethod to determine what is "equal state" for a span. The
// DefragmentMethod is a function type, allowing arbitrary comparisons between
// Span keys.
//
// Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may
// land on a physical fragment in the middle of several fragments that must be
// defragmented. A seek first degfragments in the opposite direction of
// iteration to find the beginning of the defragmented span, and then
// defragments in the iteration direction, ensuring it's found a whole
// defragmented span.
type DefragmentingIter struct {
cmp base.Compare
iter FragmentIterator
iterSpan Span
iterPos iterPos
// curr holds the span at the current iterator position. currBuf is a buffer
// for use when copying user keys for curr. keysBuf is a buffer for use when
// copying Keys for curr. currBuf is cleared between positioning methods.
//
// keyBuf is a buffer specifically for the defragmented start key when
// defragmenting backwards or the defragmented end key when defragmenting
// forwards. These bounds are overwritten repeatedly during defragmentation,
// and the defragmentation routines overwrite keyBuf repeatedly to store
// these extended bounds.
curr Span
currBuf []byte
keysBuf []Key
keyBuf []byte
// equal is a comparison function for two spans. equal is called when two
// spans are abutting to determine whether they may be defragmented.
// equal does not itself check for adjacency for the two spans.
equal DefragmentMethod
// reduce is the reducer function used to collect Keys across all spans that
// constitute a defragmented span.
reduce DefragmentReducer
}
// Assert that *DefragmentingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*DefragmentingIter)(nil)
// Init initializes the defragmenting iter using the provided defragment
// method.
func (i *DefragmentingIter) Init(
cmp base.Compare, iter FragmentIterator, equal DefragmentMethod, reducer DefragmentReducer,
) {
*i = DefragmentingIter{
cmp: cmp,
iter: iter,
equal: equal,
reduce: reducer,
}
}
// Error returns any accumulated error.
func (i *DefragmentingIter) Error() error {
return i.iter.Error()
}
// Close closes the underlying iterators.
func (i *DefragmentingIter) Close() error {
return i.iter.Close()
}
// SeekGE seeks the iterator to the first span with a start key greater than or
// equal to key and returns it.
func (i *DefragmentingIter) SeekGE(key []byte) Span {
i.iterSpan = i.iter.SeekGE(key)
if !i.iterSpan.Valid() {
i.iterPos = iterPosCurr
return i.iterSpan
}
// Save the current span and peek backwards.
i.saveCurrent(i.iterSpan)
i.iterSpan = i.iter.Prev()
if i.cmp(i.curr.Start, i.iterSpan.End) == 0 && i.checkEqual(i.iterSpan, i.curr) {
// A continuation. The span we originally landed on and defragmented
// backwards has a true Start key < key. To obey the FragmentIterator
// contract, we must not return this defragmented span. Defragment
// forward to finish defragmenting the span in the forward direction.
i.defragmentForward()
// Now we must be on a span that truly has a defragmented Start key >
// key.
return i.defragmentForward()
}
// The span previous to i.curr does not defragment, so we should return it.
// Next the underlying iterator back onto the span we previously saved to
// i.curr and then defragment forward.
i.iterSpan = i.iter.Next()
return i.defragmentForward()
}
// SeekLT seeks the iterator to the last span with a start key less than
// key and returns it.
func (i *DefragmentingIter) SeekLT(key []byte) Span {
i.iterSpan = i.iter.SeekLT(key)
// Defragment forward to find the end of the defragmented span.
i.defragmentForward()
if i.iterPos == iterPosNext {
// Prev once back onto the span.
i.iterSpan = i.iter.Prev()
}
// Defragment the full span from its end.
return i.defragmentBackward()
}
// First seeks the iterator to the first span and returns it.
func (i *DefragmentingIter) First() Span {
i.iterSpan = i.iter.First()
return i.defragmentForward()
}
// Last seeks the iterator to the last span and returns it.
func (i *DefragmentingIter) Last() Span {
i.iterSpan = i.iter.Last()
return i.defragmentBackward()
}
// Next advances to the next span and returns it.
func (i *DefragmentingIter) Next() Span {
switch i.iterPos {
case iterPosPrev:
// Switching directions; The iterator is currently positioned over the
// last span of the previous set of fragments. In the below diagram,
// the iterator is positioned over the last span that contributes to
// the defragmented x position. We want to be positioned over the first
// span that contributes to the z position.
//
// x x x y y y z z z
// ^ ^
// old new
//
// Next once to move onto y, defragment forward to land on the first z
// position.
i.iterSpan = i.iter.Next()
if !i.iterSpan.Valid() {
panic("pebble: invariant violation: no next span while switching directions")
}
// We're now positioned on the first span that was defragmented into the
// current iterator position. Skip over the rest of the current iterator
// position's constitutent fragments. In the above example, this would
// land on the first 'z'.
i.defragmentForward()
// Now that we're positioned over the first of the next set of
// fragments, defragment forward.
return i.defragmentForward()
case iterPosCurr:
// iterPosCurr is only used when the iter is exhausted or when the iterator
// is at an empty span.
if invariants.Enabled && i.iterSpan.Valid() && !i.iterSpan.Empty() {
panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
}
i.iterSpan = i.iter.Next()
return i.defragmentForward()
case iterPosNext:
// Already at the next span.
return i.defragmentForward()
default:
panic("unreachable")
}
}
// Prev steps back to the previous span and returns it.
func (i *DefragmentingIter) Prev() Span {
switch i.iterPos {
case iterPosPrev:
// Already at the previous span.
return i.defragmentBackward()
case iterPosCurr:
// iterPosCurr is only used when the iter is exhausted or when the iterator
// is at an empty span.
if invariants.Enabled && i.iterSpan.Valid() && !i.iterSpan.Empty() {
panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
}
i.iterSpan = i.iter.Prev()
return i.defragmentBackward()
case iterPosNext:
// Switching directions; The iterator is currently positioned over the
// first fragment of the next set of fragments. In the below diagram,
// the iterator is positioned over the first span that contributes to
// the defragmented z position. We want to be positioned over the last
// span that contributes to the x position.
//
// x x x y y y z z z
// ^ ^
// new old
//
// Prev once to move onto y, defragment backward to land on the last x
// position.
i.iterSpan = i.iter.Prev()
if !i.iterSpan.Valid() {
panic("pebble: invariant violation: no previous span while switching directions")
}
// We're now positioned on the last span that was defragmented into the
// current iterator position. Skip over the rest of the current iterator
// position's constitutent fragments. In the above example, this would
// land on the last 'x'.
i.defragmentBackward()
// Now that we're positioned over the last of the prev set of
// fragments, defragment backward.
return i.defragmentBackward()
default:
panic("unreachable")
}
}
// checkEqual checks the two spans for logical equivalence. It uses the passed-in
// DefragmentMethod and ensures both spans are NOT empty; not defragmenting empty
// spans is an optimization that lets us load fewer sstable blocks.
func (i *DefragmentingIter) checkEqual(left, right Span) bool {
return i.equal(i.cmp, i.iterSpan, i.curr) && !(left.Empty() && right.Empty())
}
// defragmentForward defragments spans in the forward direction, starting from
// i.iter's current position.
func (i *DefragmentingIter) defragmentForward() Span {
if !i.iterSpan.Valid() {
i.iterPos = iterPosCurr
return i.iterSpan
}
if i.iterSpan.Empty() {
// An empty span will never be equal to another span; see checkEqual for
// why. To avoid loading non-empty range keys further ahead by calling Next,
// return early.
i.iterPos = iterPosCurr
return i.iterSpan
}
i.saveCurrent(i.iterSpan)
i.iterPos = iterPosNext
i.iterSpan = i.iter.Next()
for i.iterSpan.Valid() {
if i.cmp(i.curr.End, i.iterSpan.Start) != 0 {
// Not a continuation.
break
}
if !i.checkEqual(i.iterSpan, i.curr) {
// Not a continuation.
break
}
i.keyBuf = append(i.keyBuf[:0], i.iterSpan.End...)
i.curr.End = i.keyBuf
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Next()
}
i.curr.Keys = i.keysBuf
return i.curr
}
// defragmentBackward defragments spans in the backward direction, starting from
// i.iter's current position.
func (i *DefragmentingIter) defragmentBackward() Span {
if !i.iterSpan.Valid() {
i.iterPos = iterPosCurr
return i.iterSpan
}
if i.iterSpan.Empty() {
// An empty span will never be equal to another span; see checkEqual for
// why. To avoid loading non-empty range keys further ahead by calling Next,
// return early.
i.iterPos = iterPosCurr
return i.iterSpan
}
i.saveCurrent(i.iterSpan)
i.iterPos = iterPosPrev
i.iterSpan = i.iter.Prev()
for i.iterSpan.Valid() {
if i.cmp(i.curr.Start, i.iterSpan.End) != 0 {
// Not a continuation.
break
}
if !i.checkEqual(i.iterSpan, i.curr) {
// Not a continuation.
break
}
i.keyBuf = append(i.keyBuf[:0], i.iterSpan.Start...)
i.curr.Start = i.keyBuf
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Prev()
}
i.curr.Keys = i.keysBuf
return i.curr
}
func (i *DefragmentingIter) saveCurrent(span Span) {
i.currBuf = i.currBuf[:0]
i.keysBuf = i.keysBuf[:0]
i.keyBuf = i.keyBuf[:0]
if cap(i.currBuf) > bufferReuseMaxCapacity {
i.currBuf = nil
}
if cap(i.keyBuf) > bufferReuseMaxCapacity {
i.keyBuf = nil
}
i.curr = Span{
Start: i.saveBytes(span.Start),
End: i.saveBytes(span.End),
}
for j := range span.Keys {
i.keysBuf = append(i.keysBuf, Key{
Trailer: span.Keys[j].Trailer,
Suffix: i.saveBytes(span.Keys[j].Suffix),
Value: i.saveBytes(span.Keys[j].Value),
})
}
i.curr.Keys = i.keysBuf
}
func (i *DefragmentingIter) saveBytes(b []byte) []byte {
if b == nil {
return nil
}
ret := append(i.currBuf, b...)
i.currBuf = ret[len(ret):]
return ret
}

13
vendor/github.com/cockroachdb/pebble/internal/keyspan/doc.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Package keyspan provides facilities for sorting, fragmenting and
// iterating over spans of user keys.
//
// A Span represents a range of user key space with an inclusive start
// key and exclusive end key. A span may hold any number of Keys which are
// applied over the entirety of the span's keyspace.
//
// Spans are used within Pebble as an in-memory representation of range
// deletion tombstones, and range key sets, unsets and deletes. Spans
// are fragmented at overlapping key boundaries by the Fragmenter type.
// This package's various iteration facilities require these
// non-overlapping fragmented spans.
package keyspan

98
vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go generated vendored Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
// FilterFunc defines a transform from the input Span into the output Span. The
// function returns true if the Span should be returned by the iterator, and
// false if the Span should be skipped. The FilterFunc is permitted to mutate
// the output Span, for example, to elice certain keys, or update the Span's
// bounds if so desired. The output Span's Keys slice may be reused to reduce
// allocations.
type FilterFunc func(in Span, out *Span) (keep bool)
// filteringIter is a FragmentIterator that uses a FilterFunc to select which
// Spans from the input iterator are returned in the output.
//
// A note on Span lifetimes: as the FilterFunc reuses a Span with a mutable
// slice of Keys to reduce allocations, Spans returned by this iterator are only
// valid until the next relative or absolute positioning method is called.
type filteringIter struct {
iter FragmentIterator
filterFn FilterFunc
// span is a mutable Span passed to the filterFn. The filterFn is free to
// mutate this Span. The slice of Keys in the Span is reused with every call
// to the filterFn.
span Span
}
var _ FragmentIterator = (*filteringIter)(nil)
// Filter returns a new filteringIter that will filter the Spans from the
// provided child iterator using the provided FilterFunc.
func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter}
}
// SeekGE implements FragmentIterator.
func (i *filteringIter) SeekGE(key []byte) Span {
return i.filter(i.iter.SeekGE(key), +1)
}
// SeekLT implements FragmentIterator.
func (i *filteringIter) SeekLT(key []byte) Span {
return i.filter(i.iter.SeekLT(key), -1)
}
// First implements FragmentIterator.
func (i *filteringIter) First() Span {
return i.filter(i.iter.First(), +1)
}
// Last implements FragmentIterator.
func (i *filteringIter) Last() Span {
return i.filter(i.iter.Last(), -1)
}
// Next implements FragmentIterator.
func (i *filteringIter) Next() Span {
return i.filter(i.iter.Next(), +1)
}
// Prev implements FragmentIterator.
func (i *filteringIter) Prev() Span {
return i.filter(i.iter.Prev(), -1)
}
// Error implements FragmentIterator.
func (i *filteringIter) Error() error {
return i.iter.Error()
}
// Close implements FragmentIterator.
func (i *filteringIter) Close() error {
return i.iter.Close()
}
// filter uses the filterFn (if configured) to filter and possibly mutate the
// given Span. If the current Span is to be skipped, the iterator continues
// iterating in the given direction until it lands on a Span that should be
// returned, or the iterator becomes invalid.
func (i *filteringIter) filter(span Span, dir int8) Span {
if i.filterFn == nil {
return span
}
for i.Error() == nil && span.Valid() {
if keep := i.filterFn(span, &i.span); keep {
return i.span
}
if dir == +1 {
span = i.Next()
} else {
span = i.Prev()
}
}
return span
}

447
vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go generated vendored Normal file
View File

@@ -0,0 +1,447 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"sort"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
type spansByStartKey struct {
cmp base.Compare
buf []Span
}
func (v *spansByStartKey) Len() int { return len(v.buf) }
func (v *spansByStartKey) Less(i, j int) bool {
return v.cmp(v.buf[i].Start, v.buf[j].Start) < 0
}
func (v *spansByStartKey) Swap(i, j int) {
v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
}
type spansByEndKey struct {
cmp base.Compare
buf []Span
}
func (v *spansByEndKey) Len() int { return len(v.buf) }
func (v *spansByEndKey) Less(i, j int) bool {
return v.cmp(v.buf[i].End, v.buf[j].End) < 0
}
func (v *spansByEndKey) Swap(i, j int) {
v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
}
// keysBySeqNumKind sorts spans by the start key's sequence number in
// descending order. If two spans have equal sequence number, they're compared
// by key kind in descending order. This ordering matches the ordering of
// base.InternalCompare among keys with matching user keys.
type keysBySeqNumKind []Key
func (v *keysBySeqNumKind) Len() int { return len(*v) }
func (v *keysBySeqNumKind) Less(i, j int) bool { return (*v)[i].Trailer > (*v)[j].Trailer }
func (v *keysBySeqNumKind) Swap(i, j int) { (*v)[i], (*v)[j] = (*v)[j], (*v)[i] }
// Sort the spans by start key. This is the ordering required by the
// Fragmenter. Usually spans are naturally sorted by their start key,
// but that isn't true for range deletion tombstones in the legacy
// range-del-v1 block format.
func Sort(cmp base.Compare, spans []Span) {
sorter := spansByStartKey{
cmp: cmp,
buf: spans,
}
sort.Sort(&sorter)
}
// Fragmenter fragments a set of spans such that overlapping spans are
// split at their overlap points. The fragmented spans are output to the
// supplied Output function.
type Fragmenter struct {
Cmp base.Compare
Format base.FormatKey
// Emit is called to emit a fragmented span and its keys. Every key defined
// within the emitted Span applies to the entirety of the Span's key span.
// Keys are ordered in decreasing order of their sequence numbers, and if
// equal, decreasing order of key kind.
Emit func(Span)
// pending contains the list of pending fragments that have not been
// flushed to the block writer. Note that the spans have not been
// fragmented on the end keys yet. That happens as the spans are
// flushed. All pending spans have the same Start.
pending []Span
// doneBuf is used to buffer completed span fragments when flushing to a
// specific key (e.g. TruncateAndFlushTo). It is cached in the Fragmenter to
// allow reuse.
doneBuf []Span
// sortBuf is used to sort fragments by end key when flushing.
sortBuf spansByEndKey
// flushBuf is used to sort keys by (seqnum,kind) before emitting.
flushBuf keysBySeqNumKind
// flushedKey is the key that fragments have been flushed up to. Any
// additional spans added to the fragmenter must have a start key >=
// flushedKey. A nil value indicates flushedKey has not been set.
flushedKey []byte
finished bool
}
func (f *Fragmenter) checkInvariants(buf []Span) {
for i := 1; i < len(buf); i++ {
if f.Cmp(buf[i].Start, buf[i].End) >= 0 {
panic(fmt.Sprintf("pebble: empty pending span invariant violated: %s", buf[i]))
}
if f.Cmp(buf[i-1].Start, buf[i].Start) != 0 {
panic(fmt.Sprintf("pebble: pending span invariant violated: %s %s",
f.Format(buf[i-1].Start), f.Format(buf[i].Start)))
}
}
}
// Add adds a span to the fragmenter. Spans may overlap and the
// fragmenter will internally split them. The spans must be presented in
// increasing start key order. That is, Add must be called with a series
// of spans like:
//
// a---e
// c---g
// c-----i
// j---n
// j-l
//
// We need to fragment the spans at overlap points. In the above
// example, we'd create:
//
// a-c-e
// c-e-g
// c-e-g-i
// j-l-n
// j-l
//
// The fragments need to be output sorted by start key, and for equal start
// keys, sorted by descending sequence number. This last part requires a mild
// bit of care as the fragments are not created in descending sequence number
// order.
//
// Once a start key has been seen, we know that we'll never see a smaller
// start key and can thus flush all of the fragments that lie before that
// start key.
//
// Walking through the example above, we start with:
//
// a---e
//
// Next we add [c,g) resulting in:
//
// a-c-e
// c---g
//
// The fragment [a,c) is flushed leaving the pending spans as:
//
// c-e
// c---g
//
// The next span is [c,i):
//
// c-e
// c---g
// c-----i
//
// No fragments are flushed. The next span is [j,n):
//
// c-e
// c---g
// c-----i
// j---n
//
// The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments
// by their end key, then split the fragments on the end keys:
//
// c-e
// c-e-g
// c-e---i
//
// The [c,e) fragments all get flushed leaving:
//
// e-g
// e---i
//
// This process continues until there are no more fragments to flush.
//
// WARNING: the slices backing Start, End, Keys, Key.Suffix and Key.Value are
// all retained after this method returns and should not be modified. This is
// safe for spans that are added from a memtable or batch. It is partially
// unsafe for a span read from an sstable. Specifically, the Keys slice of a
// Span returned during sstable iteration is only valid until the next iterator
// operation. The stability of the user keys depend on whether the block is
// prefix compressed, and in practice Pebble never prefix compresses range
// deletion and range key blocks, so these keys are stable. Because of this key
// stability, typically callers only need to perform a shallow clone of the Span
// before Add-ing it to the fragmenter.
func (f *Fragmenter) Add(s Span) {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if f.flushedKey != nil {
switch c := f.Cmp(s.Start, f.flushedKey); {
case c < 0:
panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
f.Format(s.Start), f.Format(f.flushedKey)))
}
}
if f.Cmp(s.Start, s.End) >= 0 {
// An empty span, we can ignore it.
return
}
if invariants.RaceEnabled {
f.checkInvariants(f.pending)
defer func() { f.checkInvariants(f.pending) }()
}
if len(f.pending) > 0 {
// Since all of the pending spans have the same start key, we only need
// to compare against the first one.
switch c := f.Cmp(f.pending[0].Start, s.Start); {
case c > 0:
panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
f.Format(f.pending[0].Start), f.Format(s.Start)))
case c == 0:
// The new span has the same start key as the existing pending
// spans. Add it to the pending buffer.
f.pending = append(f.pending, s)
return
}
// At this point we know that the new start key is greater than the pending
// spans start keys.
f.truncateAndFlush(s.Start)
}
f.pending = append(f.pending, s)
}
// Covers returns true if the specified key is covered by one of the pending
// spans. The key must be consistent with the ordering of the
// spans. That is, it is invalid to specify a key here that is out of
// order with the span start keys passed to Add.
func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if len(f.pending) == 0 {
return false
}
if f.Cmp(f.pending[0].Start, key.UserKey) > 0 {
panic(fmt.Sprintf("pebble: keys must be in order: %s > %s",
f.Format(f.pending[0].Start), key.Pretty(f.Format)))
}
seqNum := key.SeqNum()
for _, s := range f.pending {
if f.Cmp(key.UserKey, s.End) < 0 {
// NB: A range deletion tombstone does not delete a point operation
// at the same sequence number, and broadly a span is not considered
// to cover a point operation at the same sequence number.
if s.Visible(snapshot).Covers(seqNum) {
return true
}
}
}
return false
}
// Empty returns true if all fragments added so far have finished flushing.
func (f *Fragmenter) Empty() bool {
return f.finished || len(f.pending) == 0
}
// TruncateAndFlushTo flushes all of the fragments with a start key <= key,
// truncating spans to the specified end key. Used during compaction to force
// emitting of spans which straddle an sstable boundary. Consider
// the scenario:
//
// a---------k#10
// f#8
// f#7
///
// Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will
// flush this span:
//
// a-------g#10
// f#8
// f#7
//
// And leave this one in f.pending:
//
// g----k#10
//
// WARNING: The fragmenter could hold on to the specified end key. Ensure it's
// a safe byte slice that could outlast the current sstable output, and one
// that will never be modified.
func (f *Fragmenter) TruncateAndFlushTo(key []byte) {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if f.flushedKey != nil {
switch c := f.Cmp(key, f.flushedKey); {
case c < 0:
panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
f.Format(key), f.Format(f.flushedKey)))
}
}
if invariants.RaceEnabled {
f.checkInvariants(f.pending)
defer func() { f.checkInvariants(f.pending) }()
}
if len(f.pending) > 0 {
// Since all of the pending spans have the same start key, we only need
// to compare against the first one.
switch c := f.Cmp(f.pending[0].Start, key); {
case c > 0:
panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
f.Format(f.pending[0].Start), f.Format(key)))
case c == 0:
return
}
}
f.truncateAndFlush(key)
}
// Start returns the start key of the first span in the pending buffer, or nil
// if there are no pending spans. The start key of all pending spans is the same
// as that of the first one.
func (f *Fragmenter) Start() []byte {
if len(f.pending) > 0 {
return f.pending[0].Start
}
return nil
}
// Flushes all pending spans up to key (exclusive).
//
// WARNING: The specified key is stored without making a copy, so all callers
// must ensure it is safe.
func (f *Fragmenter) truncateAndFlush(key []byte) {
f.flushedKey = append(f.flushedKey[:0], key...)
done := f.doneBuf[:0]
pending := f.pending
f.pending = f.pending[:0]
// pending and f.pending share the same underlying storage. As we iterate
// over pending we append to f.pending, but only one entry is appended in
// each iteration, after we have read the entry being overwritten.
for _, s := range pending {
if f.Cmp(key, s.End) < 0 {
// s: a--+--e
// new: c------
if f.Cmp(s.Start, key) < 0 {
done = append(done, Span{
Start: s.Start,
End: key,
Keys: s.Keys,
})
}
f.pending = append(f.pending, Span{
Start: key,
End: s.End,
Keys: s.Keys,
})
} else {
// s: a-----e
// new: e----
done = append(done, s)
}
}
f.doneBuf = done[:0]
f.flush(done, nil)
}
// flush a group of range spans to the block. The spans are required to all have
// the same start key. We flush all span fragments until startKey > lastKey. If
// lastKey is nil, all span fragments are flushed. The specification of a
// non-nil lastKey occurs for range deletion tombstones during compaction where
// we want to flush (but not truncate) all range tombstones that start at or
// before the first key in the next sstable. Consider:
//
// a---e#10
// a------h#9
//
// If a compaction splits the sstables at key c we want the first sstable to
// contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally
// produce a tombstone [e,h)#9, but we don't need to output that tombstone to
// the first sstable.
func (f *Fragmenter) flush(buf []Span, lastKey []byte) {
if invariants.RaceEnabled {
f.checkInvariants(buf)
}
// Sort the spans by end key. This will allow us to walk over the spans and
// easily determine the next split point (the smallest end-key).
f.sortBuf.cmp = f.Cmp
f.sortBuf.buf = buf
sort.Sort(&f.sortBuf)
// Loop over the spans, splitting by end key.
for len(buf) > 0 {
// A prefix of spans will end at split. remove represents the count of
// that prefix.
remove := 1
split := buf[0].End
f.flushBuf = append(f.flushBuf[:0], buf[0].Keys...)
for i := 1; i < len(buf); i++ {
if f.Cmp(split, buf[i].End) == 0 {
remove++
}
f.flushBuf = append(f.flushBuf, buf[i].Keys...)
}
sort.Sort(&f.flushBuf)
f.Emit(Span{
Start: buf[0].Start,
End: split,
// Copy the sorted keys to a new slice.
//
// This allocation is an unfortunate side effect of the Fragmenter and
// the expectation that the spans it produces are available in-memory
// indefinitely.
//
// Eventually, we should be able to replace the fragmenter with the
// keyspan.MergingIter which will perform just-in-time
// fragmentation, and only guaranteeing the memory lifetime for the
// current span. The MergingIter fragments while only needing to
// access one Span per level. It only accesses the Span at the
// current position for each level. During compactions, we can write
// these spans to sstables without retaining previous Spans.
Keys: append([]Key(nil), f.flushBuf...),
})
if lastKey != nil && f.Cmp(split, lastKey) > 0 {
break
}
// Adjust the start key for every remaining span.
buf = buf[remove:]
for i := range buf {
buf[i].Start = split
}
}
}
// Finish flushes any remaining fragments to the output. It is an error to call
// this if any other spans will be added.
func (f *Fragmenter) Finish() {
if f.finished {
panic("pebble: span fragmenter already finished")
}
f.flush(f.pending, nil)
f.finished = true
}

53
vendor/github.com/cockroachdb/pebble/internal/keyspan/get.go generated vendored Normal file
View File

@@ -0,0 +1,53 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Get returns the newest span that contains the target key. If no span
// contains the target key, an empty span is returned. The snapshot
// parameter controls the visibility of spans (only spans older than the
// snapshot sequence number are visible). The iterator must contain
// fragmented spans: no span may overlap another.
func Get(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were
// to use SeekGE then the search key `c` would land on the span
// [e,i) and we'd have to backtrack. The one complexity here is what
// happens for the search key `e`. In that case SeekLT will land us
// on the span [a,e) and we'll have to move forward.
iterSpan := iter.SeekLT(key)
if !iterSpan.Valid() {
iterSpan = iter.Next()
if !iterSpan.Valid() {
// The iterator is empty.
return Span{}
}
if cmp(key, iterSpan.Start) < 0 {
// The search key lies before the first span.
return Span{}
}
}
// Invariant: key > iterSpan.Start
if cmp(key, iterSpan.End) >= 0 {
// The current span lies before the search key. Advance the iterator
// once to potentially land on a key with a start key exactly equal to
// key. (See the comment at the beginning of this function.)
iterSpan = iter.Next()
if !iterSpan.Valid() || cmp(key, iterSpan.Start) < 0 {
// We've run out of spans or we've moved on to a span which
// starts after our search key.
return Span{}
}
}
return iterSpan.Visible(snapshot)
}

798
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored Normal file
View File

@@ -0,0 +1,798 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
)
// TODO(jackson): The interleaving iterator has various invariants that it
// asserts. We should eventually gate these behind `invariants.Enabled`.
// Hooks configure the interleaving iterator's behavior.
type Hooks struct {
// SpanChanged is invoked by interleaving iterator whenever the Span
// returned by InterleavaingIterator.Span changes. As the iterator passes
// into or out of a Span, it invokes SpanChanged, passing the new Span.
SpanChanged func(Span)
// SkipPoint is invoked by the interleaving iterator whenever the iterator
// encounters a point key covered by a Span. If SkipPoint returns true, the
// interleaving iterator skips the point key without returning it. This is
// used during range key iteration to skip over point keys 'masked' by range
// keys.
SkipPoint func(userKey []byte) bool
}
// InterleavingIter combines an iterator over point keys with an iterator over
// key spans.
//
// Throughout Pebble, some keys apply at single discrete points within the user
// keyspace. Other keys apply over continuous spans of the user key space.
// Internally, iterators over point keys adhere to the base.InternalIterator
// interface, and iterators over spans adhere to the keyspan.FragmentIterator
// interface. The InterleavingIterator wraps a point iterator and span iterator,
// providing access to all the elements of both iterators.
//
// The InterleavingIterator implements the point base.InternalIterator
// interface. After any of the iterator's methods return a key, a caller may
// call Span to retrieve the span covering the returned key, if any. A span is
// considered to 'cover' a returned key if the span's [start, end) bounds
// include the key's user key.
//
// In addition to tracking the current covering span, InterleavingIter returns a
// special InternalKey at span start boundaries. Start boundaries are surfaced
// as a synthetic span marker: an InternalKey with the boundary as the user key,
// the infinite sequence number and a key kind selected from an arbitrary key
// the infinite sequence number and an arbitrary contained key's kind. Since
// which of the Span's key's kind is surfaced is undefined, the caller should
// not use the InternalKey's kind. The caller should only rely on the `Span`
// method for retrieving information about spanning keys. The interleaved
// synthetic keys have the infinite sequence number so that they're interleaved
// before any point keys with the same user key when iterating forward and after
// when iterating backward.
//
// Interleaving the synthetic start key boundaries at the maximum sequence
// number provides an opportunity for the higher-level, public Iterator to
// observe the Span, even if no live points keys exist within the boudns of the
// Span.
//
// When returning a synthetic marker key for a start boundary, InterleavingIter
// will truncate the span's start bound to the SeekGE or SeekPrefixGE search
// key. For example, a SeekGE("d") that finds a span [a, z) may return a
// synthetic span marker key `d#72057594037927935,21`.
//
// If bounds have been applied to the iterator through SetBounds,
// InterleavingIter will truncate the bounds of spans returned through Span to
// the set bounds. The bounds returned through Span are not truncated by a
// SeekGE or SeekPrefixGE search key. Consider, for example SetBounds('c', 'e'),
// with an iterator containing the Span [a,z):
//
// First() = `c#72057594037927935,21` Span() = [c,e)
// SeekGE('d') = `d#72057594037927935,21` Span() = [c,e)
//
// InterleavedIter does not interleave synthetic markers for spans that do not
// contain any keys.
//
// Hooks
//
// InterelavingIter takes a Hooks parameter that may be used to configure the
// behavior of the iterator. See the documentation on the Hooks type.
//
// All spans containing keys are exposed during iteration.
type InterleavingIter struct {
cmp base.Compare
pointIter base.InternalIteratorWithStats
keyspanIter FragmentIterator
hooks Hooks
// lower and upper hold the iteration bounds set through SetBounds.
lower, upper []byte
// keyBuf is used to copy SeekGE or SeekPrefixGE arguments when they're used
// to truncate a span. The byte slices backing a SeekGE/SeekPrefixGE search
// keys can come directly from the end user, so they're copied into keyBuf
// to ensure key stability.
keyBuf []byte
pointKey *base.InternalKey
pointVal []byte
// span, spanStart and spanEnd describe the current span state.
// span{Start,End} are a function of the span and the Iterator's bounds.
// span{Start,End} will be nil iff span is !Valid().
span Span
spanStart []byte
spanEnd []byte
// spanMarker holds the synthetic key that is returned when the iterator
// passes over a key span's start bound.
spanMarker base.InternalKey
// Keeping all of the bools together reduces the sizeof the struct.
// spanCoversKey indicates whether the current span covers the last-returned
// key.
spanCoversKey bool
// pointKeyInterleaved indicates whether the current point key has been
// interleaved in the current direction.
pointKeyInterleaved bool
// keyspanInterleaved indicates whether or not the current span has been
// interleaved at its start key in the current direction. A span marker is
// interleaved when first passing over the start key.
//
// When iterating in the forward direction, the span start key is
// interleaved when the span first begins to cover the current iterator
// position. The keyspan iterator isn't advanced until the
// InterleavingIterator moves beyond the current span's end key. This field
// is used to remember that the span has already been interleaved and
// shouldn't be interleaved again.
//
// When iterating in the reverse direction, the span start key is
// interleaved immediately before the iterator will move to a key no longer
// be covered by the span. This field behaves analagously to
// pointKeyInterleaved and if true signals that we must Prev the keyspan
// iterator on the next Prev call.
keyspanInterleaved bool
// spanMarkerTruncated is set by SeekGE/SeekPrefixGE calls that truncate a
// span's start bound marker to the search key. It's returned to false on
// the next repositioning of the keyspan iterator.
spanMarkerTruncated bool
// dir indicates the direction of iteration: forward (+1) or backward (-1)
dir int8
}
// Assert that *InterleavingIter implements the InternalIterator interface.
var _ base.InternalIterator = &InterleavingIter{}
// Init initializes the InterleavingIter to interleave point keys from pointIter
// with key spans from keyspanIter.
//
// The point iterator must already have the provided bounds. Init does not
// propagate the bounds down the iterator stack.
func (i *InterleavingIter) Init(
cmp base.Compare,
pointIter base.InternalIteratorWithStats,
keyspanIter FragmentIterator,
hooks Hooks,
lowerBound, upperBound []byte,
) {
*i = InterleavingIter{
cmp: cmp,
pointIter: pointIter,
keyspanIter: keyspanIter,
hooks: hooks,
lower: lowerBound,
upper: upperBound,
}
}
// SeekGE implements (base.InternalIterator).SeekGE.
//
// If there exists a span with a start key ≤ the first matching point key,
// SeekGE will return a synthetic span marker key for the span. If this span's
// start key is less than key, the returned marker will be truncated to key.
// Note that this search-key truncation of the marker's key is not applied to
// the span returned by Span.
//
// NB: In accordance with the base.InternalIterator contract:
// i.lower ≤ key
func (i *InterleavingIter) SeekGE(key []byte, trySeekUsingNext bool) (*base.InternalKey, []byte) {
i.pointKey, i.pointVal = i.pointIter.SeekGE(key, trySeekUsingNext)
i.pointKeyInterleaved = false
i.keyspanSeekGE(key)
i.dir = +1
return i.interleaveForward(key)
}
// SeekPrefixGE implements (base.InternalIterator).SeekPrefixGE.
//
// If there exists a span with a start key ≤ the first matching point key,
// SeekPrefixGE will return a synthetic span marker key for the span. If this
// span's start key is less than key, the returned marker will be truncated to
// key. Note that this search-key truncation of the marker's key is not applied
// to the span returned by Span.
//
// NB: In accordance with the base.InternalIterator contract:
// i.lower ≤ key
func (i *InterleavingIter) SeekPrefixGE(
prefix, key []byte, trySeekUsingNext bool,
) (*base.InternalKey, []byte) {
i.pointKey, i.pointVal = i.pointIter.SeekPrefixGE(prefix, key, trySeekUsingNext)
i.pointKeyInterleaved = false
i.keyspanSeekGE(key)
i.dir = +1
return i.interleaveForward(key)
}
// SeekLT implements (base.InternalIterator).SeekLT.
func (i *InterleavingIter) SeekLT(key []byte) (*base.InternalKey, []byte) {
i.pointKey, i.pointVal = i.pointIter.SeekLT(key)
i.pointKeyInterleaved = false
i.keyspanSeekLT(key)
i.dir = -1
return i.interleaveBackward()
}
// First implements (base.InternalIterator).First.
func (i *InterleavingIter) First() (*base.InternalKey, []byte) {
i.pointKey, i.pointVal = i.pointIter.First()
i.pointKeyInterleaved = false
i.saveKeyspan(i.checkForwardBound(i.keyspanIter.First()))
i.dir = +1
return i.interleaveForward(i.lower)
}
// Last implements (base.InternalIterator).Last.
func (i *InterleavingIter) Last() (*base.InternalKey, []byte) {
i.pointKey, i.pointVal = i.pointIter.Last()
i.pointKeyInterleaved = false
i.saveKeyspan(i.checkBackwardBound(i.keyspanIter.Last()))
i.dir = -1
return i.interleaveBackward()
}
// Next implements (base.InternalIterator).Next.
func (i *InterleavingIter) Next() (*base.InternalKey, []byte) {
if i.dir == -1 {
// Switching directions.
i.dir = +1
// The existing point key (denoted below with *) is either the last
// key we returned (the current iterator position):
// points: x (y*) z
// or the upcoming point key in the backward direction if we just
// returned a span start boundary key:
// points: x* z
// spans: ([y-?))
// direction. Either way, we must move to the next point key.
switch {
case i.pointKey == nil && i.lower == nil:
i.pointKey, i.pointVal = i.pointIter.First()
case i.pointKey == nil && i.lower != nil:
i.pointKey, i.pointVal = i.pointIter.SeekGE(i.lower, false)
default:
i.pointKey, i.pointVal = i.pointIter.Next()
}
i.pointKeyInterleaved = false
if !i.span.Valid() {
// There was no span in the reverse direction, but there may be
// a span in the forward direction.
i.saveKeyspan(i.checkForwardBound(i.keyspanIter.Next()))
i.keyspanInterleaved = false
} else {
// Regardless of the current iterator state, we mark any existing
// span as interleaved when switching to forward iteration,
// justified below.
//
// If the point key is the last key returned:
// pointIter : ... (y) z ...
// keyspanIter : ... ([x - )) ...
// ^
// The span's start key must be ≤ the point key, otherwise we'd have
// interleaved the span's start key. From a forward-iteration
// perspective, the span's start key is in the past and should be
// considered already-interleaved.
//
// If the span start boundary key is the last key returned:
// pointIter : ... (x) z ...
// keyspanIter : ... ([y - )) ...
// ^
// i.span.Start is the key we last returned during reverse
// iteration. From the perspective of forward-iteration, its start
// key was just visited.
i.keyspanInterleaved = true
}
}
// Refresh the point key if the current point key has already been
// interleaved.
if i.pointKeyInterleaved {
i.pointKey, i.pointVal = i.pointIter.Next()
i.pointKeyInterleaved = false
}
// If we already interleaved the current span start key, and the point key
// is ≥ the span's end key, move to the next span.
if i.keyspanInterleaved && i.pointKey != nil && i.span.Valid() &&
i.cmp(i.pointKey.UserKey, i.span.End) >= 0 {
i.saveKeyspan(i.checkForwardBound(i.keyspanIter.Next()))
}
return i.interleaveForward(i.lower)
}
// Prev implements (base.InternalIterator).Prev.
func (i *InterleavingIter) Prev() (*base.InternalKey, []byte) {
if i.dir == +1 {
// Switching directions.
i.dir = -1
if i.keyspanInterleaved {
// The current span's start key has already been interleaved in the
// forward direction. The start key may have been interleaved a
// while ago, or it might've been interleaved at the current
// iterator position. If it was interleaved a while ago, the current
// span is still relevant and we should not move the keyspan
// iterator.
//
// If it was just interleaved at the current iterator position, the
// span start was the last key returned to the user. We should
// prev past it so we don't return it again, with an exception.
// Consider span [a, z) and this sequence of iterator calls:
//
// SeekGE('c') = c.RANGEKEYSET#72057594037927935
// Prev() = a.RANGEKEYSET#72057594037927935
//
// If the current span's start key was last surfaced truncated due
// to a SeekGE or SeekPrefixGE call, then it's still relevant in the
// reverse direction with an untruncated start key.
//
// We can determine whether the last key returned was a point key by
// checking i.pointKeyInterleaved, because every Next/Prev will
// advance the point iterator and reset pointKeyInterleaved if it
// was.
if i.pointKeyInterleaved || i.spanMarkerTruncated {
// The last returned key was a point key, OR a truncated span
// marker key. Don't move, but re-save the span because it
// should no longer be considered truncated or interleaved.
i.saveKeyspan(i.span)
} else {
// The last returned key is this key's start boundary, so Prev
// past it so we don't return it again.
i.saveKeyspan(i.checkBackwardBound(i.keyspanIter.Prev()))
}
} else {
// If the current span's start key has not been interleaved, then
// the span's start key is greater than the current iterator
// position (denoted in parenthesis), and the current span's start
// key is ahead of our iterator position. Move it to the previous
// span:
// points: (x*)
// span: [y-z)*
i.saveKeyspan(i.checkBackwardBound(i.keyspanIter.Prev()))
}
// The existing point key (denoted below with *) is either the last
// key we returned (the current iterator position):
// points: x (y*) z
// or the upcoming point key in the forward direction if we just
// returned a span start boundary key :
// points: x z*
// spans: ([y-?))
// direction. Either way, we must move the point iterator backwards.
switch {
case i.pointKey == nil && i.upper == nil:
i.pointKey, i.pointVal = i.pointIter.Last()
case i.pointKey == nil && i.upper != nil:
i.pointKey, i.pointVal = i.pointIter.SeekLT(i.upper)
default:
i.pointKey, i.pointVal = i.pointIter.Prev()
}
i.pointKeyInterleaved = false
}
// Refresh the point key if we just returned the current point key.
if i.pointKeyInterleaved {
i.pointKey, i.pointVal = i.pointIter.Prev()
i.pointKeyInterleaved = false
}
// Refresh the span if we just returned the span's start boundary key.
if i.keyspanInterleaved {
i.saveKeyspan(i.checkBackwardBound(i.keyspanIter.Prev()))
}
return i.interleaveBackward()
}
func (i *InterleavingIter) interleaveForward(lowerBound []byte) (*base.InternalKey, []byte) {
// This loop determines whether a point key or a span marker key should be
// interleaved on each iteration. If masking is disabled and the span is
// nonempty, this loop executes for exactly one iteration. If masking is
// enabled and a masked key is determined to be interleaved next, this loop
// continues until the interleaved key is unmasked. If a span's start key
// should be interleaved next, but the span is empty, the loop continues to
// the next key.
for {
// Check invariants.
// INVARIANT: !pointKeyInterleaved
if i.pointKeyInterleaved {
panic("pebble: invariant violation: point key interleaved")
}
switch {
case !i.span.Valid():
case i.pointKey == nil:
default:
// INVARIANT: !keyspanInterleaved || pointKey < span.End
// The caller is responsible for advancing this span if it's already
// been interleaved and the span ends before the point key.
// Absolute positioning methods will never have already interleaved
// the span's start key, so only Next needs to handle the case where
// pointKey >= span.End.
if i.keyspanInterleaved && i.cmp(i.pointKey.UserKey, i.span.End) >= 0 {
panic("pebble: invariant violation: span interleaved, but point key >= span end")
}
}
// Interleave.
switch {
case !i.span.Valid():
// If we're out of spans, just return the point key.
return i.yieldPointKey(false /* covered */)
case i.pointKey == nil:
if i.pointKeyInterleaved {
panic("pebble: invariant violation: point key already interleaved")
}
// If we're out of point keys, we need to return a span marker. If
// the current span has already been interleaved, advance it. Since
// there are no more point keys, we don't need to worry about
// advancing past the current point key.
if i.keyspanInterleaved {
i.saveKeyspan(i.checkForwardBound(i.keyspanIter.Next()))
if !i.span.Valid() {
return i.yieldNil()
}
}
if i.span.Empty() {
i.keyspanInterleaved = true
continue
}
return i.yieldSyntheticSpanMarker(lowerBound)
default:
if i.cmp(i.pointKey.UserKey, i.span.Start) >= 0 {
// The span start key lies before the point key. If we haven't
// interleaved it, we should.
if !i.keyspanInterleaved {
if i.span.Empty() {
if i.pointKey != nil && i.cmp(i.pointKey.UserKey, i.span.End) >= 0 {
// Advance the keyspan iterator, as just flipping
// keyspanInterleaved would likely trip up the invariant check
// above.
i.saveKeyspan(i.checkForwardBound(i.keyspanIter.Next()))
} else {
i.keyspanInterleaved = true
}
continue
}
return i.yieldSyntheticSpanMarker(lowerBound)
}
// Otherwise, the span's start key is already interleaved and we
// need to return the point key. The current span necessarily
// must cover the point key:
//
// Since the span's start is less than or equal to the point
// key, the only way for this span to not cover the point would
// be if the span's end is less than or equal to the point.
// (For example span = [a, b), point key = c).
//
// However, the invariant at the beginning of the function
// guarantees that if:
// * we have both a point key and a span
// * and the span has already been interleaved
// => then the point key must be less than the span's end, and
// the point key must be covered by the current span.
// The span covers the point key. If a SkipPoint hook is
// configured, ask it if we should skip this point key.
if i.hooks.SkipPoint != nil && i.hooks.SkipPoint(i.pointKey.UserKey) {
i.pointKey, i.pointVal = i.pointIter.Next()
// We may have just invalidated the invariant that
// ensures the span's End is > the point key, so
// reestablish it before the next iteration.
if i.pointKey != nil && i.cmp(i.pointKey.UserKey, i.span.End) >= 0 {
i.saveKeyspan(i.checkForwardBound(i.keyspanIter.Next()))
}
continue
}
// Point key is unmasked but covered.
return i.yieldPointKey(true /* covered */)
}
return i.yieldPointKey(false /* covered */)
}
}
}
func (i *InterleavingIter) interleaveBackward() (*base.InternalKey, []byte) {
// This loop determines whether a point key or a span's start key should be
// interleaved on each iteration. If masking is disabled and the span is
// nonempty, this loop executes for exactly one iteration. If masking is
// enabled and a masked key is determined to be interleaved next, this loop
// continues until the interleaved key is unmasked. If a span's start key
// should be interleaved next, but the span is empty, the loop continues to
// the next key.
for {
// Check invariants.
// INVARIANT: !pointKeyInterleaved
if i.pointKeyInterleaved {
panic("pebble: invariant violation: point key interleaved")
}
// Interleave.
switch {
case !i.span.Valid():
// If we're out of spans, just return the point key.
return i.yieldPointKey(false /* covered */)
case i.pointKey == nil:
// If we're out of point keys, we need to return a span marker.
if i.span.Empty() {
i.saveKeyspan(i.checkBackwardBound(i.keyspanIter.Prev()))
continue
}
return i.yieldSyntheticSpanMarker(i.lower)
default:
// If the span's start key is greater than the point key, return a
// marker for the span.
if i.cmp(i.span.Start, i.pointKey.UserKey) > 0 {
if i.span.Empty() {
i.saveKeyspan(i.checkBackwardBound(i.keyspanIter.Prev()))
continue
}
return i.yieldSyntheticSpanMarker(i.lower)
}
// We have a span but it has not been interleaved and begins at a
// key equal to or before the current point key. The point key
// should be interleaved next, if it's not masked.
if i.cmp(i.pointKey.UserKey, i.span.End) < 0 {
// The span covers the point key. The point key might be masked
// too if masking is enabled.
// The span covers the point key. If a SkipPoint hook is
// configured, ask it if we should skip this point key.
if i.hooks.SkipPoint != nil && i.hooks.SkipPoint(i.pointKey.UserKey) {
i.pointKey, i.pointVal = i.pointIter.Prev()
continue
}
// Point key is unmasked but covered.
return i.yieldPointKey(true /* covered */)
}
return i.yieldPointKey(false /* covered */)
}
}
}
// keyspanSeekGE seeks the keyspan iterator to the first span covering k ≥ key.
// Note that this differs from the FragmentIterator.SeekGE semantics, which
// seek to the first span with a start key ≥ key.
func (i *InterleavingIter) keyspanSeekGE(key []byte) {
// Seek using SeekLT to look for a span that starts before key, with an end
// boundary extending beyond key.
s := i.keyspanIter.SeekLT(key)
if !s.Valid() || i.cmp(s.End, key) <= 0 {
// The iterator is exhausted in the reverse direction, or the span we
// found ends before key. Next to the first key with a start ≥ key.
s = i.keyspanIter.Next()
}
i.saveKeyspan(i.checkForwardBound(s))
}
// keyspanSeekLT seeks the keyspan iterator to the last span covering k < key.
// Note that this differs from the FragmentIterator.SeekLT semantics, which
// seek to the last span with a start key < key.
func (i *InterleavingIter) keyspanSeekLT(key []byte) {
s := i.checkBackwardBound(i.keyspanIter.SeekLT(key))
// s.Start is not guaranteed to be less than key, because of the bounds
// enforcement. Consider the following example:
//
// Bounds are set to [d,e). The user performs a SeekLT(d). The
// FragmentIterator.SeekLT lands on a span [b,f). This span has a start key
// less than d, as expected. Above, checkBackwardBound truncates the span to
// match the iterator's current bounds, modifying the span to [d,e), which
// does not overlap the search space of [-∞, d).
//
// This problem is a consequence of the SeekLT's exclusive search key and
// the fact that we don't perform bounds truncation at every leaf iterator.
if i.cmp(s.Start, key) >= 0 {
s = Span{}
}
i.saveKeyspan(s)
}
func (i *InterleavingIter) checkForwardBound(s Span) Span {
// Check the upper bound if we have one.
if s.Valid() && i.upper != nil && i.cmp(s.Start, i.upper) >= 0 {
return Span{}
}
// TODO(jackson): The key comparisons below truncate bounds whenever the
// keyspan iterator is repositioned. We could perform this lazily, and do it
// the first time the user actually asks for this span's bounds in
// SpanBounds. This would reduce work in the case where there's no span
// covering the point and the keyspan iterator is non-empty.
// NB: These truncations don't require setting `keyspanMarkerTruncated`:
// That flag only applies to truncated span marker keys.
if i.lower != nil && i.cmp(s.Start, i.lower) < 0 {
s.Start = i.lower
}
if i.upper != nil && i.cmp(i.upper, s.End) < 0 {
s.End = i.upper
}
if i.cmp(s.Start, s.End) == 0 {
return Span{}
}
return s
}
func (i *InterleavingIter) checkBackwardBound(s Span) Span {
// Check the lower bound if we have one.
if s.Valid() && i.lower != nil && i.cmp(s.End, i.lower) <= 0 {
return Span{}
}
// TODO(jackson): The key comparisons below truncate bounds whenever the
// keyspan iterator is repositioned. We could perform this lazily, and do it
// the first time the user actually asks for this span's bounds in
// SpanBounds. This would reduce work in the case where there's no span
// covering the point and the keyspan iterator is non-empty.
// NB: These truncations don't require setting `keyspanMarkerTruncated`:
// That flag only applies to truncated span marker keys.
if i.lower != nil && i.cmp(s.Start, i.lower) < 0 {
s.Start = i.lower
}
if i.upper != nil && i.cmp(i.upper, s.End) < 0 {
s.End = i.upper
}
if i.cmp(s.Start, s.End) == 0 {
return Span{}
}
return s
}
func (i *InterleavingIter) yieldNil() (*base.InternalKey, []byte) {
i.spanCoversKey = false
return i.verify(nil, nil)
}
func (i *InterleavingIter) yieldPointKey(covered bool) (*base.InternalKey, []byte) {
i.pointKeyInterleaved = true
i.spanCoversKey = covered
return i.verify(i.pointKey, i.pointVal)
}
func (i *InterleavingIter) yieldSyntheticSpanMarker(lowerBound []byte) (*base.InternalKey, []byte) {
i.spanMarker.UserKey = i.span.Start
i.spanMarker.Trailer = base.MakeTrailer(base.InternalKeySeqNumMax, i.span.Keys[0].Kind())
i.keyspanInterleaved = true
i.spanCoversKey = true
// Truncate the key we return to our lower bound if we have one. Note that
// we use the lowerBound function parameter, not i.lower. The lowerBound
// argument is guaranteed to be ≥ i.lower. It may be equal to the SetBounds
// lower bound, or it could come from a SeekGE or SeekPrefixGE search key.
if lowerBound != nil && i.cmp(lowerBound, i.span.Start) > 0 {
// Truncating to the lower bound may violate the upper bound if
// lowerBound == i.upper. For example, a SeekGE(k) uses k as a lower
// bound for truncating a span. The span a-z will be truncated to [k,
// z). If i.upper == k, we'd mistakenly try to return a span [k, k), an
// invariant violation.
if i.cmp(lowerBound, i.upper) == 0 {
return i.yieldNil()
}
// If the lowerBound argument came from a SeekGE or SeekPrefixGE
// call, and it may be backed by a user-provided byte slice that is not
// guaranteed to be stable.
//
// If the lowerBound argument is the lower bound set by SetBounds,
// Pebble owns the slice's memory. However, consider two successive
// calls to SetBounds(). The second may overwrite the lower bound.
// Although the external contract requires a seek after a SetBounds,
// Pebble's tests don't always. For this reason and to simplify
// reasoning around lifetimes, always copy the bound into keyBuf when
// truncating.
i.keyBuf = append(i.keyBuf[:0], lowerBound...)
i.spanMarker.UserKey = i.keyBuf
i.spanMarkerTruncated = true
}
return i.verify(&i.spanMarker, nil)
}
func (i *InterleavingIter) verify(k *base.InternalKey, v []byte) (*base.InternalKey, []byte) {
// TODO(jackson): Wrap the entire body of this function in an
// invariants.Enabled conditional, so that in production builds this
// function is empty and may be inlined away.
switch {
case k != nil && !i.keyspanInterleaved && !i.pointKeyInterleaved:
panic("pebble: invariant violation: both keys marked as noninterleaved")
case i.dir == -1 && k != nil && i.keyspanInterleaved == i.pointKeyInterleaved:
// During reverse iteration, if we're returning a key, either the span's
// start key must have been interleaved OR the current point key value
// is being returned, not both.
//
// This invariant holds because in reverse iteration the start key of the
// span behaves like a point. Once the start key is interleaved, we move
// the keyspan iterator to the previous span.
panic(fmt.Sprintf("pebble: invariant violation: interleaving (point %t, span %t)",
i.pointKeyInterleaved, i.keyspanInterleaved))
case i.dir == -1 && i.spanMarkerTruncated:
panic("pebble: invariant violation: truncated span key in reverse iteration")
case k != nil && i.lower != nil && i.cmp(k.UserKey, i.lower) < 0:
panic("pebble: invariant violation: key < lower bound")
case k != nil && i.upper != nil && i.cmp(k.UserKey, i.upper) >= 0:
panic("pebble: invariant violation: key ≥ upper bound")
case i.span.Valid() && k != nil && i.hooks.SkipPoint != nil && i.pointKeyInterleaved &&
i.cmp(k.UserKey, i.spanStart) >= 0 && i.cmp(k.UserKey, i.spanEnd) < 0 && i.hooks.SkipPoint(k.UserKey):
panic("pebble: invariant violation: point key eligible for skipping returned")
}
return k, v
}
func (i *InterleavingIter) saveKeyspan(s Span) {
i.keyspanInterleaved = false
i.spanMarkerTruncated = false
i.span = s
if !s.Valid() {
i.spanStart = nil
i.spanEnd = nil
if i.hooks.SpanChanged != nil {
i.hooks.SpanChanged(s)
}
return
}
i.spanStart = s.Start
i.spanEnd = s.End
if i.hooks.SpanChanged != nil {
i.hooks.SpanChanged(s)
}
}
// Span returns the span covering the last key returned, if any. A span key is
// considered to 'cover' a key if the key falls within the span's user key
// bounds.
func (i *InterleavingIter) Span() Span {
if !i.spanCoversKey {
return Span{}
}
// Return the span with truncated bounds.
return Span{
Start: i.spanStart,
End: i.spanEnd,
Keys: i.span.Keys,
}
}
// SetBounds implements (base.InternalIterator).SetBounds.
func (i *InterleavingIter) SetBounds(lower, upper []byte) {
i.lower, i.upper = lower, upper
i.pointIter.SetBounds(lower, upper)
}
// Error implements (base.InternalIterator).Error.
func (i *InterleavingIter) Error() error {
return firstError(i.pointIter.Error(), i.keyspanIter.Error())
}
// Close implements (base.InternalIterator).Close.
func (i *InterleavingIter) Close() error {
perr := i.pointIter.Close()
rerr := i.keyspanIter.Close()
return firstError(perr, rerr)
}
// String implements (base.InternalIterator).String.
func (i *InterleavingIter) String() string {
return fmt.Sprintf("keyspan-interleaving(%q)", i.pointIter.String())
}
var _ base.InternalIteratorWithStats = &InterleavingIter{}
// Stats implements InternalIteratorWithStats.
func (i *InterleavingIter) Stats() base.InternalIteratorStats {
return i.pointIter.Stats()
}
// ResetStats implements InternalIteratorWithStats.
func (i *InterleavingIter) ResetStats() {
i.pointIter.ResetStats()
}
func firstError(err0, err1 error) error {
if err0 != nil {
return err0
}
return err1
}

110
vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go generated vendored Normal file
View File

@@ -0,0 +1,110 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// InternalIteratorShim is a temporary iterator type used as a shim between
// keyspan.MergingIter and base.InternalIterator. It's used temporarily for
// range deletions during compactions, allowing range deletions to be
// interleaved by a compaction input iterator.
//
// TODO(jackson): This type should be removed, and the usages converted to using
// an InterleavingIterator type that interleaves keyspan.Spans from a
// keyspan.FragmentIterator with point keys.
type InternalIteratorShim struct {
miter MergingIter
span Span
iterKey base.InternalKey
}
// Assert that InternalIteratorShim implements InternalIterator.
var _ base.InternalIterator = &InternalIteratorShim{}
// Init initializes the internal iterator shim to merge the provided fragment
// iterators.
func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) {
i.miter.Init(cmp, noopTransform, iters...)
}
// Span returns the span containing the full set of keys over the key span at
// the current iterator position.
func (i *InternalIteratorShim) Span() Span {
return i.span
}
// SeekGE implements (base.InternalIterator).SeekGE.
func (i *InternalIteratorShim) SeekGE(
key []byte, trySeekUsingNext bool,
) (*base.InternalKey, []byte) {
panic("unimplemented")
}
// SeekPrefixGE implements (base.InternalIterator).SeekPrefixGE.
func (i *InternalIteratorShim) SeekPrefixGE(
prefix, key []byte, trySeekUsingNext bool,
) (*base.InternalKey, []byte) {
panic("unimplemented")
}
// SeekLT implements (base.InternalIterator).SeekLT.
func (i *InternalIteratorShim) SeekLT(key []byte) (*base.InternalKey, []byte) {
panic("unimplemented")
}
// First implements (base.InternalIterator).First.
func (i *InternalIteratorShim) First() (*base.InternalKey, []byte) {
i.span = i.miter.First()
for i.span.Valid() && i.span.Empty() {
i.span = i.miter.Next()
}
if !i.span.Valid() {
return nil, nil
}
i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
return &i.iterKey, i.span.End
}
// Last implements (base.InternalIterator).Last.
func (i *InternalIteratorShim) Last() (*base.InternalKey, []byte) {
panic("unimplemented")
}
// Next implements (base.InternalIterator).Next.
func (i *InternalIteratorShim) Next() (*base.InternalKey, []byte) {
i.span = i.miter.Next()
for i.span.Valid() && i.span.Empty() {
i.span = i.miter.Next()
}
if !i.span.Valid() {
return nil, nil
}
i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
return &i.iterKey, i.span.End
}
// Prev implements (base.InternalIterator).Prev.
func (i *InternalIteratorShim) Prev() (*base.InternalKey, []byte) {
panic("unimplemented")
}
// Error implements (base.InternalIterator).Error.
func (i *InternalIteratorShim) Error() error {
return i.miter.Error()
}
// Close implements (base.InternalIterator).Close.
func (i *InternalIteratorShim) Close() error {
return i.miter.Close()
}
// SetBounds implements (base.InternalIterator).SetBounds.
func (i *InternalIteratorShim) SetBounds(lower, upper []byte) {
}
// String implements fmt.Stringer.
func (i *InternalIteratorShim) String() string {
return i.miter.String()
}

199
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored Normal file
View File

@@ -0,0 +1,199 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/manifest"
)
// FragmentIterator defines an iterator interface over spans. The spans
// surfaced by a FragmentIterator must be non-overlapping. This is achieved by
// fragmenting spans at overlap points (see Fragmenter).
//
// A Span returned by a FragmentIterator is only valid until the next
// positioning method. Some implementations (eg, keyspan.Iter) may provide
// longer lifetimes but implementations need only guarantee stability until the
// next positioning method.
type FragmentIterator interface {
// SeekGE moves the iterator to the first span whose start key is greater
// than or equal to the given key.
SeekGE(key []byte) Span
// SeekLT moves the iterator to the last span whose start key is less than
// the given key.
SeekLT(key []byte) Span
// First moves the iterator to the first span.
First() Span
// Last moves the iterator to the last span.
Last() Span
// Next moves the iterator to the next span.
//
// It is valid to call Next when the iterator is positioned before the first
// key/value pair due to either a prior call to SeekLT or Prev which
// returned an invalid span. It is not allowed to call Next when the
// previous call to SeekGE, SeekPrefixGE or Next returned an invalid span.
Next() Span
// Prev moves the iterator to the previous span.
//
// It is valid to call Prev when the iterator is positioned after the last
// key/value pair due to either a prior call to SeekGE or Next which
// returned an invalid span. It is not allowed to call Prev when the
// previous call to SeekLT or Prev returned an invalid span.
Prev() Span
// Error returns any accumulated error.
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
// the iterator is not considered to be an error. It is valid to call Close
// multiple times. Other methods should not be called after the iterator has
// been closed.
Close() error
}
// TableNewSpanIter creates a new iterator for spans for the given file.
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
// per-sstable span iterators.
type SpanIterOptions struct {
// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
// when iterating over range keys.
RangeKeyFilters []base.BlockPropertyFilter
}
// Iter is an iterator over a set of fragmented spans.
type Iter struct {
cmp base.Compare
spans []Span
index int
}
// Iter implements the FragmentIterator interface.
var _ FragmentIterator = (*Iter)(nil)
// NewIter returns a new iterator over a set of fragmented spans.
func NewIter(cmp base.Compare, spans []Span) *Iter {
return &Iter{
cmp: cmp,
spans: spans,
index: -1,
}
}
// SeekGE implements FragmentIterator.SeekGE.
func (i *Iter) SeekGE(key []byte) Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
upper := len(i.spans)
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].Start) > 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
if i.index >= len(i.spans) {
return Span{}
}
return i.spans[i.index]
}
// SeekLT implements FragmentIterator.SeekLT.
func (i *Iter) SeekLT(key []byte) Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
upper := len(i.spans)
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].Start) > 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
// Since keys are strictly increasing, if i.index > 0 then i.index-1 will be
// the largest whose key is < the key sought.
i.index--
if i.index < 0 {
return Span{}
}
return i.spans[i.index]
}
// First implements FragmentIterator.First.
func (i *Iter) First() Span {
if len(i.spans) == 0 {
return Span{}
}
i.index = 0
return i.spans[i.index]
}
// Last implements FragmentIterator.Last.
func (i *Iter) Last() Span {
if len(i.spans) == 0 {
return Span{}
}
i.index = len(i.spans) - 1
return i.spans[i.index]
}
// Next implements FragmentIterator.Next.
func (i *Iter) Next() Span {
if i.index >= len(i.spans) {
return Span{}
}
i.index++
if i.index >= len(i.spans) {
return Span{}
}
return i.spans[i.index]
}
// Prev implements FragmentIterator.Prev.
func (i *Iter) Prev() Span {
if i.index < 0 {
return Span{}
}
i.index--
if i.index < 0 {
return Span{}
}
return i.spans[i.index]
}
// Error implements FragmentIterator.Error.
func (i *Iter) Error() error {
return nil
}
// Close implements FragmentIterator.Close.
func (i *Iter) Close() error {
return nil
}
func (i *Iter) String() string {
return "fragmented-spans"
}

471
vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go generated vendored Normal file
View File

@@ -0,0 +1,471 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/manifest"
)
// Logger defines an interface for writing log messages.
type Logger interface {
Infof(format string, args ...interface{})
Fatalf(format string, args ...interface{})
}
// LevelIter provides a merged view of spans from sstables in a level.
// It takes advantage of level invariants to only have one sstable span block
// open at one time, opened using the newIter function passed in.
type LevelIter struct {
logger Logger
cmp base.Compare
// Denotes if this level iter should read point key spans (i.e. rangedels,
// or range keys. If key type is Point, no straddle spans are emitted between
// files, and point key bounds are used to find files instead of range key
// bounds.
//
// TODO(bilal): Straddle spans can safely be produced in rangedel mode once
// we can guarantee that we will never read sstables in a level that split
// user keys across them. This might be guaranteed in a future release, but
// as of CockroachDB 22.2 it is not guaranteed, so to be safe disable it when
// keyType == KeyTypePoint
keyType manifest.KeyType
// The LSM level this LevelIter is initialized for. Used in logging.
level manifest.Level
// The below fields are used to fill in gaps between adjacent files' range
// key spaces. This is an optimization to avoid unnecessarily loading files
// in cases where range keys are sparse and rare. dir is set by every
// positioning operation, straddleDir is set to dir whenever a straddling
// Span is synthesized, and straddle is Valid() whenever the last positioning
// operation returned a synthesized straddle span.
//
// Note that when a straddle span is initialized, iterFile is modified to
// point to the next file in the straddleDir direction. A change of direction
// on a straddle key therefore necessitates the value of iterFile to be
// reverted.
dir int
straddle Span
straddleDir int
// The iter for the current file. It is nil under any of the following conditions:
// - files.Current() == nil
// - err != nil
// - straddle.Valid(), in which case iterFile is not nil and points to the
// next file (in the straddleDir direction).
// - some other constraint, like the bounds in opts, caused the file at index to not
// be relevant to the iteration.
iter FragmentIterator
iterFile *manifest.FileMetadata
newIter TableNewSpanIter
files manifest.LevelIterator
err error
// The options that were passed in.
tableOpts SpanIterOptions
// TODO(bilal): Add InternalIteratorStats.
}
// LevelIter implements the keyspan.FragmentIterator interface.
var _ FragmentIterator = (*LevelIter)(nil)
// newLevelIter returns a LevelIter.
func newLevelIter(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
logger Logger,
keyType manifest.KeyType,
) *LevelIter {
l := &LevelIter{}
l.Init(opts, cmp, newIter, files, level, logger, keyType)
return l
}
// Init initializes a LevelIter.
func (l *LevelIter) Init(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
logger Logger,
keyType manifest.KeyType,
) {
l.err = nil
l.level = level
l.logger = logger
l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
l.cmp = cmp
l.iterFile = nil
l.newIter = newIter
switch keyType {
case manifest.KeyTypePoint, manifest.KeyTypeRange:
l.keyType = keyType
l.files = files.Filter(keyType)
default:
panic(fmt.Sprintf("unsupported key type: %v", keyType))
}
}
func (l *LevelIter) findFileGE(key []byte) *manifest.FileMetadata {
// Find the earliest file whose largest key is >= key.
//
// If the earliest file has its largest key == key and that largest key is a
// range deletion sentinel, we know that we manufactured this sentinel to convert
// the exclusive range deletion end key into an inclusive key (reminder: [start, end)#seqnum
// is the form of a range deletion sentinel which can contribute a largest key = end#sentinel).
// In this case we don't return this as the earliest file since there is nothing actually
// equal to key in it.
m := l.files.SeekGE(l.cmp, key)
for m != nil {
largestKey := m.LargestRangeKey
if l.keyType == manifest.KeyTypePoint {
largestKey = m.LargestPointKey
}
if !largestKey.IsExclusiveSentinel() || l.cmp(largestKey.UserKey, key) != 0 {
break
}
m = l.files.Next()
}
return m
}
func (l *LevelIter) findFileLT(key []byte) *manifest.FileMetadata {
// Find the last file whose smallest key is < key.
return l.files.SeekLT(l.cmp, key)
}
type loadFileReturnIndicator int8
const (
noFileLoaded loadFileReturnIndicator = iota
fileAlreadyLoaded
newFileLoaded
)
func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileReturnIndicator {
indicator := noFileLoaded
if l.iterFile == file {
if l.err != nil {
return noFileLoaded
}
if l.iter != nil {
// We are already at the file, but we would need to check for bounds.
// Set indicator accordingly.
indicator = fileAlreadyLoaded
}
// We were already at file, but don't have an iterator, probably because the file was
// beyond the iteration bounds. It may still be, but it is also possible that the bounds
// have changed. We handle that below.
}
// Note that LevelIter.Close() can be called multiple times.
if indicator != fileAlreadyLoaded {
if err := l.Close(); err != nil {
return noFileLoaded
}
}
l.iterFile = file
if file == nil {
return noFileLoaded
}
if indicator != fileAlreadyLoaded {
l.iter, l.err = l.newIter(l.files.Current(), &l.tableOpts)
indicator = newFileLoaded
}
if l.err != nil {
return noFileLoaded
}
return indicator
}
// SeekGE implements keyspan.FragmentIterator.
func (l *LevelIter) SeekGE(key []byte) Span {
l.dir = +1
l.err = nil // clear cached iteration error
f := l.findFileGE(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
// Return a straddling key instead of loading the file.
l.iterFile = f
l.iter = nil
l.straddleDir = +1
// The synthetic span that we are creating starts at the seeked key. This
// is an optimization as it prevents us from loading the adjacent file's
// bounds, at the expense of this iterator appearing "inconsistent" to its
// callers i.e.:
//
// SeekGE(bb) -> {bb-c, empty}
// Next() -> {c-d, RANGEKEYSET}
// Prev() -> {a-c, empty}
//
// Seeing as the inconsistency will only be around empty spans, which are
// expected to be elided by one of the higher-level iterators (either
// top-level Iterator or the defragmenting iter), the entire iterator should
// still appear consistent to the user.
l.straddle = Span{
Start: key,
End: f.SmallestRangeKey.UserKey,
Keys: nil,
}
return l.straddle
}
loadFileIndicator := l.loadFile(f, +1)
if loadFileIndicator == noFileLoaded {
return Span{}
}
if span := l.iter.SeekGE(key); span.Valid() {
return span
}
return l.skipEmptyFileForward()
}
// SeekLT implements keyspan.FragmentIterator.
func (l *LevelIter) SeekLT(key []byte) Span {
l.dir = -1
l.err = nil // clear cached iteration error
f := l.findFileLT(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
// Return a straddling key instead of loading the file.
l.iterFile = f
l.iter = nil
l.straddleDir = -1
// The synthetic span that we are creating ends at the seeked key. This
// is an optimization as it prevents us from loading the adjacent file's
// bounds, at the expense of this iterator appearing "inconsistent" to its
// callers i.e.:
//
// SeekLT(dd) -> {d-dd, empty}
// Prev() -> {c-d, RANGEKEYSET}
// Next() -> {d-e, empty}
//
// Seeing as the inconsistency will only be around empty spans, which are
// expected to be elided by one of the higher-level iterators (either
// top-level Iterator or the defragmenting iter), the entire iterator should
// still appear consistent to the user.
l.straddle = Span{
Start: f.LargestRangeKey.UserKey,
End: key,
Keys: nil,
}
return l.straddle
}
if l.loadFile(l.findFileLT(key), -1) == noFileLoaded {
return Span{}
}
if span := l.iter.SeekLT(key); span.Valid() {
return span
}
return l.skipEmptyFileBackward()
}
// First implements keyspan.FragmentIterator.
func (l *LevelIter) First() Span {
l.dir = +1
l.err = nil // clear cached iteration error
if l.loadFile(l.files.First(), +1) == noFileLoaded {
return Span{}
}
if span := l.iter.First(); span.Valid() {
return span
}
return l.skipEmptyFileForward()
}
// Last implements keyspan.FragmentIterator.
func (l *LevelIter) Last() Span {
l.dir = -1
l.err = nil // clear cached iteration error
if l.loadFile(l.files.Last(), -1) == noFileLoaded {
return Span{}
}
if span := l.iter.Last(); span.Valid() {
return span
}
return l.skipEmptyFileBackward()
}
// Next implements keyspan.FragmentIterator.
func (l *LevelIter) Next() Span {
l.dir = +1
if l.err != nil || (l.iter == nil && l.iterFile == nil) {
return Span{}
}
if l.iter != nil {
if span := l.iter.Next(); span.Valid() {
return span
}
}
return l.skipEmptyFileForward()
}
// Prev implements keyspan.FragmentIterator.
func (l *LevelIter) Prev() Span {
l.dir = -1
if l.err != nil || (l.iter == nil && l.iterFile == nil) {
return Span{}
}
if l.iter != nil {
if span := l.iter.Prev(); span.Valid() {
return span
}
}
return l.skipEmptyFileBackward()
}
func (l *LevelIter) skipEmptyFileForward() Span {
if !l.straddle.Valid() && l.keyType == manifest.KeyTypeRange &&
l.iterFile != nil && l.iter != nil {
// We were at a file that had spans. Check if the next file that has
// spans is not directly adjacent to the current file i.e. there is a
// gap in the span keyspace between the two files. In that case, synthesize
// a "straddle span" in l.straddle and return that.
//
// Straddle spans are not created in rangedel mode.
if err := l.Close(); err != nil {
l.err = err
return Span{}
}
startKey := l.iterFile.LargestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
// does not change the logic in loadFile() as long as l.iter is also nil;
// which it should be due to the Close() call above.
l.iterFile = l.files.Next()
if l.iterFile == nil {
return Span{}
}
endKey := l.iterFile.SmallestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
// There is a gap between the two files. Synthesize a straddling span
// to avoid unnecessarily loading the next file.
l.straddle = Span{
Start: startKey,
End: endKey,
}
l.straddleDir = l.dir
return l.straddle
}
} else if l.straddle.Valid() && l.straddleDir < 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
// was already moved backward by skipEmptyFileBackward, so advance it
// forward.
l.iterFile = l.files.Next()
}
l.straddle = Span{}
var span Span
for span.Empty() {
fileToLoad := l.iterFile
if l.keyType == manifest.KeyTypePoint {
// We haven't iterated to the next file yet if we're in point key
// (rangedel) mode.
fileToLoad = l.files.Next()
}
if l.loadFile(fileToLoad, +1) == noFileLoaded {
return Span{}
}
span = l.iter.First()
// In rangedel mode, we can expect to get empty files that we'd need to
// skip over, but not in range key mode.
if l.keyType == manifest.KeyTypeRange {
break
}
}
return span
}
func (l *LevelIter) skipEmptyFileBackward() Span {
// We were at a file that had spans. Check if the previous file that has
// spans is not directly adjacent to the current file i.e. there is a
// gap in the span keyspace between the two files. In that case, synthesize
// a "straddle span" in l.straddle and return that.
//
// Straddle spans are not created in rangedel mode.
if !l.straddle.Valid() && l.keyType == manifest.KeyTypeRange &&
l.iterFile != nil && l.iter != nil {
if err := l.Close(); err != nil {
l.err = err
return Span{}
}
endKey := l.iterFile.SmallestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
// does not change the logic in loadFile() as long as l.iter is also nil;
// which it should be due to the Close() call above.
l.iterFile = l.files.Prev()
if l.iterFile == nil {
return Span{}
}
startKey := l.iterFile.LargestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
// There is a gap between the two files. Synthesize a straddling span
// to avoid unnecessarily loading the next file.
l.straddle = Span{
Start: startKey,
End: endKey,
}
l.straddleDir = l.dir
return l.straddle
}
} else if l.straddle.Valid() && l.straddleDir > 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
// was already advanced forward by skipEmptyFileForward, so move it
// backward.
l.iterFile = l.files.Prev()
}
l.straddle = Span{}
var span Span
for span.Empty() {
fileToLoad := l.iterFile
if l.keyType == manifest.KeyTypePoint {
fileToLoad = l.files.Prev()
}
if l.loadFile(fileToLoad, -1) == noFileLoaded {
return Span{}
}
span = l.iter.Last()
// In rangedel mode, we can expect to get empty files that we'd need to
// skip over, but not in range key mode as the filter on the FileMetadata
// should guarantee we always get a non-empty file.
if l.keyType == manifest.KeyTypeRange {
break
}
}
return span
}
// Error implements keyspan.FragmentIterator.
func (l *LevelIter) Error() error {
if l.err != nil || l.iter == nil {
return l.err
}
return l.iter.Error()
}
// Close implements keyspan.FragmentIterator.
func (l *LevelIter) Close() error {
if l.iter != nil {
l.err = l.iter.Close()
l.iter = nil
}
return l.err
}
// String implements keyspan.FragmentIterator.
func (l *LevelIter) String() string {
if l.iterFile != nil {
return fmt.Sprintf("%s: fileNum=%s", l.level, l.iterFile.FileNum)
}
return fmt.Sprintf("%s: fileNum=<nil>", l.level)
}

948
vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go generated vendored Normal file
View File

@@ -0,0 +1,948 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"bytes"
"fmt"
"sort"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// TODO(jackson): Consider implementing an optimization to seek lower levels
// past higher levels' RANGEKEYDELs. This would be analaogous to the
// optimization pebble.mergingIter performs for RANGEDELs during point key
// seeks. It may not be worth it, because range keys are rare and cascading
// seeks would require introducing key comparisons to switchTo{Min,Max}Heap
// where there currently are none.
// Transform defines a transform function to be applied to a Span. A Transform
// takes a Span as input and writes the transformed Span to the provided output
// *Span pointer. The output Span's Keys slice may be reused by Transform to
// reduce allocations.
type Transform func(cmp base.Compare, in Span, out *Span) error
func noopTransform(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
}
// visibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func visibleTransform(snapshot uint64) Transform {
return func(_ base.Compare, s Span, dst *Span) error {
s = s.Visible(snapshot)
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
}
}
// MergingIter merges spans across levels of the LSM, exposing an iterator over
// spans that yields sets of spans fragmented at unique user key boundaries.
//
// A MergingIter is initialized with an arbitrary number of child iterators over
// fragmented spans. Each child iterator exposes fragmented key spans, such that
// overlapping keys are surfaced in a single Span. Key spans from one child
// iterator may overlap key spans from another child iterator arbitrarily.
//
// Algorithm
//
// The merging iterator wraps child iterators, merging and fragmenting spans
// across levels. The high-level algorithm is:
//
// 1. Initialize the heap with bound keys from child iterators' spans.
// 2. Find the next [or previous] two unique user keys' from bounds.
// 3. Consider the span formed between the two unique user keys a candidate
// span.
// 4. Determine if any of the child iterators' spans overlap the candidate
// span.
// 4a. If any of the child iterator's current bounds are end keys
// (during forward iteration) or start keys (during reverse
// iteration), then all the spans with that bound overlap the
// candidate span.
// 4b. Apply the configured transform, which may remove keys.
// 4b. If no spans overlap, forget the smallest (forward iteration)
// or largest (reverse iteration) unique user key and advance
// the iterators to the next unique user key. Start again from 3.
//
// Detailed algorithm
//
// Each level (i0, i1, ...) has a user-provided input FragmentIterator. The
// merging iterator steps through individual boundaries of the underlying
// spans separately. If the underlying FragmentIterator has fragments
// [a,b){#2,#1} [b,c){#1} the mergingIterLevel.{next,prev} step through:
//
// (a, start), (b, end), (b, start), (c, end)
//
// Note that (a, start) and (b, end) are observed ONCE each, despite two keys
// sharing those bounds. Also note that (b, end) and (b, start) are two distinct
// iterator positions of a mergingIterLevel.
//
// The merging iterator maintains a heap (min during forward iteration, max
// during reverse iteration) containing the boundKeys. Each boundKey is a
// 3-tuple holding the bound user key, whether the bound is a start or end key
// and the set of keys from that level that have that bound. The heap orders
// based on the boundKey's user key only.
//
// The merging iterator is responsible for merging spans across levels to
// determine which span is next, but it's also responsible for fragmenting
// overlapping spans. Consider the example:
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
//
// fragments: a-b-c-d-e-----h-----k----------p
//
// None of the individual child iterators contain a span with the exact bounds
// [c,d), but the merging iterator must produce a span [c,d). To accomplish
// this, the merging iterator visits every span between unique boundary user
// keys. In the above example, this is:
//
// [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p)
//
// The merging iterator first initializes the heap to prepare for iteration.
// The description below discusses the mechanics of forward iteration after a
// call to First, but the mechanics are similar for reverse iteration and
// other positioning methods.
//
// During a call to First, the heap is initialized by seeking every
// mergingIterLevel to the first bound of the first fragment. In the above
// example, this seeks the child iterators to:
//
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (a, boundKindFragmentStart, [ [a,c) ])
// i2: (a, boundKindFragmentStart, [ [a,p) ])
//
// After fixing up the heap, the root of the heap is a boundKey with the
// smallest user key ('a' in the example). Once the heap is setup for iteration
// in the appropriate direction and location, the merging iterator uses
// find{Next,Prev}FragmentSet to find the next/previous span bounds.
//
// During forward iteration, the root of the heap's user key is the start key
// key of next merged span. findNextFragmentSet sets m.start to this user
// key. The heap may contain other boundKeys with the same user key if another
// level has a fragment starting or ending at the same key, so the
// findNextFragmentSet method pulls from the heap until it finds the first key
// greater than m.start. This key is used as the end key.
//
// In the above example, this results in m.start = 'a', m.end = 'b' and child
// iterators in the following positions:
//
// i0: (b, boundKindFragmentStart, [ [b,d) ])
// i1: (c, boundKindFragmentEnd, [ [a,c) ])
// i2: (p, boundKindFragmentEnd, [ [a,p) ])
//
// With the user key bounds of the next merged span established,
// findNextFragmentSet must determine which, if any, fragments overlap the span.
// During forward iteration any child iterator that is now positioned at an end
// boundary has an overlapping span. (Justification: The child iterator's end
// boundary is ≥ m.end. The corresponding start boundary must be ≤ m.start since
// there were no other user keys between m.start and m.end. So the fragments
// associated with the iterator's current end boundary have start and end bounds
// such that start ≤ m.start < m.end ≤ end).
//
// findNextFragmentSet iterates over the levels, collecting keys from any child
// iterators positioned at end boundaries. In the above example, i1 and i2 are
// positioned at end boundaries, so findNextFragmentSet collects the keys of
// [a,c) and [a,p). These spans contain the merging iterator's [m.start, m.end)
// span, but they may also extend beyond the m.start and m.end. The merging
// iterator returns the keys with the merging iter's m.start and m.end bounds,
// preserving the underlying keys' sequence numbers, key kinds and values.
//
// A MergingIter is configured with a Transform that's applied to the span
// before surfacing it to the iterator user. A Transform may remove keys
// arbitrarily, but it may not modify the values themselves.
//
// It may be the case that findNextFragmentSet finds no levels positioned at end
// boundaries, or that there are no spans remaining after applying a transform,
// in which case the span [m.start, m.end) overlaps with nothing. In this case
// findNextFragmentSet loops, repeating the above process again until it finds a
// span that does contain keys.
//
// Memory safety
//
// The FragmentIterator interface only guarantees stability of a Span and its
// associated slices until the next positioning method is called. Adjacent Spans
// may be contained in different sstables, requring the FragmentIterator
// implementation to close one sstable, releasing its memory, before opening the
// next. Most of the state used by the MergingIter is derived from spans at
// current child iterator positions only, ensuring state is stable. The one
// exception is the start bound during forward iteration and the end bound
// during reverse iteration.
//
// If the heap root originates from an end boundary when findNextFragmentSet
// begins, a Next on the heap root level may invalidate the end boundary. To
// accommodate this, find{Next,Prev}FragmentSet copy the initial boundary if the
// subsequent Next/Prev would move to the next span.
type MergingIter struct {
levels []mergingIterLevel
heap mergingIterHeap
// start and end hold the bounds for the span currently under the
// iterator position.
//
// Invariant: None of the levels' iterators contain spans with a bound
// between start and end. For all bounds b, b ≤ start || b ≥ end.
start, end []byte
// buf is a buffer used to save [start, end) boundary keys.
buf []byte
// keys holds all of the keys across all levels that overlap the key span
// [start, end), sorted by sequence number and kind descending. This slice
// is reconstituted in synthesizeKeys from each mergingIterLevel's keys
// every time the [start, end) bounds change.
//
// Each element points into a child iterator's memory, so the keys may not
// be directly modified.
keys keysBySeqNumKind
// transform defines a function to be applied to a span before it's yielded
// to the user. A transform may filter individual keys contained within the
// span.
transform Transform
// span holds the iterator's current span. This span is used as the
// destination for transforms. Every tranformed span overwrites the
// previous.
span Span
err error
dir int8
}
// MergingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*MergingIter)(nil)
type mergingIterLevel struct {
iter FragmentIterator
// heapKey holds the current key at this level for use within the heap.
heapKey boundKey
}
func (l *mergingIterLevel) next() {
if l.heapKey.kind == boundKindFragmentStart {
l.heapKey = endBound(l.heapKey.span)
return
}
l.heapKey = startBound(l.iter.Next())
}
func (l *mergingIterLevel) prev() {
if l.heapKey.kind == boundKindFragmentEnd {
l.heapKey = startBound(l.heapKey.span)
return
}
l.heapKey = endBound(l.iter.Prev())
}
// Init initializes the merging iterator with the provided fragment iterators.
func (m *MergingIter) Init(cmp base.Compare, transform Transform, iters ...FragmentIterator) {
levels, items := m.levels, m.heap.items
*m = MergingIter{
heap: mergingIterHeap{cmp: cmp},
transform: transform,
}
// Invariant: cap(levels) == cap(items)
if cap(levels) < len(iters) {
m.levels = make([]mergingIterLevel, len(iters))
m.heap.items = make([]mergingIterItem, 0, len(iters))
} else {
m.levels = levels[:len(iters)]
m.heap.items = items[:0]
}
for i := range m.levels {
m.levels[i] = mergingIterLevel{}
m.levels[i].iter = iters[i]
}
}
// SeekGE moves the iterator to the first span with a start key greater than or
// equal to key.
func (m *MergingIter) SeekGE(key []byte) Span {
m.invalidate() // clear state about current position
for i := range m.levels {
l := &m.levels[i]
// A SeekGE requires we position each level at the smallest bound ≥ key.
// We must search through both inclusive start and exclusive end bounds.
// Note that this search requirement differs from FragmentIterator's
// .SeekGE'semantics, which returns the span with the smallest start key
// ≥ key. To remedy this difference, we find the last span less than
// key. If its end boundary is greater than or equal to key, we use it.
// Otherwise we use the start boundary of the next span which
// necessarily has a start ≥ key.
s := l.iter.SeekLT(key)
if s.Valid() && m.cmp(s.End, key) >= 0 {
// s.End ≥ key
// We need to use this span's end bound.
l.heapKey = endBound(s)
continue
}
// s.End < key
// The span `s` ends before key. Next to the first span with a Start ≥
// key, and use that.
s = l.iter.Next()
l.heapKey = startBound(s)
}
m.initMinHeap()
return m.findNextFragmentSet()
}
// SeekLT moves the iterator to the last span with a start key less than key.
func (m *MergingIter) SeekLT(key []byte) Span {
// TODO(jackson): Evaluate whether there's an implementation of SeekLT
// independent of SeekGE that is more efficient. It's tricky, because the
// span we should return might straddle `key` itself.
//
// Consider the scenario:
// a----------l #2
// b-----------m #1
//
// The merged, fully-fragmented spans that MergingIter exposes to the caller
// have bounds:
// a-b #2
// b--------l #2
// b--------l #1
// l-m #1
//
// A call SeekLT(c) must return the largest of the above spans with a
// Start user key < key: [b,l)#1. This requires examining bounds both < 'c'
// (the 'b' of [b,m)#1's start key) and bounds ≥ 'c' (the 'l' of ([a,l)#2's
// end key).
if s := m.SeekGE(key); !s.Valid() && m.err != nil {
return Span{}
}
// Prev to the previous span.
return m.Prev()
}
// First seeks the iterator to the first span.
func (m *MergingIter) First() Span {
m.invalidate() // clear state about current position
for i := range m.levels {
l := &m.levels[i]
l.heapKey = startBound(l.iter.First())
}
m.initMinHeap()
return m.findNextFragmentSet()
}
// Last seeks the iterator to the last span.
func (m *MergingIter) Last() Span {
m.invalidate() // clear state about current position
for i := range m.levels {
l := &m.levels[i]
l.heapKey = endBound(l.iter.Last())
}
m.initMaxHeap()
return m.findPrevFragmentSet()
}
// Next advances the iterator to the next span.
func (m *MergingIter) Next() Span {
if m.err != nil {
return Span{}
}
if m.dir == +1 && (m.end == nil || m.start == nil) {
return Span{}
}
if m.dir != +1 {
m.switchToMinHeap()
}
return m.findNextFragmentSet()
}
// Prev advances the iterator to the previous span.
func (m *MergingIter) Prev() Span {
if m.err != nil {
return Span{}
}
if m.dir == -1 && (m.end == nil || m.start == nil) {
return Span{}
}
if m.dir != -1 {
m.switchToMaxHeap()
}
return m.findPrevFragmentSet()
}
// Error returns any accumulated error.
func (m *MergingIter) Error() error {
if m.heap.len() == 0 || m.err != nil {
return m.err
}
return m.levels[m.heap.items[0].index].iter.Error()
}
// Close closes the iterator, releasing all acquired resources.
func (m *MergingIter) Close() error {
for i := range m.levels {
if err := m.levels[i].iter.Close(); err != nil && m.err == nil {
m.err = err
}
}
m.levels = nil
m.heap.items = m.heap.items[:0]
return m.err
}
// String implements fmt.Stringer.
func (m *MergingIter) String() string {
return "merging-keyspan"
}
func (m *MergingIter) initMinHeap() {
m.dir = +1
m.heap.reverse = false
m.initHeap()
}
func (m *MergingIter) initMaxHeap() {
m.dir = -1
m.heap.reverse = true
m.initHeap()
}
func (m *MergingIter) initHeap() {
m.heap.items = m.heap.items[:0]
for i := range m.levels {
if l := &m.levels[i]; l.heapKey.kind != boundKindInvalid {
m.heap.items = append(m.heap.items, mergingIterItem{
index: i,
boundKey: &l.heapKey,
})
} else {
m.err = firstError(m.err, l.iter.Error())
if m.err != nil {
return
}
}
}
m.heap.init()
}
func (m *MergingIter) switchToMinHeap() {
// switchToMinHeap reorients the heap for forward iteration, without moving
// the current MergingIter position.
// The iterator is currently positioned at the span [m.start, m.end),
// oriented in the reverse direction, so each level's iterator is positioned
// to the largest key ≤ m.start. To reorient in the forward direction, we
// must advance each level's iterator to the smallest key ≥ m.end. Consider
// this three-level example.
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
//
// merged: a-b-c-d-e-----h-----k----------p
//
// If currently positioned at the merged span [c,d), then the level
// iterators' heap keys are:
//
// i0: (b, [b, d)) i1: (c, [a,c)) i2: (a, [a,p))
//
// Reversing the heap should not move the merging iterator and should not
// change the current [m.start, m.end) bounds. It should only prepare for
// forward iteration by updating the child iterators' heap keys to:
//
// i0: (d, [b, d)) i1: (h, [h,k)) i2: (p, [a,p))
//
// In every level the first key ≥ m.end is the next in the iterator.
// Justification: Suppose not and a level iterator's next key was some key k
// such that k < m.end. The max-heap invariant dictates that the current
// iterator position is the largest entry with a user key ≥ m.start. This
// means k > m.start. We started with the assumption that k < m.end, so
// m.start < k < m.end. But then k is between our current span bounds,
// and reverse iteration would have constructed the current interval to be
// [k, m.end) not [m.start, m.end).
if invariants.Enabled {
for i := range m.levels {
l := &m.levels[i]
if l.heapKey.kind != boundKindInvalid && m.cmp(l.heapKey.key, m.start) > 0 {
panic("pebble: invariant violation: max-heap key > m.start")
}
}
}
for i := range m.levels {
m.levels[i].next()
}
m.initMinHeap()
}
func (m *MergingIter) switchToMaxHeap() {
// switchToMaxHeap reorients the heap for reverse iteration, without moving
// the current MergingIter position.
// The iterator is currently positioned at the span [m.start, m.end),
// oriented in the forward direction. Each level's iterator is positioned at
// the smallest bound ≥ m.end. To reorient in the reverse direction, we must
// move each level's iterator to the largest key ≤ m.start. Consider this
// three-level example.
//
// i0: b---d e-----h
// i1: a---c h-----k
// i2: a------------------------------p
//
// merged: a-b-c-d-e-----h-----k----------p
//
// If currently positioned at the merged span [c,d), then the level
// iterators' heap keys are:
//
// i0: (d, [b, d)) i1: (h, [h,k)) i2: (p, [a,p))
//
// Reversing the heap should not move the merging iterator and should not
// change the current [m.start, m.end) bounds. It should only prepare for
// reverse iteration by updating the child iterators' heap keys to:
//
// i0: (b, [b, d)) i1: (c, [a,c)) i2: (a, [a,p))
//
// In every level the largest key ≤ m.start is the prev in the iterator.
// Justification: Suppose not and a level iterator's prev key was some key k
// such that k > m.start. The min-heap invariant dictates that the current
// iterator position is the smallest entry with a user key ≥ m.end. This
// means k < m.end, otherwise the iterator would be positioned at k. We
// started with the assumption that k > m.start, so m.start < k < m.end. But
// then k is between our current span bounds, and reverse iteration
// would have constructed the current interval to be [m.start, k) not
// [m.start, m.end).
if invariants.Enabled {
for i := range m.levels {
l := &m.levels[i]
if l.heapKey.kind != boundKindInvalid && m.cmp(l.heapKey.key, m.end) < 0 {
panic("pebble: invariant violation: min-heap key < m.end")
}
}
}
for i := range m.levels {
m.levels[i].prev()
}
m.initMaxHeap()
}
func (m *MergingIter) cmp(a, b []byte) int {
return m.heap.cmp(a, b)
}
func (m *MergingIter) findNextFragmentSet() Span {
// Each iteration of this loop considers a new merged span between unique
// user keys. An iteration may find that there exists no overlap for a given
// span, (eg, if the spans [a,b), [d, e) exist within level iterators, the
// below loop will still consider [b,d) before continuing to [d, e)). It
// returns when it finds a span that is covered by at least one key.
for m.heap.len() > 0 && m.err == nil {
// Initialize the next span's start bound. SeekGE and First prepare the
// heap without advancing. Next leaves the heap in a state such that the
// root is the smallest bound key equal to the returned span's end key,
// so the heap is already positioned at the next merged span's start key.
// NB: m.heapRoot() might be either an end boundary OR a start boundary
// of a level's span. Both end and start boundaries may still be a start
// key of a span in the set of fragmented spans returned by MergingIter.
// Consider the scenario:
// a----------l #1
// b-----------m #2
//
// The merged, fully-fragmented spans that MergingIter exposes to the caller
// have bounds:
// a-b #1
// b--------l #1
// b--------l #2
// l-m #2
//
// When advancing to l-m#2, we must set m.start to 'l', which originated
// from [a,l)#1's end boundary.
m.start = m.heap.items[0].boundKey.key
// Before calling nextEntry, consider whether it might invalidate our
// start boundary. If the start boundary key originated from an end
// boundary, then we need to copy the start key before advancing the
// underlying iterator to the next Span.
if m.heap.items[0].boundKey.kind == boundKindFragmentEnd {
m.buf = append(m.buf[:0], m.start...)
m.start = m.buf
}
// There may be many entries all with the same user key. Spans in other
// levels may also start or end at this same user key. For eg:
// L1: [a, c) [c, d)
// L2: [c, e)
// If we're positioned at L1's end(c) end boundary, we want to advance
// to the first bound > c.
m.nextEntry()
for len(m.heap.items) > 0 && m.err == nil && m.cmp(m.heapRoot(), m.start) == 0 {
m.nextEntry()
}
if len(m.heap.items) == 0 || m.err != nil {
break
}
// The current entry at the top of the heap is the first key > m.start.
// It must become the end bound for the span we will return to the user.
// In the above example, the root of the heap is L1's end(d).
m.end = m.heap.items[0].boundKey.key
// Each level within m.levels may have a span that overlaps the
// fragmented key span [m.start, m.end). Update m.keys to point to them
// and sort them by kind, sequence number. There may not be any keys
// defined over [m.start, m.end) if we're between the end of one span
// and the start of the next, OR if the configured transform filters any
// keys out. We allow empty spans that were emitted by child iterators, but
// we elide empty spans created by the mergingIter itself that don't overlap
// with any child iterator returned spans (i.e. empty spans that bridge two
// distinct child-iterator-defined spans).
if found, s := m.synthesizeKeys(+1); found && s.Valid() {
return s
}
}
// Exhausted.
m.clear()
return Span{}
}
func (m *MergingIter) findPrevFragmentSet() Span {
// Each iteration of this loop considers a new merged span between unique
// user keys. An iteration may find that there exists no overlap for a given
// span, (eg, if the spans [a,b), [d, e) exist within level iterators, the
// below loop will still consider [b,d) before continuing to [a, b)). It
// returns when it finds a span that is covered by at least one key.
for m.heap.len() > 0 && m.err == nil {
// Initialize the next span's end bound. SeekLT and Last prepare the
// heap without advancing. Prev leaves the heap in a state such that the
// root is the largest bound key equal to the returned span's start key,
// so the heap is already positioned at the next merged span's end key.
// NB: m.heapRoot() might be either an end boundary OR a start boundary
// of a level's span. Both end and start boundaries may still be a start
// key of a span returned by MergingIter. Consider the scenario:
// a----------l #2
// b-----------m #1
//
// The merged, fully-fragmented spans that MergingIter exposes to the caller
// have bounds:
// a-b #2
// b--------l #2
// b--------l #1
// l-m #1
//
// When Preving to a-b#2, we must set m.end to 'b', which originated
// from [b,m)#1's start boundary.
m.end = m.heap.items[0].boundKey.key
// Before calling prevEntry, consider whether it might invalidate our
// end boundary. If the end boundary key originated from a start
// boundary, then we need to copy the end key before advancing the
// underlying iterator to the previous Span.
if m.heap.items[0].boundKey.kind == boundKindFragmentStart {
m.buf = append(m.buf[:0], m.end...)
m.end = m.buf
}
// There may be many entries all with the same user key. Spans in other
// levels may also start or end at this same user key. For eg:
// L1: [a, c) [c, d)
// L2: [c, e)
// If we're positioned at L1's start(c) start boundary, we want to prev
// to move to the first bound < c.
m.prevEntry()
for len(m.heap.items) > 0 && m.err == nil && m.cmp(m.heapRoot(), m.end) == 0 {
m.prevEntry()
}
if len(m.heap.items) == 0 || m.err != nil {
break
}
// The current entry at the top of the heap is the first key < m.end.
// It must become the start bound for the span we will return to the
// user. In the above example, the root of the heap is L1's start(a).
m.start = m.heap.items[0].boundKey.key
// Each level within m.levels may have a set of keys that overlap the
// fragmented key span [m.start, m.end). Update m.keys to point to them
// and sort them by kind, sequence number. There may not be any keys
// spanning [m.start, m.end) if we're between the end of one span and
// the start of the next, OR if the configured transform filters any
// keys out. We allow empty spans that were emitted by child iterators, but
// we elide empty spans created by the mergingIter itself that don't overlap
// with any child iterator returned spans (i.e. empty spans that bridge two
// distinct child-iterator-defined spans).
if found, s := m.synthesizeKeys(-1); found && s.Valid() {
return s
}
}
// Exhausted.
m.clear()
return Span{}
}
func (m *MergingIter) heapRoot() []byte {
return m.heap.items[0].boundKey.key
}
// synthesizeKeys is called by find{Next,Prev}FragmentSet to populate and
// sort the set of keys overlapping [m.start, m.end).
//
// During forward iteration, if the current heap item is a fragment end,
// then the fragment's start must be ≤ m.start and the fragment overlaps the
// current iterator position of [m.start, m.end).
//
// During reverse iteration, if the current heap item is a fragment start,
// then the fragment's end must be ≥ m.end and the fragment overlaps the
// current iteration position of [m.start, m.end).
//
// The boolean return value, `found`, is true if the returned span overlaps
// with a span returned by a child iterator.
func (m *MergingIter) synthesizeKeys(dir int8) (bool, Span) {
if invariants.Enabled {
if m.cmp(m.start, m.end) >= 0 {
panic(fmt.Sprintf("pebble: invariant violation: span start ≥ end: %s >= %s", m.start, m.end))
}
}
m.keys = m.keys[:0]
found := false
for i := range m.levels {
if dir == +1 && m.levels[i].heapKey.kind == boundKindFragmentEnd ||
dir == -1 && m.levels[i].heapKey.kind == boundKindFragmentStart {
m.keys = append(m.keys, m.levels[i].heapKey.span.Keys...)
found = true
}
}
sort.Sort(&m.keys)
// Apply the configured transform. See VisibleTransform.
s := Span{
Start: m.start,
End: m.end,
Keys: m.keys,
}
if err := m.transform(m.cmp, s, &m.span); err != nil {
m.err = err
return false, Span{}
}
return found, m.span
}
func (m *MergingIter) invalidate() {
m.err = nil
}
func (m *MergingIter) clear() {
for fi := range m.keys {
m.keys[fi] = Key{}
}
m.keys = m.keys[:0]
}
// nextEntry steps to the next entry.
func (m *MergingIter) nextEntry() {
l := &m.levels[m.heap.items[0].index]
l.next()
if !l.heapKey.valid() {
// l.iter is exhausted.
m.err = l.iter.Error()
if m.err == nil {
m.heap.pop()
}
return
}
if m.heap.len() > 1 {
m.heap.fix(0)
}
}
// prevEntry steps to the previous entry.
func (m *MergingIter) prevEntry() {
l := &m.levels[m.heap.items[0].index]
l.prev()
if !l.heapKey.valid() {
// l.iter is exhausted.
m.err = l.iter.Error()
if m.err == nil {
m.heap.pop()
}
return
}
if m.heap.len() > 1 {
m.heap.fix(0)
}
}
// DebugString returns a string representing the current internal state of the
// merging iterator and its heap for debugging purposes.
func (m *MergingIter) DebugString() string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "Current bounds: [%q, %q)\n", m.start, m.end)
for i := range m.levels {
fmt.Fprintf(&buf, "%d: heap key %s\n", i, m.levels[i].heapKey)
}
return buf.String()
}
type mergingIterItem struct {
// boundKey points to the corresponding mergingIterLevel's `iterKey`.
*boundKey
// index is the index of this level within the MergingIter's levels field.
index int
}
// mergingIterHeap is copied from mergingIterHeap defined in the root pebble
// package for use with point keys.
type mergingIterHeap struct {
cmp base.Compare
reverse bool
items []mergingIterItem
}
func (h *mergingIterHeap) len() int {
return len(h.items)
}
func (h *mergingIterHeap) less(i, j int) bool {
// This key comparison only uses the user key and not the boundKind. Bound
// kind doesn't matter because when stepping over a user key,
// findNextFragmentSet and findPrevFragmentSet skip past all heap items with
// that user key, and makes no assumptions on ordering. All other heap
// examinations only consider the user key.
ik, jk := h.items[i].key, h.items[j].key
c := h.cmp(ik, jk)
if h.reverse {
return c > 0
}
return c < 0
}
func (h *mergingIterHeap) swap(i, j int) {
h.items[i], h.items[j] = h.items[j], h.items[i]
}
// init, fix, up and down are copied from the go stdlib.
func (h *mergingIterHeap) init() {
// heapify
n := h.len()
for i := n/2 - 1; i >= 0; i-- {
h.down(i, n)
}
}
func (h *mergingIterHeap) fix(i int) {
if !h.down(i, h.len()) {
h.up(i)
}
}
func (h *mergingIterHeap) pop() *mergingIterItem {
n := h.len() - 1
h.swap(0, n)
h.down(0, n)
item := &h.items[n]
h.items = h.items[:n]
return item
}
func (h *mergingIterHeap) up(j int) {
for {
i := (j - 1) / 2 // parent
if i == j || !h.less(j, i) {
break
}
h.swap(i, j)
j = i
}
}
func (h *mergingIterHeap) down(i0, n int) bool {
i := i0
for {
j1 := 2*i + 1
if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
break
}
j := j1 // left child
if j2 := j1 + 1; j2 < n && h.less(j2, j1) {
j = j2 // = 2*i + 2 // right child
}
if !h.less(j, i) {
break
}
h.swap(i, j)
i = j
}
return i > i0
}
func startBound(s Span) boundKey {
if !s.Valid() {
return boundKey{kind: boundKindInvalid}
}
return boundKey{
kind: boundKindFragmentStart,
key: s.Start,
span: s,
}
}
func endBound(s Span) boundKey {
if !s.Valid() {
return boundKey{kind: boundKindInvalid}
}
return boundKey{
kind: boundKindFragmentEnd,
key: s.End,
span: s,
}
}
type boundKind int8
const (
boundKindInvalid boundKind = iota
boundKindFragmentStart
boundKindFragmentEnd
)
type boundKey struct {
kind boundKind
key []byte
// span holds the span the bound key comes from.
//
// If kind is boundKindFragmentStart, then key is span.Start. If kind is
// boundKindFragmentEnd, then key is span.End.
span Span
}
func (k boundKey) valid() bool {
return k.kind != boundKindInvalid
}
func (k boundKey) String() string {
var buf bytes.Buffer
switch k.kind {
case boundKindInvalid:
fmt.Fprint(&buf, "invalid")
case boundKindFragmentStart:
fmt.Fprint(&buf, "fragment-start")
case boundKindFragmentEnd:
fmt.Fprint(&buf, "fragment-end ")
default:
fmt.Fprintf(&buf, "unknown-kind(%d)", k.kind)
}
fmt.Fprintf(&buf, " %s [", k.key)
fmt.Fprintf(&buf, "%s", k.span)
fmt.Fprint(&buf, "]")
return buf.String()
}

98
vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go generated vendored Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// TODO(jackson): Move snapshot visiblity outside of SeekGE/SeekLE and into the
// callsite. This should simplify the code some more.
// SeekGE seeks to the span that contains the target key or the first span past
// the target key. The snapshot parameter controls the visibility of keys (only
// spans older than the snapshot sequence number are visible).
func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were to
// use SeekGE then the search key `c` would land on the span [e,i) and
// we'd have to backtrack. The one complexity here is what happens for the
// search key `e`. In that case SeekLT will land us on the span [a,e)
// and we'll have to move forward.
iterSpan := iter.SeekLT(key)
// Invariant: key > iterSpan.Start
if !iterSpan.Valid() || cmp(key, iterSpan.End) >= 0 {
// The current span lies entirely before the search key, or the iterator
// is exhausted. Advance the iterator to the next span which is
// guaranteed to lie at or past the search key.
iterSpan = iter.Next()
}
// The iterator is positioned at a valid iterSpan which is the first
// iterator position that satisfies the requirement that it contains or is
// past the target key. It may not be visible based on the snapshot. So now
// we only need to move forward and return the first span with visible keys.
iterSpan = iterSpan.Visible(snapshot)
for iterSpan.Valid() && iterSpan.Empty() {
iterSpan = iter.Next().Visible(snapshot)
}
return iterSpan
}
// SeekLE seeks to the span that contains or is before the target key. The
// snapshot parameter controls the visibility of keys (only spans older than the
// snapshot sequence number are visible).
func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte, snapshot uint64) Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were to
// use SeekGE then the search key `c` would land on the span [e,i) and
// we'd have to backtrack. The one complexity here is what happens for the
// search key `e`. In that case SeekLT will land us on the span [a,e)
// and we'll have to move forward.
iterSpan := iter.SeekLT(key)
if !iterSpan.Valid() {
// Advance the iterator once to see if the next span has a start key
// equal to key.
iterSpan = iter.Next()
if !iterSpan.Valid() || cmp(key, iterSpan.Start) < 0 {
// The iterator is exhausted or we've hit the next span.
return Span{}
}
} else {
// Invariant: key > iterSpan.Start
if cmp(key, iterSpan.End) >= 0 {
// The current span lies entirely before the search key. Check to see if
// the next span contains the search key. If it doesn't, we'll backup
// and return to our earlier candidate.
iterSpan = iter.Next()
if !iterSpan.Valid() || cmp(key, iterSpan.Start) < 0 {
// The next span is past our search key or there is no next span. Go
// back.
iterSpan = iter.Prev()
}
}
}
// We're positioned at a span that contains or is before the search key. It
// may not be visible based on the snapshot. So now we only need to move
// backward and return the first span with visible keys.
iterSpan = iterSpan.Visible(snapshot)
for iterSpan.Valid() && iterSpan.Empty() {
iterSpan = iter.Prev().Visible(snapshot)
}
return iterSpan
}

278
vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go generated vendored Normal file
View File

@@ -0,0 +1,278 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan // import "github.com/cockroachdb/pebble/internal/keyspan"
import (
"fmt"
"sort"
"strconv"
"strings"
"unicode"
"github.com/cockroachdb/pebble/internal/base"
)
// Span represents a set of keys over a span of user key space. All of the keys
// within a Span are applied across the span's key span indicated by Start and
// End. Each internal key applied over the user key span appears as a separate
// Key, with its own kind and sequence number. Optionally, each Key may also
// have a Suffix and/or Value.
//
// Note that the start user key is inclusive and the end user key is exclusive.
//
// Currently the only supported key kinds are:
// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
type Span struct {
// Start and End encode the user key range of all the contained items, with
// an inclusive start key and exclusive end key. Both Start and End must be
// non-nil, or both nil if representing an invalid Span.
Start, End []byte
// Keys holds the set of keys applied over the [Start, End) user key range.
// Keys is sorted by (SeqNum, Kind) descending. If SeqNum and Kind are
// equal, the order of Keys is undefined. Keys may be empty, even if Start
// and End are non-nil.
//
// Keys are a decoded representation of the internal keys stored in batches
// or sstable blocks. A single internal key in a range key block may produce
// several decoded Keys.
Keys []Key
}
// Key represents a single key applied over a span of user keys. A Key is
// contained by a Span which specifies the span of user keys over which the Key
// is applied.
type Key struct {
// Trailer contains the key kind and sequence number.
Trailer uint64
// Suffix holds an optional suffix associated with the key. This is only
// non-nil for RANGEKEYSET and RANGEKEYUNSET keys.
Suffix []byte
// Value holds a logical value associated with the Key. It is NOT the
// internal value stored in a range key or range deletion block. This is
// only non-nil for RANGEKEYSET keys.
Value []byte
}
// SeqNum returns the sequence number component of the key.
func (k Key) SeqNum() uint64 {
return k.Trailer >> 8
}
// Kind returns the kind component of the key.
func (k Key) Kind() base.InternalKeyKind {
return base.InternalKeyKind(k.Trailer & 0xff)
}
// Valid returns true if the span is defined.
func (s Span) Valid() bool {
return s.Start != nil && s.End != nil
}
// Empty returns true if the span does not contain any keys. An empty span may
// still be Valid. A non-empty span must be Valid.
//
// An Empty span may be produced by Visible, or be produced by iterators in
// order to surface the gaps between keys.
func (s Span) Empty() bool {
return len(s.Keys) == 0
}
// SmallestKey returns the smallest internal key defined by the span's keys.
// It panics if the span contains no keys.
func (s Span) SmallestKey() base.InternalKey {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
}
// The first key has the highest (sequence number,kind) tuple.
return base.InternalKey{
UserKey: s.Start,
Trailer: s.Keys[0].Trailer,
}
}
// LargestKey returns the largest internal key defined by the span's keys. The
// returned key will always be a "sentinel key" at the end boundary. The
// "sentinel key" models the exclusive end boundary by returning an InternalKey
// with the maximal sequence number, ensuring all InternalKeys with the same
// user key sort after the sentinel key.
//
// It panics if the span contains no keys.
func (s Span) LargestKey() base.InternalKey {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
}
// The last key has the lowest (sequence number,kind) tuple.
kind := s.Keys[len(s.Keys)-1].Kind()
return base.MakeExclusiveSentinelKey(kind, s.End)
}
// SmallestSeqNum returns the smallest sequence number of a key contained within
// the span. It panics if the span contains no keys.
func (s Span) SmallestSeqNum() uint64 {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
}
return s.Keys[len(s.Keys)-1].SeqNum()
}
// LargestSeqNum returns the largest sequence number of a key contained within
// the span. It panics if the span contains no keys.
func (s Span) LargestSeqNum() uint64 {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
}
return s.Keys[0].SeqNum()
}
// Visible returns a span with the subset of keys visible at the provided
// sequence number.
func (s Span) Visible(seqNum uint64) Span {
ret := Span{Start: s.Start, End: s.End}
for i := range s.Keys {
if base.Visible(s.Keys[i].SeqNum(), seqNum) {
ret.Keys = s.Keys[i:]
break
}
}
return ret
}
// ShallowClone returns the span with a Keys slice owned by the span itself.
// None of the key byte slices are cloned (see Span.DeepClone).
func (s Span) ShallowClone() Span {
c := Span{
Start: s.Start,
End: s.End,
Keys: make([]Key, len(s.Keys)),
}
copy(c.Keys, s.Keys)
return c
}
// DeepClone clones the span, creating copies of all contained slices. DeepClone
// is intended for non-production code paths like tests, the level checker, etc
// because it is allocation heavy.
func (s Span) DeepClone() Span {
c := Span{
Start: make([]byte, len(s.Start)),
End: make([]byte, len(s.End)),
Keys: make([]Key, len(s.Keys)),
}
copy(c.Start, s.Start)
copy(c.End, s.End)
for i := range s.Keys {
c.Keys[i].Trailer = s.Keys[i].Trailer
if len(s.Keys[i].Suffix) > 0 {
c.Keys[i].Suffix = make([]byte, len(s.Keys[i].Suffix))
copy(c.Keys[i].Suffix, s.Keys[i].Suffix)
}
if len(s.Keys[i].Value) > 0 {
c.Keys[i].Value = make([]byte, len(s.Keys[i].Value))
copy(c.Keys[i].Value, s.Keys[i].Value)
}
}
return c
}
// Contains returns true if the specified key resides within the span's bounds.
func (s Span) Contains(cmp base.Compare, key []byte) bool {
return cmp(s.Start, key) <= 0 && cmp(key, s.End) < 0
}
// Covers returns true if the span covers keys at seqNum.
func (s Span) Covers(seqNum uint64) bool {
return !s.Empty() && s.Keys[0].SeqNum() > seqNum
}
// String returns a string representation of the span.
func (s Span) String() string {
return fmt.Sprint(prettySpan{Span: s, formatKey: base.DefaultFormatter})
}
// Pretty returns a formatter for the span.
func (s Span) Pretty(f base.FormatKey) fmt.Formatter {
return prettySpan{s, f}
}
type prettySpan struct {
Span
formatKey base.FormatKey
}
func (s prettySpan) Format(fs fmt.State, c rune) {
if !s.Valid() {
fmt.Fprintf(fs, "<invalid>")
return
}
fmt.Fprintf(fs, "%s-%s:{", s.formatKey(s.Start), s.formatKey(s.End))
for i, k := range s.Keys {
if i > 0 {
fmt.Fprint(fs, " ")
}
fmt.Fprintf(fs, "(#%d,%s", k.SeqNum(), k.Kind())
if len(k.Suffix) > 0 || len(k.Value) > 0 {
fmt.Fprintf(fs, ",%s", k.Suffix)
}
if len(k.Value) > 0 {
fmt.Fprintf(fs, ",%s", k.Value)
}
fmt.Fprint(fs, ")")
}
fmt.Fprintf(fs, "}")
}
// SortKeys sorts a keys slice by trailer.
func SortKeys(keys []Key) {
sorted := keysBySeqNumKind(keys)
sort.Sort(&sorted)
}
// ParseSpan parses the string representation of a Span. It's intended for
// tests. ParseSpan panics if passed a malformed span representation.
func ParseSpan(input string) Span {
var s Span
parts := strings.FieldsFunc(input, func(r rune) bool {
switch r {
case '-', ':', '{', '}':
return true
default:
return unicode.IsSpace(r)
}
})
s.Start, s.End = []byte(parts[0]), []byte(parts[1])
// Each of the remaining parts represents a single Key.
s.Keys = make([]Key, 0, len(parts)-2)
for _, p := range parts[2:] {
keyFields := strings.FieldsFunc(p, func(r rune) bool {
switch r {
case '#', ',', '(', ')':
return true
default:
return unicode.IsSpace(r)
}
})
var k Key
// Parse the sequence number.
seqNum, err := strconv.ParseUint(keyFields[0], 10, 64)
if err != nil {
panic(fmt.Sprintf("invalid sequence number: %q: %s", keyFields[0], err))
}
// Parse the key kind.
kind := base.ParseKind(keyFields[1])
k.Trailer = base.MakeTrailer(seqNum, kind)
// Parse the optional suffix.
if len(keyFields) >= 3 {
k.Suffix = []byte(keyFields[2])
}
// Parse the optional value.
if len(keyFields) >= 4 {
k.Value = []byte(keyFields[3])
}
s.Keys = append(s.Keys, k)
}
return s
}

60
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Truncate creates a new iterator where every span in the supplied iterator is
// truncated to be contained within the range [lower, upper). If start and end
// are specified, filter out any spans that are completely outside those bounds.
func Truncate(
cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
) FragmentIterator {
return Filter(iter, func(in Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
// Ignore this span if it lies completely outside [start, end].
//
// The comparison between s.End and start is by user key only, as
// the span is exclusive at s.End, so comparing by user keys
// is sufficient.
if start != nil && cmp(in.End, start.UserKey) <= 0 {
return false
}
if end != nil {
v := cmp(in.Start, end.UserKey)
switch {
case v > 0:
// Wholly outside the end bound. Skip it.
return false
case v == 0:
// This span begins at the same user key as `end`. Whether or
// not any of the keys contained within the span are relevant is
// dependent on Trailers. Any keys contained within the span
// with trailers larger than end cover the small sliver of
// keyspace between [k#inf, k#<end-seqnum>]. Since keys are
// sorted descending by Trailer within the span, we need to find
// the prefix of keys with larger trailers.
for i := range in.Keys {
if in.Keys[i].Trailer < end.Trailer {
out.Keys = out.Keys[:i]
break
}
}
default:
// Wholly within the end bound. Keep it.
}
}
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
out.Start = lower
}
if cmp(in.End, upper) > 0 {
out.End = upper
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}

1152
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

2006
vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

46
vendor/github.com/cockroachdb/pebble/internal/manifest/level.go generated vendored Normal file
View File

@@ -0,0 +1,46 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import "fmt"
const (
// 3 bits are necessary to represent level values from 0-6.
levelBits = 3
levelMask = (1 << levelBits) - 1
// invalidSublevel denotes an invalid or non-applicable sublevel.
invalidSublevel = -1
)
// Level encodes a level and optional sublevel for use in log and error
// messages. The encoding has the property that Level(0) ==
// L0Sublevel(invalidSublevel).
type Level uint32
func makeLevel(level, sublevel int) Level {
return Level(((sublevel + 1) << levelBits) | level)
}
// LevelToInt returns the int representation of a Level
func LevelToInt(l Level) int {
return int(l) & levelMask
}
// L0Sublevel returns a Level representing the specified L0 sublevel.
func L0Sublevel(sublevel int) Level {
if sublevel < 0 {
panic(fmt.Sprintf("invalid L0 sublevel: %d", sublevel))
}
return makeLevel(0, sublevel)
}
func (l Level) String() string {
level := int(l) & levelMask
sublevel := (int(l) >> levelBits) - 1
if sublevel != invalidSublevel {
return fmt.Sprintf("L%d.%d", level, sublevel)
}
return fmt.Sprintf("L%d", level)
}

596
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored Normal file
View File

@@ -0,0 +1,596 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import (
"bytes"
"fmt"
"github.com/cockroachdb/pebble/internal/base"
)
// LevelMetadata contains metadata for all of the files within
// a level of the LSM.
type LevelMetadata struct {
level int
tree btree
}
// clone makes a copy of the level metadata, implicitly increasing the ref
// count of every file contained within lm.
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
tree: lm.tree.clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
return lm.tree.release()
}
func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetadata {
bcmp := btreeCmpSeqNum
if level > 0 {
bcmp = btreeCmpSmallestKey(cmp)
}
var lm LevelMetadata
lm.level = level
lm.tree, _ = makeBTree(bcmp, files)
return lm
}
func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
var t btree
t.cmp = cmp
for _, f := range files {
t.insert(f)
}
return t, LevelSlice{iter: t.iter(), length: t.length}
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.length == 0
}
// Len returns the number of files within the level.
func (lm *LevelMetadata) Len() int {
return lm.tree.length
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.iter()}
}
// Slice constructs a slice containing the entire level.
func (lm *LevelMetadata) Slice() LevelSlice {
return LevelSlice{iter: lm.tree.iter(), length: lm.tree.length}
}
// Find finds the provided file in the level if it exists.
func (lm *LevelMetadata) Find(cmp base.Compare, m *FileMetadata) *LevelFile {
iter := lm.Iter()
if lm.level != 0 {
// If lm holds files for levels >0, we can narrow our search by binary
// searching by bounds.
o := overlaps(iter, cmp, m.Smallest.UserKey,
m.Largest.UserKey, m.Largest.IsExclusiveSentinel())
iter = o.Iter()
}
for f := iter.First(); f != nil; f = iter.Next() {
if f == m {
lf := iter.Take()
return &lf
}
}
return nil
}
// Annotation lazily calculates and returns the annotation defined by
// Annotator. The Annotator is used as the key for pre-calculated
// values, so equal Annotators must be used to avoid duplicate computations
// and cached annotations. Annotation must not be called concurrently, and in
// practice this is achieved by requiring callers to hold DB.mu.
func (lm *LevelMetadata) Annotation(annotator Annotator) interface{} {
if lm.Empty() {
return annotator.Zero(nil)
}
v, _ := lm.tree.root.annotation(annotator)
return v
}
// InvalidateAnnotation clears any cached annotations defined by Annotator. The
// Annotator is used as the key for pre-calculated values, so equal Annotators
// must be used to clear the appropriate cached annotation. InvalidateAnnotation
// must not be called concurrently, and in practice this is achieved by
// requiring callers to hold DB.mu.
func (lm *LevelMetadata) InvalidateAnnotation(annotator Annotator) {
if lm.Empty() {
return
}
lm.tree.root.invalidateAnnotation(annotator)
}
// LevelFile holds a file's metadata along with its position
// within a level of the LSM.
type LevelFile struct {
*FileMetadata
slice LevelSlice
}
// Slice constructs a LevelSlice containing only this file.
func (lf LevelFile) Slice() LevelSlice {
return lf.slice
}
// NewLevelSliceSeqSorted constructs a LevelSlice over the provided files,
// sorted by the L0 sequence number sort order.
// TODO(jackson): Can we improve this interface or avoid needing to export
// a slice constructor like this?
func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSeqNum, files)
tr.release()
return slice
}
// NewLevelSliceKeySorted constructs a LevelSlice over the provided files,
// sorted by the files smallest keys.
// TODO(jackson): Can we improve this interface or avoid needing to export
// a slice constructor like this?
func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSmallestKey(cmp), files)
tr.release()
return slice
}
// NewLevelSliceSpecificOrder constructs a LevelSlice over the provided files,
// ordering the files by their order in the provided slice. It's used in
// tests.
// TODO(jackson): Update tests to avoid requiring this and remove it.
func NewLevelSliceSpecificOrder(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSpecificOrder(files), files)
tr.release()
return slice
}
// LevelSlice contains a slice of the files within a level of the LSM.
// A LevelSlice is immutable once created, but may be used to construct a
// mutable LevelIterator over the slice's files.
type LevelSlice struct {
iter iterator
length int
// start and end form the inclusive bounds of a slice of files within a
// level of the LSM. They may be nil if the entire B-Tree backing iter is
// accessible.
start *iterator
end *iterator
}
// Each invokes fn for each element in the slice.
func (ls LevelSlice) Each(fn func(*FileMetadata)) {
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
fn(f)
}
}
// String implements fmt.Stringer.
func (ls LevelSlice) String() string {
var buf bytes.Buffer
ls.Each(func(f *FileMetadata) {
if buf.Len() > 0 {
fmt.Fprintf(&buf, " ")
}
fmt.Fprint(&buf, f)
})
return buf.String()
}
// Empty indicates whether the slice contains any files.
func (ls *LevelSlice) Empty() bool {
return emptyWithBounds(ls.iter, ls.start, ls.end)
}
// Iter constructs a LevelIterator that iterates over the slice.
func (ls *LevelSlice) Iter() LevelIterator {
return LevelIterator{
start: ls.start,
end: ls.end,
iter: ls.iter.clone(),
}
}
// Len returns the number of files in the slice. Its runtime is constant.
func (ls *LevelSlice) Len() int {
return ls.length
}
// SizeSum sums the size of all files in the slice. Its runtime is linear in
// the length of the slice.
func (ls *LevelSlice) SizeSum() uint64 {
var sum uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
sum += f.Size
}
return sum
}
// Reslice constructs a new slice backed by the same underlying level, with
// new start and end positions. Reslice invokes the provided function, passing
// two LevelIterators: one positioned to i's inclusive start and one
// positioned to i's inclusive end. The resliceFunc may move either iterator
// forward or backwards, including beyond the callee's original bounds to
// capture additional files from the underlying level. Reslice constructs and
// returns a new LevelSlice with the final bounds of the iterators after
// calling resliceFunc.
func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelSlice {
if ls.iter.r == nil {
return ls
}
var start, end LevelIterator
if ls.start == nil {
start.iter = ls.iter.clone()
start.iter.first()
} else {
start.iter = ls.start.clone()
}
if ls.end == nil {
end.iter = ls.iter.clone()
end.iter.last()
} else {
end.iter = ls.end.clone()
}
resliceFunc(&start, &end)
s := LevelSlice{
iter: start.iter.clone(),
start: &start.iter,
end: &end.iter,
}
// Calculate the new slice's length.
iter := s.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
s.length++
}
return s
}
// KeyType is used to specify the type of keys we're looking for in
// LevelIterator positioning operations. Files not containing any keys of the
// desired type are skipped.
type KeyType int8
const (
// KeyTypePointAndRange denotes a search among the entire keyspace, including
// both point keys and range keys. No sstables are skipped.
KeyTypePointAndRange KeyType = iota
// KeyTypePoint denotes a search among the point keyspace. SSTables with no
// point keys will be skipped. Note that the point keyspace includes rangedels.
KeyTypePoint
// KeyTypeRange denotes a search among the range keyspace. SSTables with no
// range keys will be skipped.
KeyTypeRange
)
type keyTypeAnnotator struct{}
var _ Annotator = keyTypeAnnotator{}
func (k keyTypeAnnotator) Zero(dst interface{}) interface{} {
var val *KeyType
if dst != nil {
val = dst.(*KeyType)
} else {
val = new(KeyType)
}
*val = KeyTypePoint
return val
}
func (k keyTypeAnnotator) Accumulate(m *FileMetadata, dst interface{}) (interface{}, bool) {
v := dst.(*KeyType)
switch *v {
case KeyTypePoint:
if m.HasRangeKeys {
*v = KeyTypePointAndRange
}
case KeyTypePointAndRange:
// Do nothing.
default:
panic("unexpected key type")
}
return v, true
}
func (k keyTypeAnnotator) Merge(src interface{}, dst interface{}) interface{} {
v := dst.(*KeyType)
srcVal := src.(*KeyType)
switch *v {
case KeyTypePoint:
if *srcVal == KeyTypePointAndRange {
*v = KeyTypePointAndRange
}
case KeyTypePointAndRange:
// Do nothing.
default:
panic("unexpected key type")
}
return v
}
// LevelIterator iterates over a set of files' metadata. Its zero value is an
// empty iterator.
type LevelIterator struct {
iter iterator
start *iterator
end *iterator
filter KeyType
}
func (i LevelIterator) String() string {
var buf bytes.Buffer
iter := i.iter.clone()
iter.first()
iter.prev()
if i.iter.pos == -1 {
fmt.Fprint(&buf, "(<start>)*")
}
iter.next()
for ; iter.valid(); iter.next() {
if buf.Len() > 0 {
fmt.Fprint(&buf, " ")
}
if i.start != nil && cmpIter(iter, *i.start) == 0 {
fmt.Fprintf(&buf, " [ ")
}
isCurrentPos := cmpIter(iter, i.iter) == 0
if isCurrentPos {
fmt.Fprint(&buf, " ( ")
}
fmt.Fprint(&buf, iter.cur().String())
if isCurrentPos {
fmt.Fprint(&buf, " )*")
}
if i.end != nil && cmpIter(iter, *i.end) == 0 {
fmt.Fprintf(&buf, " ]")
}
}
if i.iter.n != nil && i.iter.pos >= i.iter.n.count {
if buf.Len() > 0 {
fmt.Fprint(&buf, " ")
}
fmt.Fprint(&buf, "(<end>)*")
}
return buf.String()
}
// Clone copies the iterator, returning an independent iterator at the same
// position.
func (i *LevelIterator) Clone() LevelIterator {
if i.iter.r == nil {
return *i
}
// The start and end iterators are not cloned and are treated as
// immutable.
return LevelIterator{
iter: i.iter.clone(),
start: i.start,
end: i.end,
filter: i.filter,
}
}
// Current returns the item at the current iterator position.
func (i *LevelIterator) Current() *FileMetadata {
if !i.iter.valid() {
return nil
}
return i.iter.cur()
}
func (i *LevelIterator) empty() bool {
return emptyWithBounds(i.iter, i.start, i.end)
}
// Filter clones the iterator and sets the desired KeyType as the key to filter
// files on.
func (i *LevelIterator) Filter(keyType KeyType) LevelIterator {
l := i.Clone()
l.filter = keyType
return l
}
func emptyWithBounds(i iterator, start, end *iterator) bool {
// If i.r is nil, the iterator was constructed from an empty btree.
// If the end bound is before the start bound, the bounds represent an
// empty slice of the B-Tree.
return i.r == nil || (start != nil && end != nil && cmpIter(*end, *start) < 0)
}
// First seeks to the first file in the iterator and returns it.
func (i *LevelIterator) First() *FileMetadata {
if i.empty() {
return nil
}
if i.start != nil {
i.iter = i.start.clone()
} else {
i.iter.first()
}
if !i.iter.valid() {
return nil
}
return i.filteredNextFile(i.iter.cur())
}
// Last seeks to the last file in the iterator and returns it.
func (i *LevelIterator) Last() *FileMetadata {
if i.empty() {
return nil
}
if i.end != nil {
i.iter = i.end.clone()
} else {
i.iter.last()
}
if !i.iter.valid() {
return nil
}
return i.filteredPrevFile(i.iter.cur())
}
// Next advances the iterator to the next file and returns it.
func (i *LevelIterator) Next() *FileMetadata {
i.iter.next()
if !i.iter.valid() {
return nil
}
if i.end != nil && cmpIter(i.iter, *i.end) > 0 {
return nil
}
return i.filteredNextFile(i.iter.cur())
}
// Prev moves the iterator the previous file and returns it.
func (i *LevelIterator) Prev() *FileMetadata {
i.iter.prev()
if !i.iter.valid() {
return nil
}
if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
return nil
}
return i.filteredPrevFile(i.iter.cur())
}
// SeekGE seeks to the first file in the iterator's file set with a largest
// user key greater than or equal to the provided user key. The iterator must
// have been constructed from L1+, because it requires the underlying files to
// be sorted by user keys and non-overlapping.
func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.empty() {
return nil
}
meta := i.seek(func(m *FileMetadata) bool {
return cmp(m.Largest.UserKey, userKey) >= 0
})
return i.filteredNextFile(meta)
}
// SeekLT seeks to the last file in the iterator's file set with a smallest
// user key less than the provided user key. The iterator must have been
// constructed from L1+, because it requires the underlying files to be sorted
// by user keys and non-overlapping.
func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.empty() {
return nil
}
i.seek(func(m *FileMetadata) bool {
return cmp(m.Smallest.UserKey, userKey) >= 0
})
meta := i.Prev()
return i.filteredPrevFile(meta)
}
func (i *LevelIterator) filteredNextFile(meta *FileMetadata) *FileMetadata {
switch i.filter {
case KeyTypePoint:
for meta != nil && !meta.HasPointKeys {
meta = i.Next()
}
return meta
case KeyTypeRange:
// TODO(bilal): Range keys are expected to be rare and sparse. Add an
// optimization to annotate the tree and efficiently skip over files that
// do not contain range keys right at the seek step, to reduce iterations
// here.
for meta != nil && !meta.HasRangeKeys {
meta = i.Next()
}
return meta
default:
return meta
}
}
func (i *LevelIterator) filteredPrevFile(meta *FileMetadata) *FileMetadata {
switch i.filter {
case KeyTypePoint:
for meta != nil && !meta.HasPointKeys {
meta = i.Prev()
}
return meta
case KeyTypeRange:
// TODO(bilal): Range keys are expected to be rare and sparse. Add an
// optimization to annotate the tree and efficiently skip over files that
// do not contain range keys right at the seek step, to reduce iterations
// here.
for meta != nil && !meta.HasRangeKeys {
meta = i.Prev()
}
return meta
default:
return meta
}
}
func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
i.iter.seek(fn)
// i.iter.seek seeked in the unbounded underlying B-Tree. If the iterator
// has start or end bounds, we may have exceeded them. Reset to the bounds
// if necessary.
//
// NB: The LevelIterator and LevelSlice semantics require that a bounded
// LevelIterator/LevelSlice containing files x0, x1, ..., xn behave
// identically to an unbounded LevelIterator/LevelSlice of a B-Tree
// containing x0, x1, ..., xn. In other words, any files outside the
// LevelIterator's bounds should not influence the iterator's behavior.
// When seeking, this means a SeekGE that seeks beyond the end bound,
// followed by a Prev should return the last element within bounds.
if i.end != nil && cmpIter(i.iter, *i.end) > 0 {
i.iter = i.end.clone()
// Since seek(fn) positioned beyond i.end, we know there is nothing to
// return within bounds.
i.iter.next()
return nil
} else if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
i.iter = i.start.clone()
return i.iter.cur()
}
if !i.iter.valid() {
return nil
}
return i.iter.cur()
}
// Take constructs a LevelFile containing the file at the iterator's current
// position. Take panics if the iterator is not currently positioned over a
// file.
func (i *LevelIterator) Take() LevelFile {
m := i.Current()
if m == nil {
panic("Take called on invalid LevelIterator")
}
// LevelSlice's start and end fields are immutable and are positioned to
// the same position for a LevelFile because they're inclusive, so we can
// share one iterator stack between the two bounds.
boundsIter := i.iter.clone()
return LevelFile{
FileMetadata: m,
slice: LevelSlice{
iter: i.iter.clone(),
start: &boundsIter,
end: &boundsIter,
length: 1,
},
}
}

1119
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

775
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored Normal file
View File

@@ -0,0 +1,775 @@
// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import (
"bufio"
"bytes"
"encoding/binary"
"io"
"sync/atomic"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
)
// TODO(peter): describe the MANIFEST file format, independently of the C++
// project.
var errCorruptManifest = base.CorruptionErrorf("pebble: corrupt manifest")
type byteReader interface {
io.ByteReader
io.Reader
}
// Tags for the versionEdit disk format.
// Tag 8 is no longer used.
const (
// LevelDB tags.
tagComparator = 1
tagLogNumber = 2
tagNextFileNumber = 3
tagLastSequence = 4
tagCompactPointer = 5
tagDeletedFile = 6
tagNewFile = 7
tagPrevLogNumber = 9
// RocksDB tags.
tagNewFile2 = 100
tagNewFile3 = 102
tagNewFile4 = 103
tagColumnFamily = 200
tagColumnFamilyAdd = 201
tagColumnFamilyDrop = 202
tagMaxColumnFamily = 203
// Pebble tags.
tagNewFile5 = 104 // Range keys.
// The custom tags sub-format used by tagNewFile4 and above.
customTagTerminate = 1
customTagNeedsCompaction = 2
customTagCreationTime = 6
customTagPathID = 65
customTagNonSafeIgnoreMask = 1 << 6
)
// DeletedFileEntry holds the state for a file deletion from a level. The file
// itself might still be referenced by another level.
type DeletedFileEntry struct {
Level int
FileNum base.FileNum
}
// NewFileEntry holds the state for a new file or one moved from a different
// level.
type NewFileEntry struct {
Level int
Meta *FileMetadata
}
// VersionEdit holds the state for an edit to a Version along with other
// on-disk state (log numbers, next file number, and the last sequence number).
type VersionEdit struct {
// ComparerName is the value of Options.Comparer.Name. This is only set in
// the first VersionEdit in a manifest (either when the DB is created, or
// when a new manifest is created) and is used to verify that the comparer
// specified at Open matches the comparer that was previously used.
ComparerName string
// MinUnflushedLogNum is the smallest WAL log file number corresponding to
// mutations that have not been flushed to an sstable.
//
// This is an optional field, and 0 represents it is not set.
MinUnflushedLogNum base.FileNum
// ObsoletePrevLogNum is a historic artifact from LevelDB that is not used by
// Pebble, RocksDB, or even LevelDB. Its use in LevelDB was deprecated in
// 6/2011. We keep it around purely for informational purposes when
// displaying MANIFEST contents.
ObsoletePrevLogNum uint64
// The next file number. A single counter is used to assign file numbers
// for the WAL, MANIFEST, sstable, and OPTIONS files.
NextFileNum base.FileNum
// LastSeqNum is an upper bound on the sequence numbers that have been
// assigned in flushed WALs. Unflushed WALs (that will be replayed during
// recovery) may contain sequence numbers greater than this value.
LastSeqNum uint64
// A file num may be present in both deleted files and new files when it
// is moved from a lower level to a higher level (when the compaction
// found that there was no overlapping file at the higher level).
DeletedFiles map[DeletedFileEntry]*FileMetadata
NewFiles []NewFileEntry
}
// Decode decodes an edit from the specified reader.
func (v *VersionEdit) Decode(r io.Reader) error {
br, ok := r.(byteReader)
if !ok {
br = bufio.NewReader(r)
}
d := versionEditDecoder{br}
for {
tag, err := binary.ReadUvarint(br)
if err == io.EOF {
break
}
if err != nil {
return err
}
switch tag {
case tagComparator:
s, err := d.readBytes()
if err != nil {
return err
}
v.ComparerName = string(s)
case tagLogNumber:
n, err := d.readFileNum()
if err != nil {
return err
}
v.MinUnflushedLogNum = n
case tagNextFileNumber:
n, err := d.readFileNum()
if err != nil {
return err
}
v.NextFileNum = n
case tagLastSequence:
n, err := d.readUvarint()
if err != nil {
return err
}
v.LastSeqNum = n
case tagCompactPointer:
if _, err := d.readLevel(); err != nil {
return err
}
if _, err := d.readBytes(); err != nil {
return err
}
// NB: RocksDB does not use compaction pointers anymore.
case tagDeletedFile:
level, err := d.readLevel()
if err != nil {
return err
}
fileNum, err := d.readFileNum()
if err != nil {
return err
}
if v.DeletedFiles == nil {
v.DeletedFiles = make(map[DeletedFileEntry]*FileMetadata)
}
v.DeletedFiles[DeletedFileEntry{level, fileNum}] = nil
case tagNewFile, tagNewFile2, tagNewFile3, tagNewFile4, tagNewFile5:
level, err := d.readLevel()
if err != nil {
return err
}
fileNum, err := d.readFileNum()
if err != nil {
return err
}
if tag == tagNewFile3 {
// The pathID field appears unused in RocksDB.
_ /* pathID */, err := d.readUvarint()
if err != nil {
return err
}
}
size, err := d.readUvarint()
if err != nil {
return err
}
// We read the smallest / largest key bounds differently depending on
// whether we have point, range or both types of keys present in the
// table.
var (
smallestPointKey, largestPointKey []byte
smallestRangeKey, largestRangeKey []byte
parsedPointBounds bool
boundsMarker byte
)
if tag != tagNewFile5 {
// Range keys not present in the table. Parse the point key bounds.
smallestPointKey, err = d.readBytes()
if err != nil {
return err
}
largestPointKey, err = d.readBytes()
if err != nil {
return err
}
} else {
// Range keys are present in the table. Determine whether we have point
// keys to parse, in addition to the bounds.
boundsMarker, err = d.ReadByte()
if err != nil {
return err
}
// Parse point key bounds, if present.
if boundsMarker&maskContainsPointKeys > 0 {
smallestPointKey, err = d.readBytes()
if err != nil {
return err
}
largestPointKey, err = d.readBytes()
if err != nil {
return err
}
parsedPointBounds = true
} else {
// The table does not have point keys.
// Sanity check: the bounds must be range keys.
if boundsMarker&maskSmallest != 0 || boundsMarker&maskLargest != 0 {
return base.CorruptionErrorf(
"new-file-4-range-keys: table without point keys has point key bounds: marker=%x",
boundsMarker,
)
}
}
// Parse range key bounds.
smallestRangeKey, err = d.readBytes()
if err != nil {
return err
}
largestRangeKey, err = d.readBytes()
if err != nil {
return err
}
}
var smallestSeqNum uint64
var largestSeqNum uint64
if tag != tagNewFile {
smallestSeqNum, err = d.readUvarint()
if err != nil {
return err
}
largestSeqNum, err = d.readUvarint()
if err != nil {
return err
}
}
var markedForCompaction bool
var creationTime uint64
if tag == tagNewFile4 || tag == tagNewFile5 {
for {
customTag, err := d.readUvarint()
if err != nil {
return err
}
if customTag == customTagTerminate {
break
}
field, err := d.readBytes()
if err != nil {
return err
}
switch customTag {
case customTagNeedsCompaction:
if len(field) != 1 {
return base.CorruptionErrorf("new-file4: need-compaction field wrong size")
}
markedForCompaction = (field[0] == 1)
case customTagCreationTime:
var n int
creationTime, n = binary.Uvarint(field)
if n != len(field) {
return base.CorruptionErrorf("new-file4: invalid file creation time")
}
case customTagPathID:
return base.CorruptionErrorf("new-file4: path-id field not supported")
default:
if (customTag & customTagNonSafeIgnoreMask) != 0 {
return base.CorruptionErrorf("new-file4: custom field not supported: %d", customTag)
}
}
}
}
m := &FileMetadata{
FileNum: fileNum,
Size: size,
CreationTime: int64(creationTime),
SmallestSeqNum: smallestSeqNum,
LargestSeqNum: largestSeqNum,
MarkedForCompaction: markedForCompaction,
}
if tag != tagNewFile5 { // no range keys present
m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
m.LargestPointKey = base.DecodeInternalKey(largestPointKey)
m.HasPointKeys = true
m.Smallest, m.Largest = m.SmallestPointKey, m.LargestPointKey
m.boundTypeSmallest, m.boundTypeLargest = boundTypePointKey, boundTypePointKey
} else { // range keys present
// Set point key bounds, if parsed.
if parsedPointBounds {
m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
m.LargestPointKey = base.DecodeInternalKey(largestPointKey)
m.HasPointKeys = true
}
// Set range key bounds.
m.SmallestRangeKey = base.DecodeInternalKey(smallestRangeKey)
m.LargestRangeKey = base.DecodeInternalKey(largestRangeKey)
m.HasRangeKeys = true
// Set overall bounds (by default assume range keys).
m.Smallest, m.Largest = m.SmallestRangeKey, m.LargestRangeKey
m.boundTypeSmallest, m.boundTypeLargest = boundTypeRangeKey, boundTypeRangeKey
if boundsMarker&maskSmallest == maskSmallest {
m.Smallest = m.SmallestPointKey
m.boundTypeSmallest = boundTypePointKey
}
if boundsMarker&maskLargest == maskLargest {
m.Largest = m.LargestPointKey
m.boundTypeLargest = boundTypePointKey
}
}
m.boundsSet = true
v.NewFiles = append(v.NewFiles, NewFileEntry{
Level: level,
Meta: m,
})
case tagPrevLogNumber:
n, err := d.readUvarint()
if err != nil {
return err
}
v.ObsoletePrevLogNum = n
case tagColumnFamily, tagColumnFamilyAdd, tagColumnFamilyDrop, tagMaxColumnFamily:
return base.CorruptionErrorf("column families are not supported")
default:
return errCorruptManifest
}
}
return nil
}
// Encode encodes an edit to the specified writer.
func (v *VersionEdit) Encode(w io.Writer) error {
e := versionEditEncoder{new(bytes.Buffer)}
if v.ComparerName != "" {
e.writeUvarint(tagComparator)
e.writeString(v.ComparerName)
}
if v.MinUnflushedLogNum != 0 {
e.writeUvarint(tagLogNumber)
e.writeUvarint(uint64(v.MinUnflushedLogNum))
}
if v.ObsoletePrevLogNum != 0 {
e.writeUvarint(tagPrevLogNumber)
e.writeUvarint(v.ObsoletePrevLogNum)
}
if v.NextFileNum != 0 {
e.writeUvarint(tagNextFileNumber)
e.writeUvarint(uint64(v.NextFileNum))
}
// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
// even though its value is zero. We detect this by encoding LastSeqNum when
// ComparerName is set.
if v.LastSeqNum != 0 || v.ComparerName != "" {
e.writeUvarint(tagLastSequence)
e.writeUvarint(v.LastSeqNum)
}
for x := range v.DeletedFiles {
e.writeUvarint(tagDeletedFile)
e.writeUvarint(uint64(x.Level))
e.writeUvarint(uint64(x.FileNum))
}
for _, x := range v.NewFiles {
customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
var tag uint64
switch {
case x.Meta.HasRangeKeys:
tag = tagNewFile5
case customFields:
tag = tagNewFile4
default:
tag = tagNewFile2
}
e.writeUvarint(tag)
e.writeUvarint(uint64(x.Level))
e.writeUvarint(uint64(x.Meta.FileNum))
e.writeUvarint(x.Meta.Size)
if !x.Meta.HasRangeKeys {
// If we have no range keys, preserve the original format and write the
// smallest and largest point keys.
e.writeKey(x.Meta.SmallestPointKey)
e.writeKey(x.Meta.LargestPointKey)
} else {
// When range keys are present, we first write a marker byte that
// indicates if the table also contains point keys, in addition to how the
// overall bounds for the table should be reconstructed. This byte is
// followed by the keys themselves.
b, err := x.Meta.boundsMarker()
if err != nil {
return err
}
if err = e.WriteByte(b); err != nil {
return err
}
// Write point key bounds (if present).
if x.Meta.HasPointKeys {
e.writeKey(x.Meta.SmallestPointKey)
e.writeKey(x.Meta.LargestPointKey)
}
// Write range key bounds.
e.writeKey(x.Meta.SmallestRangeKey)
e.writeKey(x.Meta.LargestRangeKey)
}
e.writeUvarint(x.Meta.SmallestSeqNum)
e.writeUvarint(x.Meta.LargestSeqNum)
if customFields {
if x.Meta.CreationTime != 0 {
e.writeUvarint(customTagCreationTime)
var buf [binary.MaxVarintLen64]byte
n := binary.PutUvarint(buf[:], uint64(x.Meta.CreationTime))
e.writeBytes(buf[:n])
}
if x.Meta.MarkedForCompaction {
e.writeUvarint(customTagNeedsCompaction)
e.writeBytes([]byte{1})
}
e.writeUvarint(customTagTerminate)
}
}
_, err := w.Write(e.Bytes())
return err
}
type versionEditDecoder struct {
byteReader
}
func (d versionEditDecoder) readBytes() ([]byte, error) {
n, err := d.readUvarint()
if err != nil {
return nil, err
}
s := make([]byte, n)
_, err = io.ReadFull(d, s)
if err != nil {
if err == io.ErrUnexpectedEOF {
return nil, errCorruptManifest
}
return nil, err
}
return s, nil
}
func (d versionEditDecoder) readLevel() (int, error) {
u, err := d.readUvarint()
if err != nil {
return 0, err
}
if u >= NumLevels {
return 0, errCorruptManifest
}
return int(u), nil
}
func (d versionEditDecoder) readFileNum() (base.FileNum, error) {
u, err := d.readUvarint()
if err != nil {
return 0, err
}
return base.FileNum(u), nil
}
func (d versionEditDecoder) readUvarint() (uint64, error) {
u, err := binary.ReadUvarint(d)
if err != nil {
if err == io.EOF {
return 0, errCorruptManifest
}
return 0, err
}
return u, nil
}
type versionEditEncoder struct {
*bytes.Buffer
}
func (e versionEditEncoder) writeBytes(p []byte) {
e.writeUvarint(uint64(len(p)))
e.Write(p)
}
func (e versionEditEncoder) writeKey(k InternalKey) {
e.writeUvarint(uint64(k.Size()))
e.Write(k.UserKey)
buf := k.EncodeTrailer()
e.Write(buf[:])
}
func (e versionEditEncoder) writeString(s string) {
e.writeUvarint(uint64(len(s)))
e.WriteString(s)
}
func (e versionEditEncoder) writeUvarint(u uint64) {
var buf [binary.MaxVarintLen64]byte
n := binary.PutUvarint(buf[:], u)
e.Write(buf[:n])
}
// BulkVersionEdit summarizes the files added and deleted from a set of version
// edits.
type BulkVersionEdit struct {
Added [NumLevels][]*FileMetadata
Deleted [NumLevels]map[base.FileNum]*FileMetadata
// AddedByFileNum maps file number to file metadata for all added files
// from accumulated version edits. AddedByFileNum is only populated if set
// to non-nil by a caller. It must be set to non-nil when replaying
// version edits read from a MANIFEST (as opposed to VersionEdits
// constructed in-memory). While replaying a MANIFEST file,
// VersionEdit.DeletedFiles map entries have nil values, because the
// on-disk deletion record encodes only the file number. Accumulate
// uses AddedByFileNum to correctly populate the BulkVersionEdit's Deleted
// field with non-nil *FileMetadata.
AddedByFileNum map[base.FileNum]*FileMetadata
// MarkedForCompactionCountDiff holds the aggregated count of files
// marked for compaction added or removed.
MarkedForCompactionCountDiff int
}
// Accumulate adds the file addition and deletions in the specified version
// edit to the bulk edit's internal state.
func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
for df, m := range ve.DeletedFiles {
dmap := b.Deleted[df.Level]
if dmap == nil {
dmap = make(map[base.FileNum]*FileMetadata)
b.Deleted[df.Level] = dmap
}
if m == nil {
// m is nil only when replaying a MANIFEST.
if b.AddedByFileNum == nil {
return errors.Errorf("deleted file L%d.%s's metadata is absent and bve.AddedByFileNum is nil", df.Level, df.FileNum)
}
m = b.AddedByFileNum[df.FileNum]
if m == nil {
return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", df.Level, df.FileNum)
}
}
if m.MarkedForCompaction {
b.MarkedForCompactionCountDiff--
}
dmap[df.FileNum] = m
}
for _, nf := range ve.NewFiles {
// A new file should not have been deleted in this or a preceding
// VersionEdit at the same level (though files can move across levels).
if dmap := b.Deleted[nf.Level]; dmap != nil {
if _, ok := dmap[nf.Meta.FileNum]; ok {
return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
}
}
b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta)
if b.AddedByFileNum != nil {
b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta
}
if nf.Meta.MarkedForCompaction {
b.MarkedForCompactionCountDiff++
}
}
return nil
}
// Apply applies the delta b to the current version to produce a new
// version. The new version is consistent with respect to the comparer cmp.
//
// curr may be nil, which is equivalent to a pointer to a zero version.
//
// On success, a map of zombie files containing the file numbers and sizes of
// deleted files is returned. These files are considered zombies because they
// are no longer referenced by the returned Version, but cannot be deleted from
// disk as they are still in use by the incoming Version.
func (b *BulkVersionEdit) Apply(
curr *Version,
cmp Compare,
formatKey base.FormatKey,
flushSplitBytes int64,
readCompactionRate int64,
) (_ *Version, zombies map[base.FileNum]uint64, _ error) {
addZombie := func(fileNum base.FileNum, size uint64) {
if zombies == nil {
zombies = make(map[base.FileNum]uint64)
}
zombies[fileNum] = size
}
// The remove zombie function is used to handle tables that are moved from
// one level to another during a version edit (i.e. a "move" compaction).
removeZombie := func(fileNum base.FileNum) {
if zombies != nil {
delete(zombies, fileNum)
}
}
v := new(Version)
// Adjust the count of files marked for compaction.
if curr != nil {
v.Stats.MarkedForCompaction = curr.Stats.MarkedForCompaction
}
v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff
if v.Stats.MarkedForCompaction < 0 {
return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
}
for level := range v.Levels {
if curr == nil || curr.Levels[level].tree.root == nil {
v.Levels[level] = makeLevelMetadata(cmp, level, nil /* files */)
} else {
v.Levels[level] = curr.Levels[level].clone()
}
if len(b.Added[level]) == 0 && len(b.Deleted[level]) == 0 {
// There are no edits on this level.
if level == 0 {
// Initialize L0Sublevels.
if curr == nil || curr.L0Sublevels == nil {
if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
}
} else {
v.L0Sublevels = curr.L0Sublevels
v.L0SublevelFiles = v.L0Sublevels.Levels
}
}
continue
}
// Some edits on this level.
lm := &v.Levels[level]
addedFiles := b.Added[level]
deletedMap := b.Deleted[level]
if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
return nil, nil, base.CorruptionErrorf(
"pebble: internal error: No current or added files but have deleted files: %d",
errors.Safe(len(deletedMap)))
}
// NB: addedFiles may be empty and it also is not necessarily
// internally consistent: it does not reflect deletions in deletedMap.
for _, f := range deletedMap {
addZombie(f.FileNum, f.Size)
if obsolete := v.Levels[level].tree.delete(f); obsolete {
// Deleting a file from the B-Tree may decrement its
// reference count. However, because we cloned the
// previous level's B-Tree, this should never result in a
// file's reference count dropping to zero.
err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
return nil, nil, err
}
}
var sm, la *FileMetadata
for _, f := range addedFiles {
if _, ok := deletedMap[f.FileNum]; ok {
// Already called addZombie on this file in the preceding
// loop, so we don't need to do it here.
continue
}
// NB: allowedSeeks is used for read triggered compactions. It is set using
// Options.Experimental.ReadCompactionRate which defaults to 32KB.
var allowedSeeks int64
if readCompactionRate != 0 {
allowedSeeks = int64(f.Size) / readCompactionRate
}
if allowedSeeks < 100 {
allowedSeeks = 100
}
atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks)
f.InitAllowedSeeks = allowedSeeks
err := lm.tree.insert(f)
if err != nil {
return nil, nil, errors.Wrap(err, "pebble")
}
removeZombie(f.FileNum)
// Track the keys with the smallest and largest keys, so that we can
// check consistency of the modified span.
if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 {
sm = f
}
if la == nil || base.InternalCompare(cmp, la.Largest, f.Largest) < 0 {
la = f
}
}
if level == 0 {
if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 {
// Flushes and ingestions that do not delete any L0 files do not require
// a regeneration of L0Sublevels from scratch. We can instead generate
// it incrementally.
var err error
// AddL0Files requires addedFiles to be sorted in seqnum order.
addedFiles = append([]*FileMetadata(nil), addedFiles...)
SortBySeqNum(addedFiles)
v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0])
if errors.Is(err, errInvalidL0SublevelsOpt) {
err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes)
}
if err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
}
v.L0SublevelFiles = v.L0Sublevels.Levels
} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
}
if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
}
continue
}
// Check consistency of the level in the vicinity of our edits.
if sm != nil && la != nil {
overlap := overlaps(v.Levels[level].Iter(), cmp, sm.Smallest.UserKey,
la.Largest.UserKey, la.Largest.IsExclusiveSentinel())
// overlap contains all of the added files. We want to ensure that
// the added files are consistent with neighboring existing files
// too, so reslice the overlap to pull in a neighbor on each side.
check := overlap.Reslice(func(start, end *LevelIterator) {
if m := start.Prev(); m == nil {
start.Next()
}
if m := end.Next(); m == nil {
end.Prev()
}
})
if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
return nil, nil, errors.Wrap(err, "pebble: internal error")
}
}
}
return v, zombies, nil
}

60
vendor/github.com/cockroachdb/pebble/internal/manual/manual.go generated vendored Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manual
// #include <stdlib.h>
import "C"
import "unsafe"
// The go:linkname directives provides backdoor access to private functions in
// the runtime. Below we're accessing the throw function.
//go:linkname throw runtime.throw
func throw(s string)
// TODO(peter): Rather than relying an C malloc/free, we could fork the Go
// runtime page allocator and allocate large chunks of memory using mmap or
// similar.
// New allocates a slice of size n. The returned slice is from manually managed
// memory and MUST be released by calling Free. Failure to do so will result in
// a memory leak.
func New(n int) []byte {
if n == 0 {
return make([]byte, 0)
}
// We need to be conscious of the Cgo pointer passing rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// ...
// Note: the current implementation has a bug. While Go code is permitted
// to write nil or a C pointer (but not a Go pointer) to C memory, the
// current implementation may sometimes cause a runtime error if the
// contents of the C memory appear to be a Go pointer. Therefore, avoid
// passing uninitialized C memory to Go code if the Go code is going to
// store pointer values in it. Zero out the memory in C before passing it
// to Go.
ptr := C.calloc(C.size_t(n), 1)
if ptr == nil {
// NB: throw is like panic, except it guarantees the process will be
// terminated. The call below is exactly what the Go runtime invokes when
// it cannot allocate memory.
throw("out of memory")
}
// Interpret the C pointer as a pointer to a Go array, then slice.
return (*[MaxArrayLen]byte)(unsafe.Pointer(ptr))[:n:n]
}
// Free frees the specified slice.
func Free(b []byte) {
if cap(b) != 0 {
if len(b) == 0 {
b = b[:cap(b)]
}
ptr := unsafe.Pointer(&b[0])
C.free(ptr)
}
}

13
vendor/github.com/cockroachdb/pebble/internal/manual/manual_32bit.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build 386 || amd64p32 || arm || armbe || mips || mipsle || mips64p32 || mips64p32le || ppc || sparc
// +build 386 amd64p32 arm armbe mips mipsle mips64p32 mips64p32le ppc sparc
package manual
const (
// MaxArrayLen is a safe maximum length for slices on this architecture.
MaxArrayLen = 1<<31 - 1
)

13
vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64
package manual
const (
// MaxArrayLen is a safe maximum length for slices on this architecture.
MaxArrayLen = 1<<50 - 1
)

20
vendor/github.com/cockroachdb/pebble/internal/manual/manual_nocgo.go generated vendored Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !cgo
// +build !cgo
package manual
// Provides versions of New and Free when cgo is not available (e.g. cross
// compilation).
// New allocates a slice of size n.
func New(n int) []byte {
return make([]byte, n)
}
// Free frees the specified slice.
func Free(b []byte) {
}

19
vendor/github.com/cockroachdb/pebble/internal/private/batch.go generated vendored Normal file
View File

@@ -0,0 +1,19 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// BatchSort is a hook for constructing iterators over the point and range
// mutations contained in a batch in sorted order. It is intended for testing
// use only.
var BatchSort func(interface{}) (
points base.InternalIterator,
rangeDels keyspan.FragmentIterator,
rangeKeys keyspan.FragmentIterator,
)

31
vendor/github.com/cockroachdb/pebble/internal/private/flush_external.go generated vendored Normal file
View File

@@ -0,0 +1,31 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import "github.com/cockroachdb/pebble/internal/manifest"
// FlushExternalTable is a hook for linking files into L0 without assigning a
// global sequence number, mimicking a flush. FlushExternalTable takes a
// *pebble.DB, the path and the metadata of a sstable to flush directly into
// L0.
//
// Calls to flush a sstable may fail if the file's sequence numbers are not
// greater than the current commit pipeline's sequence number. On success the
// commit pipeline's published sequence number will be moved to the file's
// highest sequence number.
//
// This function is wrapped in a safer, more ergonomic API in the
// internal/replay package. Clients should use the replay package rather than
// calling this private hook directly.
var FlushExternalTable func(interface{}, string, *manifest.FileMetadata) error
// RatchetSeqNum is a hook for allocating and publishing sequence numbers up
// to a specific absolute value. Its first parameter is a *pebble.DB and its
// second is the new next sequence number. RatchetSeqNum does nothing if the
// next sequence is already greater than or equal to nextSeqNum.
//
// This function is used by the internal/replay package to ensure replayed
// operations receive the same absolute sequence number.
var RatchetSeqNum func(interface{}, uint64)

28
vendor/github.com/cockroachdb/pebble/internal/private/sstable.go generated vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import "github.com/cockroachdb/pebble/internal/base"
// SSTableCacheOpts is a hook for specifying cache options to
// sstable.NewReader.
var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{}
// SSTableRawTombstonesOpt is a sstable.Reader option for disabling
// fragmentation of the range tombstones returned by
// sstable.Reader.NewRangeDelIter(). Used by debug tools to get a raw view of
// the tombstones contained in an sstable.
var SSTableRawTombstonesOpt interface{}
// SSTableWriterDisableKeyOrderChecks is a hook for disabling the key ordering
// invariant check performed by sstable.Writer. It is intended for internal use
// only in the construction of invalid sstables for testing. See
// tool/make_test_sstables.go.
var SSTableWriterDisableKeyOrderChecks func(interface{})
// SSTableInternalTableOpt is an sstable.Writer option that sets properties for
// sstables being created by the db itself (i.e. through flushes and
// compactions), as opposed to those meant for ingestion.
var SSTableInternalTableOpt interface{}

43
vendor/github.com/cockroachdb/pebble/internal/rangedel/rangedel.go generated vendored Normal file
View File

@@ -0,0 +1,43 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package rangedel
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// Encode takes a Span containing only range deletions. It invokes the provided
// closure with the encoded internal keys that represent the Span's state. The
// keys and values passed to emit are only valid until the closure returns. If
// emit returns an error, Encode stops and returns the error.
func Encode(s keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
for _, k := range s.Keys {
if k.Kind() != base.InternalKeyKindRangeDelete {
return base.CorruptionErrorf("pebble: rangedel.Encode cannot encode %s key", k.Kind())
}
ik := base.InternalKey{
UserKey: s.Start,
Trailer: k.Trailer,
}
if err := emit(ik, s.End); err != nil {
return err
}
}
return nil
}
// Decode takes an internal key pair encoding a range deletion and returns a
// decoded keyspan containing the key. If keysDst is provided, the key will be
// appended to keysDst, avoiding an allocation.
func Decode(ik base.InternalKey, v []byte, keysDst []keyspan.Key) keyspan.Span {
return keyspan.Span{
Start: ik.UserKey,
End: v,
Keys: append(keysDst, keyspan.Key{
Trailer: ik.Trailer,
}),
}
}

259
vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go generated vendored Normal file
View File

@@ -0,0 +1,259 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package rangekey
import (
"bytes"
"sort"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// InitUserIteration initializes the range key iterator stack for user
// iteration. The resulting fragment iterator applies range key semantics,
// defragments spans according to their user-observable state and removes all
// Keys other than RangeKeySets describing the current state of range keys.
//
// The snapshot sequence number parameter determines which keys are visible. Any
// keys not visible at the provided snapshot are ignored.
func InitUserIteration(
cmp base.Compare,
snapshot uint64,
miter *keyspan.MergingIter,
diter *keyspan.DefragmentingIter,
levelIters ...keyspan.FragmentIterator,
) keyspan.FragmentIterator {
miter.Init(cmp, userIterationTransform(snapshot), levelIters...)
diter.Init(cmp, miter, userIterationDefragmenter(), keyspan.StaticDefragmentReducer)
return diter
}
// userIterationTransform returns a keyspan.Transform for use with a
// keyspan.MergingIter that transforms spans by resolving range keys at the
// provided snapshot sequence number. Shadowing of keys is resolved (eg, removal
// of unset keys, removal of keys overwritten by a set at the same suffix, etc)
// and then non-RangeKeySet keys are removed. The resulting transformed spans
// only contain RangeKeySets describing the state visible at the provided
// sequence number.
func userIterationTransform(snapshot uint64) keyspan.Transform {
return func(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
// Apply shadowing of keys.
if err := Coalesce(cmp, s.Visible(snapshot), dst); err != nil {
return err
}
// During user iteration over range keys, unsets and deletes don't
// matter. Remove them. This step helps logical defragmentation during
// iteration.
keys := dst.Keys
dst.Keys = dst.Keys[:0]
for i := range keys {
switch keys[i].Kind() {
case base.InternalKeyKindRangeKeySet:
dst.Keys = append(dst.Keys, keys[i])
case base.InternalKeyKindRangeKeyUnset:
// Skip.
continue
case base.InternalKeyKindRangeKeyDelete:
// Skip.
continue
default:
return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
}
}
return nil
}
}
// userIterationDefragmenter constructs a DefragmentMethod that configures a
// DefragmentingIter to defragment spans of range keys if their user-visible
// state is identical. This defragmenting method assumes the provided spans have
// already been transformed through UserIterationTransform, so all RangeKeySets
// are user-visible sets. This defragmenter checks for equality between set
// suffixes and values (ignoring sequence numbers). It's intended for use during
// user iteration, when the wrapped keyspan iterator is merging spans across all
// levels of the LSM.
//
// The returned defragmenting method is stateful, and must not be used on
// multiple DefragmentingIters concurrently.
func userIterationDefragmenter() keyspan.DefragmentMethod {
var bufA keysBySuffix
var bufB keysBySuffix
return func(cmp base.Compare, a, b keyspan.Span) bool {
// UserIterationDefragmenter must only be used on spans that have
// transformed by UserIterationTransform. The transform applies
// shadowing and removes all keys besides the resulting Sets. Since
// shadowing has been applied, each Set must set a unique suffix. If the
// two spans are equivalent, they must have the same number of range key
// sets.
if len(a.Keys) != len(b.Keys) {
return false
}
// The keys in both spans must be all sets. In order to be equivalent
// during user iteration, they must set identical suffix-value mappings.
// They do not need to have the same sequence number.
//
// Currently, the Keys are sorted by Trailer (as per the keyspan.Span
// and keyspan.FragmentIterator interfaces). Collect all the keys in
// both spans into buffers and sort them by suffix for comparison.
bufA.cmp = cmp
bufA.keys = append(bufA.keys[:0], a.Keys...)
sort.Sort(bufA)
bufB.cmp = cmp
bufB.keys = append(bufB.keys[:0], b.Keys...)
sort.Sort(bufB)
for i := range bufA.keys {
if bufA.keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
bufB.keys[i].Kind() != base.InternalKeyKindRangeKeySet {
panic("pebble: unexpected non-RangeKeySet during defragmentation")
}
if cmp(bufA.keys[i].Suffix, bufB.keys[i].Suffix) != 0 {
return false
}
if !bytes.Equal(bufA.keys[i].Value, bufB.keys[i].Value) {
return false
}
}
return true
}
}
// Coalesce imposes range key semantics and coalesces range keys with the same
// bounds. Coalesce drops any keys shadowed by more recent sets, unsets or
// deletes. Coalesce modifies the provided span's Keys slice, reslicing the
// slice to remove dropped keys.
//
// Coalescence has subtle behavior with respect to sequence numbers. Coalesce
// depends on a keyspan.Span's Keys being sorted in sequence number descending
// order. The first key has the largest sequence number. The returned coalesced
// span includes only the largest sequence number. All other sequence numbers
// are forgotten. When a compaction constructs output range keys from a
// coalesced span, it produces at most one RANGEKEYSET, one RANGEKEYUNSET and
// one RANGEKEYDEL. Each one of these keys adopt the largest sequence number.
//
// This has the potentially surprising effect of 'promoting' a key to a higher
// sequence number. This is okay, because:
// - There are no other overlapping keys within the coalesced span of
// sequence numbers (otherwise they would be in the compaction, due to
// the LSM invariant).
// - Range key sequence numbers are never compared to point key sequence
// numbers. Range keys and point keys have parallel existences.
// - Compactions only coalesce within snapshot stripes.
//
// Additionally, internal range keys at the same sequence number have subtle
// mechanics:
// * RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix.
// * RANGEKEYDELs only apply to keys at lower sequence numbers.
// This is required for ingestion. Ingested sstables are assigned a single
// sequence number for the file, at which all of the file's keys are visible.
// The RANGEKEYSET, RANGEKEYUNSET and RANGEKEYDEL key kinds are ordered such
// that among keys with equal sequence numbers (thus ordered by their kinds) the
// keys do not affect one another. Ingested sstables are expected to be
// consistent with respect to the set/unset suffixes: A given suffix should be
// set or unset but not both.
func Coalesce(cmp base.Compare, span keyspan.Span, dst *keyspan.Span) error {
// TODO(jackson): Currently, Coalesce doesn't actually perform the sequence
// number promotion described in the comment above.
keysBySuffix := keysBySuffix{
cmp: cmp,
keys: dst.Keys[:0],
}
var deleted bool
for i := 0; i < len(span.Keys) && !deleted; i++ {
k := span.Keys[i]
if invariants.Enabled && i > 0 && k.Trailer > span.Keys[i-1].Trailer {
panic("pebble: invariant violation: span keys unordered")
}
// NB: Within a given sequence number, keys are ordered as:
// RangeKeySet > RangeKeyUnset > RangeKeyDelete
// This is significant, because this ensures that none of the range keys
// sharing a sequence number shadow each other.
switch k.Kind() {
case base.InternalKeyKindRangeKeySet:
n := len(keysBySuffix.keys)
if keysBySuffix.get(n, k.Suffix) < n {
// This suffix is already set or unset at a higher sequence
// number. Skip.
continue
}
keysBySuffix.keys = append(keysBySuffix.keys, k)
sort.Sort(keysBySuffix)
case base.InternalKeyKindRangeKeyUnset:
n := len(keysBySuffix.keys)
if keysBySuffix.get(n, k.Suffix) < n {
// This suffix is already set or unset at a higher sequence
// number. Skip.
continue
}
keysBySuffix.keys = append(keysBySuffix.keys, k)
sort.Sort(keysBySuffix)
case base.InternalKeyKindRangeKeyDelete:
// All remaining range keys in this span have been deleted by this
// RangeKeyDelete. There's no need to continue looping, because all
// the remaining keys are shadowed by this one. The for loop
// condition will terminate when it sees the last key is a
// range key deletion.
keysBySuffix.keys = append(keysBySuffix.keys, k)
deleted = true
default:
return base.CorruptionErrorf("pebble: unexpected range key kind %s", k.Kind())
}
}
// Update the span with the (potentially reduced) keys slice, and re-sort it
// by Trailer.
*dst = keyspan.Span{
Start: span.Start,
End: span.End,
Keys: keysBySuffix.keys,
}
keyspan.SortKeys(dst.Keys)
return nil
}
// SortBySuffix sorts the provided keys by suffix.
func SortBySuffix(cmp base.Compare, keys []keyspan.Key) {
bySuffix := keysBySuffix{
cmp: cmp,
keys: keys,
}
sort.Sort(bySuffix)
}
type keysBySuffix struct {
cmp base.Compare
keys []keyspan.Key
}
// get searches for suffix among the first n keys in keys. If the suffix is
// found, it returns the index of the item with the suffix. If the suffix is not
// found, it returns n.
func (s keysBySuffix) get(n int, suffix []byte) (i int) {
// Binary search for the suffix to see if there's an existing key with the
// suffix. Only binary search among the first n items. get is called while
// appending new keys with suffixes that may sort before existing keys.
// The n parameter indicates what portion of the keys slice is sorted and
// may contain relevant keys.
i = sort.Search(n, func(i int) bool {
return s.cmp(s.keys[i].Suffix, suffix) >= 0
})
if i < n && s.cmp(s.keys[i].Suffix, suffix) == 0 {
return i
}
return n
}
func (s keysBySuffix) Len() int { return len(s.keys) }
func (s keysBySuffix) Less(i, j int) bool { return s.cmp(s.keys[i].Suffix, s.keys[j].Suffix) < 0 }
func (s keysBySuffix) Swap(i, j int) { s.keys[i], s.keys[j] = s.keys[j], s.keys[i] }

411
vendor/github.com/cockroachdb/pebble/internal/rangekey/rangekey.go generated vendored Normal file
View File

@@ -0,0 +1,411 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package rangekey provides facilities for encoding, decoding and merging range
// keys.
//
// Range keys map a span of keyspan `[start, end)`, at an optional suffix, to a
// value.
//
// # Encoding
//
// Unlike other Pebble keys, range keys encode several fields of information:
// start key, end key, suffix and value. Internally within Pebble and its
// sstables, all keys including range keys are represented as a key-value tuple.
// Range keys map to internal key-value tuples by mapping the start key to the
// key and encoding the remainder of the fields in the value.
//
// ## `RANGEKEYSET`
//
// A `RANGEKEYSET` represents one more range keys set over a single region of
// user key space. Each represented range key must have a unique suffix. A
// `RANGEKEYSET` encapsulates a start key, an end key and a set of SuffixValue
// pairs.
//
// A `RANGEKEYSET` key's user key holds the start key. Its value is a varstring
// end key, followed by a set of SuffixValue pairs. A `RANGEKEYSET` may have
// multiple SuffixValue pairs if the keyspan was set at multiple unique suffix
// values.
//
// ## `RANGEKEYUNSET`
//
// A `RANGEKEYUNSET` represents the removal of range keys at specific suffixes
// over a single region of user key space. A `RANGEKEYUNSET` encapsulates a
// start key, an end key and a set of suffixes.
//
// A `RANGEKEYUNSET` key's user key holds the start key. Its value is a
// varstring end key, followed by a set of suffixes. A `RANGEKEYUNSET` may have
// multiple suffixes if the keyspan was unset at multiple unique suffixes.
//
// ## `RANGEKEYDEL`
//
// A `RANGEKEYDEL` represents the removal of all range keys over a single region
// of user key space, regardless of suffix. A `RANGEKEYDEL` encapsulates a
// start key and an end key. The end key is stored in the value, without any
// varstring length prefixing.
package rangekey
// TODO(jackson): Document the encoding of RANGEKEYSET and RANGEKEYUNSET values
// once we're confident they're stable.
import (
"encoding/binary"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// Encode takes a Span containing only range keys. It invokes the provided
// closure with the encoded internal keys that represent the Span's state. The
// keys and values passed to emit are only valid until the closure returns.
// If emit returns an error, Encode stops and returns the error.
func Encode(s keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
enc := Encoder{Emit: emit}
return enc.Encode(s)
}
// An Encoder encodes range keys into their on-disk InternalKey format. An
// Encoder holds internal buffers, reused between Emit calls.
type Encoder struct {
Emit func(base.InternalKey, []byte) error
buf []byte
unsets [][]byte
sets []SuffixValue
}
// Encode takes a Span containing only range keys. It invokes the Encoder's Emit
// closure with the encoded internal keys that represent the Span's state. The
// keys and values passed to emit are only valid until the closure returns. If
// Emit returns an error, Encode stops and returns the error.
//
// The encoded key-value pair passed to Emit is only valid until the closure
// completes.
func (e *Encoder) Encode(s keyspan.Span) error {
if s.Empty() {
return nil
}
// This for loop iterates through the span's keys, which are sorted by
// sequence number descending, grouping them into sequence numbers. All keys
// with identical sequence numbers are flushed together.
var del bool
var seqNum uint64
for i := range s.Keys {
if i == 0 || s.Keys[i].SeqNum() != seqNum {
if i > 0 {
// Flush all the existing internal keys that exist at seqNum.
if err := e.flush(s, seqNum, del); err != nil {
return err
}
}
// Reset sets, unsets, del.
seqNum = s.Keys[i].SeqNum()
del = false
e.sets = e.sets[:0]
e.unsets = e.unsets[:0]
}
switch s.Keys[i].Kind() {
case base.InternalKeyKindRangeKeySet:
e.sets = append(e.sets, SuffixValue{
Suffix: s.Keys[i].Suffix,
Value: s.Keys[i].Value,
})
case base.InternalKeyKindRangeKeyUnset:
e.unsets = append(e.unsets, s.Keys[i].Suffix)
case base.InternalKeyKindRangeKeyDelete:
del = true
default:
return base.CorruptionErrorf("pebble: %s key kind is not a range key", s.Keys[i].Kind())
}
}
return e.flush(s, seqNum, del)
}
// flush constructs internal keys for accumulated key state, and emits the
// internal keys.
func (e *Encoder) flush(s keyspan.Span, seqNum uint64, del bool) error {
if len(e.sets) > 0 {
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeySet)
l := EncodedSetValueLen(s.End, e.sets)
if l > cap(e.buf) {
e.buf = make([]byte, l)
}
EncodeSetValue(e.buf[:l], s.End, e.sets)
if err := e.Emit(ik, e.buf[:l]); err != nil {
return err
}
}
if len(e.unsets) > 0 {
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeyUnset)
l := EncodedUnsetValueLen(s.End, e.unsets)
if l > cap(e.buf) {
e.buf = make([]byte, l)
}
EncodeUnsetValue(e.buf[:l], s.End, e.unsets)
if err := e.Emit(ik, e.buf[:l]); err != nil {
return err
}
}
if del {
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeyDelete)
// s.End is stored directly in the value for RangeKeyDeletes.
if err := e.Emit(ik, s.End); err != nil {
return err
}
}
return nil
}
// Decode takes an internal key pair encoding range key(s) and returns a decoded
// keyspan containing the keys. If keysDst is provided, keys will be appended to
// keysDst.
func Decode(ik base.InternalKey, v []byte, keysDst []keyspan.Key) (keyspan.Span, error) {
var s keyspan.Span
// Hydrate the user key bounds.
s.Start = ik.UserKey
var ok bool
s.End, v, ok = DecodeEndKey(ik.Kind(), v)
if !ok {
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key end from %s", ik.Kind())
}
s.Keys = keysDst
// Hydrate the contents of the range key(s).
switch ik.Kind() {
case base.InternalKeyKindRangeKeySet:
for len(v) > 0 {
var sv SuffixValue
sv, v, ok = decodeSuffixValue(v)
if !ok {
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key suffix-value tuple")
}
s.Keys = append(s.Keys, keyspan.Key{
Trailer: ik.Trailer,
Suffix: sv.Suffix,
Value: sv.Value,
})
}
case base.InternalKeyKindRangeKeyUnset:
for len(v) > 0 {
var suffix []byte
suffix, v, ok = decodeSuffix(v)
if !ok {
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key unset suffix")
}
s.Keys = append(s.Keys, keyspan.Key{
Trailer: ik.Trailer,
Suffix: suffix,
})
}
case base.InternalKeyKindRangeKeyDelete:
if len(v) > 0 {
return keyspan.Span{}, base.CorruptionErrorf("pebble: RANGEKEYDELs must not contain additional data")
}
s.Keys = append(s.Keys, keyspan.Key{Trailer: ik.Trailer})
default:
return keyspan.Span{}, base.CorruptionErrorf("pebble: %s is not a range key", ik.Kind())
}
return s, nil
}
// SuffixValue represents a tuple of a suffix and a corresponding value. A
// physical RANGEKEYSET key may contain many logical RangeKeySets, each
// represented with a separate SuffixValue tuple.
type SuffixValue struct {
Suffix []byte
Value []byte
}
// encodedSetSuffixValuesLen precomputes the length of the given slice of
// SuffixValues, when encoded for a RangeKeySet. It may be used to construct a
// buffer of the appropriate size before encoding.
func encodedSetSuffixValuesLen(suffixValues []SuffixValue) int {
var n int
for i := 0; i < len(suffixValues); i++ {
n += lenVarint(len(suffixValues[i].Suffix))
n += len(suffixValues[i].Suffix)
n += lenVarint(len(suffixValues[i].Value))
n += len(suffixValues[i].Value)
}
return n
}
// encodeSetSuffixValues encodes a slice of SuffixValues for a RangeKeySet into
// dst. The length of dst must be greater than or equal to
// encodedSetSuffixValuesLen. encodeSetSuffixValues returns the number of bytes
// written, which should always equal the EncodedSetValueLen with the same
// arguments.
func encodeSetSuffixValues(dst []byte, suffixValues []SuffixValue) int {
// Encode the list of (suffix, value-len) tuples.
var n int
for i := 0; i < len(suffixValues); i++ {
// Encode the length of the suffix.
n += binary.PutUvarint(dst[n:], uint64(len(suffixValues[i].Suffix)))
// Encode the suffix itself.
n += copy(dst[n:], suffixValues[i].Suffix)
// Encode the value length.
n += binary.PutUvarint(dst[n:], uint64(len(suffixValues[i].Value)))
// Encode the value itself.
n += copy(dst[n:], suffixValues[i].Value)
}
return n
}
// EncodedSetValueLen precomputes the length of a RangeKeySet's value when
// encoded. It may be used to construct a buffer of the appropriate size before
// encoding.
func EncodedSetValueLen(endKey []byte, suffixValues []SuffixValue) int {
n := lenVarint(len(endKey))
n += len(endKey)
n += encodedSetSuffixValuesLen(suffixValues)
return n
}
// EncodeSetValue encodes a RangeKeySet's value into dst. The length of dst must
// be greater than or equal to EncodedSetValueLen. EncodeSetValue returns the
// number of bytes written, which should always equal the EncodedSetValueLen
// with the same arguments.
func EncodeSetValue(dst []byte, endKey []byte, suffixValues []SuffixValue) int {
// First encode the end key as a varstring.
n := binary.PutUvarint(dst, uint64(len(endKey)))
n += copy(dst[n:], endKey)
n += encodeSetSuffixValues(dst[n:], suffixValues)
return n
}
// DecodeEndKey reads the end key from the beginning of a range key (RANGEKEYSET,
// RANGEKEYUNSET or RANGEKEYDEL)'s physical encoded value. Both sets and unsets
// encode the range key, plus additional data in the value.
func DecodeEndKey(kind base.InternalKeyKind, data []byte) (endKey, value []byte, ok bool) {
switch kind {
case base.InternalKeyKindRangeKeyDelete:
// No splitting is necessary for range key deletes. The value is the end
// key, and there is no additional associated value.
return data, nil, true
case base.InternalKeyKindRangeKeySet, base.InternalKeyKindRangeKeyUnset:
v, n := binary.Uvarint(data)
if n <= 0 || uint64(n)+v >= uint64(len(data)) {
return nil, nil, false
}
endKey, value = data[n:n+int(v)], data[n+int(v):]
return endKey, value, true
default:
panic(errors.Newf("key kind %s is not a range key kind", kind))
}
}
// decodeSuffixValue decodes a single encoded SuffixValue from a RangeKeySet's
// split value. The end key must have already been stripped from the
// RangeKeySet's value (see DecodeEndKey).
func decodeSuffixValue(data []byte) (sv SuffixValue, rest []byte, ok bool) {
// Decode the suffix.
sv.Suffix, data, ok = decodeVarstring(data)
if !ok {
return SuffixValue{}, nil, false
}
// Decode the value.
sv.Value, data, ok = decodeVarstring(data)
if !ok {
return SuffixValue{}, nil, false
}
return sv, data, true
}
// encodedUnsetSuffixesLen precomputes the length of the given slice of
// suffixes, when encoded for a RangeKeyUnset. It may be used to construct a
// buffer of the appropriate size before encoding.
func encodedUnsetSuffixesLen(suffixes [][]byte) int {
var n int
for i := 0; i < len(suffixes); i++ {
n += lenVarint(len(suffixes[i]))
n += len(suffixes[i])
}
return n
}
// encodeUnsetSuffixes encodes a slice of suffixes for a RangeKeyUnset into dst.
// The length of dst must be greater than or equal to EncodedUnsetSuffixesLen.
// EncodeUnsetSuffixes returns the number of bytes written, which should always
// equal the EncodedUnsetSuffixesLen with the same arguments.
func encodeUnsetSuffixes(dst []byte, suffixes [][]byte) int {
// Encode the list of (suffix, value-len) tuples.
var n int
for i := 0; i < len(suffixes); i++ {
// Encode the length of the suffix.
n += binary.PutUvarint(dst[n:], uint64(len(suffixes[i])))
// Encode the suffix itself.
n += copy(dst[n:], suffixes[i])
}
return n
}
// EncodedUnsetValueLen precomputes the length of a RangeKeyUnset's value when
// encoded. It may be used to construct a buffer of the appropriate size before
// encoding.
func EncodedUnsetValueLen(endKey []byte, suffixes [][]byte) int {
n := lenVarint(len(endKey))
n += len(endKey)
n += encodedUnsetSuffixesLen(suffixes)
return n
}
// EncodeUnsetValue encodes a RangeKeyUnset's value into dst. The length of dst
// must be greater than or equal to EncodedUnsetValueLen. EncodeUnsetValue
// returns the number of bytes written, which should always equal the
// EncodedUnsetValueLen with the same arguments.
func EncodeUnsetValue(dst []byte, endKey []byte, suffixes [][]byte) int {
// First encode the end key as a varstring.
n := binary.PutUvarint(dst, uint64(len(endKey)))
n += copy(dst[n:], endKey)
n += encodeUnsetSuffixes(dst[n:], suffixes)
return n
}
// decodeSuffix decodes a single suffix from the beginning of data. If decoding
// suffixes from a RangeKeyUnset's value, the end key must have already been
// stripped from the RangeKeyUnset's value (see DecodeEndKey).
func decodeSuffix(data []byte) (suffix, rest []byte, ok bool) {
return decodeVarstring(data)
}
func decodeVarstring(data []byte) (v, rest []byte, ok bool) {
// Decode the length of the string.
l, n := binary.Uvarint(data)
if n <= 0 {
return nil, nil, ok
}
// Extract the string itself.
return data[n : n+int(l)], data[n+int(l):], true
}
// IsRangeKey returns true if the given key kind is one of the range key kinds.
func IsRangeKey(kind base.InternalKeyKind) bool {
switch kind {
case base.InternalKeyKindRangeKeyDelete,
base.InternalKeyKindRangeKeyUnset,
base.InternalKeyKindRangeKeySet:
return true
default:
return false
}
}
func lenVarint(v int) (n int) {
x := uint32(v)
n++
for x >= 0x80 {
x >>= 7
n++
}
return n
}

385
vendor/github.com/cockroachdb/pebble/internal/rate/rate.go generated vendored Normal file
View File

@@ -0,0 +1,385 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package rate provides a rate limiter.
package rate // import "github.com/cockroachdb/pebble/internal/rate"
import (
"context"
"math"
"sync"
"time"
"github.com/cockroachdb/errors"
)
// Limit defines the maximum frequency of some events.
// Limit is represented as number of events per second.
// A zero Limit allows no events.
type Limit float64
// Inf is the infinite rate limit; it allows all events (even if burst is zero).
const Inf = Limit(math.MaxFloat64)
// Every converts a minimum time interval between events to a Limit.
func Every(interval time.Duration) Limit {
if interval <= 0 {
return Inf
}
return 1 / Limit(interval.Seconds())
}
// A Limiter controls how frequently events are allowed to happen.
// It implements a "token bucket" of size b, initially full and refilled
// at rate r tokens per second.
// Informally, in any large enough time interval, the Limiter limits the
// rate to r tokens per second, with a maximum burst size of b events.
// As a special case, if r == Inf (the infinite rate), b is ignored.
// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
//
// The zero value is a valid Limiter, but it will reject all events.
// Use NewLimiter to create non-zero Limiters.
//
// Limiter has three main methods, Allow, Reserve, and Wait.
// Most callers should use Wait.
//
// Each of the three methods consumes a single token.
// They differ in their behavior when no token is available.
// If no token is available, Allow returns false.
// If no token is available, Reserve returns a reservation for a future token
// and the amount of time the caller must wait before using it.
// If no token is available, Wait blocks until one can be obtained
// or its associated context.Context is canceled.
//
// The methods AllowN, ReserveN, and WaitN consume n tokens.
type Limiter struct {
limit Limit
burst int
mu sync.Mutex
tokens float64
// last is the last time the limiter's tokens field was updated
last time.Time
// lastEvent is the latest time of a rate-limited event (past or future)
lastEvent time.Time
}
// Limit returns the maximum overall event rate.
func (lim *Limiter) Limit() Limit {
lim.mu.Lock()
defer lim.mu.Unlock()
return lim.limit
}
// Burst returns the maximum burst size. Burst is the maximum number of tokens
// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
// Burst values allow more events to happen at once.
// A zero Burst allows no events, unless limit == Inf.
func (lim *Limiter) Burst() int {
return lim.burst
}
// NewLimiter returns a new Limiter that allows events up to rate r and permits
// bursts of at most b tokens.
func NewLimiter(r Limit, b int) *Limiter {
return &Limiter{
limit: r,
burst: b,
}
}
// Allow is shorthand for AllowN(time.Now(), 1).
func (lim *Limiter) Allow() bool {
return lim.AllowN(time.Now(), 1)
}
// AllowN reports whether n events may happen at time now.
// Use this method if you intend to drop / skip events that exceed the rate limit.
// Otherwise use Reserve or Wait.
func (lim *Limiter) AllowN(now time.Time, n int) bool {
return lim.reserveN(now, n, 0).ok
}
// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
// A Reservation may be canceled, which may enable the Limiter to permit additional events.
type Reservation struct {
ok bool
lim *Limiter
tokens int
timeToAct time.Time
// This is the Limit at reservation time, it can change later.
limit Limit
}
// OK returns whether the limiter can provide the requested number of tokens
// within the maximum wait time. If OK is false, Delay returns InfDuration, and
// Cancel does nothing.
func (r *Reservation) OK() bool {
return r.ok
}
// Delay is shorthand for DelayFrom(time.Now()).
func (r *Reservation) Delay() time.Duration {
return r.DelayFrom(time.Now())
}
// InfDuration is the duration returned by Delay when a Reservation is not OK.
const InfDuration = time.Duration(1<<63 - 1)
// DelayFrom returns the duration for which the reservation holder must wait
// before taking the reserved action. Zero duration means act immediately.
// InfDuration means the limiter cannot grant the tokens requested in this
// Reservation within the maximum wait time.
func (r *Reservation) DelayFrom(now time.Time) time.Duration {
if !r.ok {
return InfDuration
}
delay := r.timeToAct.Sub(now)
if delay < 0 {
return 0
}
return delay
}
// Cancel is shorthand for CancelAt(time.Now()).
func (r *Reservation) Cancel() {
r.CancelAt(time.Now())
}
// CancelAt indicates that the reservation holder will not perform the reserved action
// and reverses the effects of this Reservation on the rate limit as much as possible,
// considering that other reservations may have already been made.
func (r *Reservation) CancelAt(now time.Time) {
if !r.ok {
return
}
r.lim.mu.Lock()
defer r.lim.mu.Unlock()
if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
return
}
// calculate tokens to restore
// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
// after r was obtained. These tokens should not be restored.
restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
if restoreTokens <= 0 {
return
}
// advance time to now
now, _, tokens := r.lim.advance(now)
// calculate new number of tokens
tokens += restoreTokens
if burst := float64(r.lim.burst); tokens > burst {
tokens = burst
}
// update state
r.lim.last = now
r.lim.tokens = tokens
if r.timeToAct == r.lim.lastEvent {
prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
if !prevEvent.Before(now) {
r.lim.lastEvent = prevEvent
}
}
}
// Reserve is shorthand for ReserveN(time.Now(), 1).
func (lim *Limiter) Reserve() Reservation {
return lim.ReserveN(time.Now(), 1)
}
// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
// The Limiter takes this Reservation into account when allowing future events.
// ReserveN returns false if n exceeds the Limiter's burst size.
// Usage example:
// r := lim.ReserveN(time.Now(), 1)
// if !r.OK() {
// // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
// return
// }
// time.Sleep(r.Delay())
// Act()
// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
// If you need to respect a deadline or cancel the delay, use Wait instead.
// To drop or skip events exceeding rate limit, use Allow instead.
func (lim *Limiter) ReserveN(now time.Time, n int) Reservation {
return lim.reserveN(now, n, InfDuration)
}
// Wait is shorthand for WaitN(ctx, 1).
func (lim *Limiter) Wait(ctx context.Context) (err error) {
return lim.WaitN(ctx, 1)
}
// WaitN blocks until lim permits n events to happen.
// It returns an error if n exceeds the Limiter's burst size, the Context is
// canceled, or the expected wait time exceeds the Context's Deadline.
func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
if n > lim.burst && lim.limit != Inf {
return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst))
}
// Check if ctx is already cancelled
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// Determine wait limit
now := time.Now()
waitLimit := InfDuration
if deadline, ok := ctx.Deadline(); ok {
waitLimit = deadline.Sub(now)
}
// Reserve
r := lim.reserveN(now, n, waitLimit)
if !r.ok {
return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n))
}
// Wait
d := r.DelayFrom(now)
if d <= 0 {
return nil
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
// We can proceed.
return nil
case <-ctx.Done():
// Context was canceled before we could proceed. Cancel the
// reservation, which may permit other events to proceed sooner.
r.Cancel()
return ctx.Err()
}
}
// Delay is shorthand for DelayN(time.Now(), 1).
func (lim *Limiter) Delay() time.Duration {
return lim.DelayN(time.Now(), 1)
}
// DelayN returns the delay to wait to permit n events to happen. Zero duration
// means act immediately. InfDuration means the limiter cannot grant the tokens
// requested within the maximum wait time.
func (lim *Limiter) DelayN(now time.Time, n int) time.Duration {
r := lim.reserveN(now, n, InfDuration)
return r.DelayFrom(now)
}
// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
func (lim *Limiter) SetLimit(newLimit Limit) {
lim.SetLimitAt(time.Now(), newLimit)
}
// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
// before SetLimitAt was called.
func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
lim.last = now
lim.tokens = tokens
lim.limit = newLimit
}
// reserveN is a helper method for AllowN, ReserveN, and WaitN.
// maxFutureReserve specifies the maximum reservation wait duration allowed.
// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
lim.mu.Lock()
if lim.limit == Inf {
lim.mu.Unlock()
return Reservation{
ok: true,
lim: lim,
tokens: n,
timeToAct: now,
}
}
now, last, tokens := lim.advance(now)
// Calculate the remaining number of tokens resulting from the request.
tokens -= float64(n)
// Calculate the wait duration
var waitDuration time.Duration
if tokens < 0 {
waitDuration = lim.limit.durationFromTokens(-tokens)
}
// Decide result
ok := n <= lim.burst && waitDuration <= maxFutureReserve
// Prepare reservation
r := Reservation{
ok: ok,
lim: lim,
limit: lim.limit,
}
if ok {
r.tokens = n
r.timeToAct = now.Add(waitDuration)
}
// Update state
if ok {
lim.last = now
lim.tokens = tokens
lim.lastEvent = r.timeToAct
} else {
lim.last = last
}
lim.mu.Unlock()
return r
}
// advance calculates and returns an updated state for lim resulting from the passage of time.
// lim is not changed.
func (lim *Limiter) advance(
now time.Time,
) (newNow time.Time, newLast time.Time, newTokens float64) {
last := lim.last
if now.Before(last) {
last = now
}
// Avoid making delta overflow below when last is very old.
maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
elapsed := now.Sub(last)
if elapsed > maxElapsed {
elapsed = maxElapsed
}
// Calculate the new number of tokens, due to time that passed.
delta := lim.limit.tokensFromDuration(elapsed)
tokens := lim.tokens + delta
if burst := float64(lim.burst); tokens > burst {
tokens = burst
}
return now, last, tokens
}
// durationFromTokens is a unit conversion function from the number of tokens to the duration
// of time it takes to accumulate them at a rate of limit tokens per second.
func (limit Limit) durationFromTokens(tokens float64) time.Duration {
seconds := tokens / float64(limit)
return time.Nanosecond * time.Duration(1e9*seconds)
}
// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
// which could be accumulated during that duration at a rate of limit tokens per second.
func (limit Limit) tokensFromDuration(d time.Duration) float64 {
return d.Seconds() * float64(limit)
}

Some files were not shown because too many files have changed in this diff Show More