aleesa-bot/aleesa-jabber-go
0
Fork 0
Code Activity

Initial commit

Browse Source
This commit is contained in:
eleksir
2023-09-17 12:18:09 +03:00
commit 843dd418b0
1514 changed files with 643531 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
vendor/github.com/cockroachdb/pebble/.editorconfig generated vendored Normal file
View File

@@ -0,0 +1,10 @@
# See http://editorconfig.org
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
# For non-go files, we indent with two spaces. In go files we indent
# with tabs but still set indent_size to control the github web viewer.
indent_size=2

9
vendor/github.com/cockroachdb/pebble/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,9 @@
# Github action artifacts.
artifacts
# Profiling artifacts.
cpu.*.prof
heap.prof
mutex.prof
coverprofile.out
# Testing artifacts
meta.*.test

27
vendor/github.com/cockroachdb/pebble/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,27 @@
Copyright (c) 2011 The LevelDB-Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

131
vendor/github.com/cockroachdb/pebble/Makefile generated vendored Normal file
View File

@@ -0,0 +1,131 @@
GO := go
PKG := ./...
GOFLAGS :=
STRESSFLAGS :=
TAGS := invariants
TESTS := .
LATEST_RELEASE := $(shell git fetch origin && git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$$' | sort | tail -1)
COVER_PROFILE := coverprofile.out
.PHONY: all
all:
@echo usage:
@echo " make test"
@echo " make testrace"
@echo " make stress"
@echo " make stressrace"
@echo " make stressmeta"
@echo " make crossversion-meta"
@echo " make testcoverage"
@echo " make mod-update"
@echo " make generate"
@echo " make generate-test-data"
@echo " make clean"
override testflags :=
.PHONY: test
test:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG}
.PHONY: testcoverage
testcoverage:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} -coverprofile ${COVER_PROFILE}
.PHONY: testrace
testrace: testflags += -race -timeout 20m
testrace: test
testasan: testflags += -asan -timeout 20m
testasan: test
testmsan: export CC=clang
testmsan: testflags += -msan -timeout 20m
testmsan: test
.PHONY: testobjiotracing
testobjiotracing:
${GO} test -tags '$(TAGS) pebble_obj_io_tracing' ${testflags} -run ${TESTS} ./objstorage/objstorageprovider/objiotracing
.PHONY: lint
lint:
${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ./internal/lint
.PHONY: stress stressrace
stressrace: testflags += -race
stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 -test.v
stress stressrace: test
.PHONY: stressmeta
stressmeta: override PKG = ./internal/metamorphic
stressmeta: override STRESSFLAGS += -p 1
stressmeta: override TESTS = TestMeta$$
stressmeta: stress
.PHONY: crossversion-meta
crossversion-meta:
git checkout ${LATEST_RELEASE}; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \
git checkout -; \
${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
.PHONY: stress-crossversion
stress-crossversion:
STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 crl-release-23.1 master
.PHONY: generate
generate:
${GO} generate ${PKG}
generate:
# Note that the output of generate-test-data is not deterministic. This should
# only be run manually as needed.
.PHONY: generate-test-data
generate-test-data:
${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
${GO} run -tags make_test_find_db ./tool/make_test_find_db.go
${GO} run -tags make_test_sstables ./tool/make_test_sstables.go
${GO} run -tags make_test_remotecat ./tool/make_test_remotecat.go
mod-update:
${GO} get -u
${GO} mod tidy
.PHONY: clean
clean:
rm -f $(patsubst %,%.test,$(notdir $(shell go list ${PKG})))
git_dirty := $(shell git status -s)
.PHONY: git-clean-check
git-clean-check:
ifneq ($(git_dirty),)
@echo "Git repository is dirty!"
@false
else
@echo "Git repository is clean."
endif
.PHONY: mod-tidy-check
mod-tidy-check:
ifneq ($(git_dirty),)
$(error mod-tidy-check must be invoked on a clean repository)
endif
@${GO} mod tidy
$(MAKE) git-clean-check
# TODO(radu): switch back to @latest once bogus doc changes are
# addressed; see https://github.com/cockroachdb/crlfmt/pull/44
.PHONY: format
format:
go install github.com/cockroachdb/crlfmt@44a36ec7 && crlfmt -w -tab 2 .
.PHONY: format-check
format-check:
ifneq ($(git_dirty),)
$(error format-check must be invoked on a clean repository)
endif
$(MAKE) format
git diff
$(MAKE) git-clean-check

157
vendor/github.com/cockroachdb/pebble/README.md generated vendored Normal file
View File

@@ -0,0 +1,157 @@
# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble)
#### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
Pebble is a LevelDB/RocksDB inspired key-value store focused on
performance and internal usage by CockroachDB. Pebble inherits the
RocksDB file formats and a few extensions such as range deletion
tombstones, table-level bloom filters, and updates to the MANIFEST
format.
Pebble intentionally does not aspire to include every feature in
RocksDB and is specifically targetting the use case and feature set
needed by CockroachDB:
* Block-based tables
* Checkpoints
* Indexed batches
* Iterator options (lower/upper bound, table filter)
* Level-based compaction
* Manual compaction
* Merge operator
* Prefix bloom filters
* Prefix iteration
* Range deletion tombstones
* Reverse iteration
* SSTable ingestion
* Single delete
* Snapshots
* Table-level bloom filters
RocksDB has a large number of features that are not implemented in
Pebble:
* Backups
* Column families
* Delete files in range
* FIFO compaction style
* Forward iterator / tailing iterator
* Hash table format
* Memtable bloom filter
* Persistent cache
* Pin iterator key / value
* Plain table format
* SSTable ingest-behind
* Sub-compactions
* Transactions
* Universal compaction style
***WARNING***: Pebble may silently corrupt data or behave incorrectly if
used with a RocksDB database that uses a feature Pebble doesn't
support. Caveat emptor!
## Production Ready
Pebble was introduced as an alternative storage engine to RocksDB in
CockroachDB v20.1 (released May 2020) and was used in production
successfully at that time. Pebble was made the default storage engine
in CockroachDB v20.2 (released Nov 2020). Pebble is being used in
production by users of CockroachDB at scale and is considered stable
and production ready.
## Advantages
Pebble offers several improvements over RocksDB:
* Faster reverse iteration via backwards links in the memtable's
skiplist.
* Faster commit pipeline that achieves better concurrency.
* Seamless merged iteration of indexed batches. The mutations in the
batch conceptually occupy another memtable level.
* Smaller, more approachable code base.
See the [Pebble vs RocksDB: Implementation
Differences](docs/rocksdb.md) doc for more details on implementation
differences.
## RocksDB Compatibility
Pebble strives for forward compatibility with RocksDB 6.2.1 (the latest
version of RocksDB used by CockroachDB). Forward compatibility means
that a DB generated by RocksDB can be used by Pebble. Currently, Pebble
provides bidirectional compatibility with RocksDB (a Pebble generated DB
can be used by RocksDB) when using its FormatMostCompatible format. New
functionality that is backwards incompatible is gated behind new format
major versions. In general, Pebble only provides compatibility with the
subset of functionality and configuration used by CockroachDB. The scope
of RocksDB functionality and configuration is too large to adequately
test and document all the incompatibilities. The list below contains
known incompatibilities.
* Pebble's use of WAL recycling is only compatible with RocksDB's
`kTolerateCorruptedTailRecords` WAL recovery mode. Older versions of
RocksDB would automatically map incompatible WAL recovery modes to
`kTolerateCorruptedTailRecords`. New versions of RocksDB will
disable WAL recycling.
* Column families. Pebble does not support column families, nor does
it attempt to detect their usage when opening a DB that may contain
them.
* Hash table format. Pebble does not support the hash table sstable
format.
* Plain table format. Pebble does not support the plain table sstable
format.
* SSTable format version 3 and 4. Pebble does not currently support
version 3 and version 4 format sstables. The sstable format version
is controlled by the `BlockBasedTableOptions::format_version`
option. See [#97](https://github.com/cockroachdb/pebble/issues/97).
## Pedigree
Pebble is based on the incomplete Go version of LevelDB:
https://github.com/golang/leveldb
The Go version of LevelDB is based on the C++ original:
https://github.com/google/leveldb
Optimizations and inspiration were drawn from RocksDB:
https://github.com/facebook/rocksdb
## Getting Started
### Example Code
```go
package main
import (
"fmt"
"log"
"github.com/cockroachdb/pebble"
)
func main() {
db, err := pebble.Open("demo", &pebble.Options{})
if err != nil {
log.Fatal(err)
}
key := []byte("hello")
if err := db.Set(key, []byte("world"), pebble.Sync); err != nil {
log.Fatal(err)
}
value, closer, err := db.Get(key)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s %s\n", key, value)
if err := closer.Close(); err != nil {
log.Fatal(err)
}
if err := db.Close(); err != nil {
log.Fatal(err)
}
}
```

2251
vendor/github.com/cockroachdb/pebble/batch.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

23
vendor/github.com/cockroachdb/pebble/cache.go generated vendored Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import "github.com/cockroachdb/pebble/internal/cache"
// Cache exports the cache.Cache type.
type Cache = cache.Cache
// NewCache creates a new cache of the specified size. Memory for the cache is
// allocated on demand, not during initialization. The cache is created with a
// reference count of 1. Each DB it is associated with adds a reference, so the
// creator of the cache should usually release their reference after the DB is
// created.
//
// c := pebble.NewCache(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func NewCache(size int64) *cache.Cache {
return cache.New(size)
}

431
vendor/github.com/cockroachdb/pebble/checkpoint.go generated vendored Normal file
View File

@@ -0,0 +1,431 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"io"
"os"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/record"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
// checkpointOptions hold the optional parameters to construct checkpoint
// snapshots.
type checkpointOptions struct {
// flushWAL set to true will force a flush and sync of the WAL prior to
// checkpointing.
flushWAL bool
// If set, any SSTs that don't overlap with these spans are excluded from a checkpoint.
restrictToSpans []CheckpointSpan
}
// CheckpointOption set optional parameters used by `DB.Checkpoint`.
type CheckpointOption func(*checkpointOptions)
// WithFlushedWAL enables flushing and syncing the WAL prior to constructing a
// checkpoint. This guarantees that any writes committed before calling
// DB.Checkpoint will be part of that checkpoint.
//
// Note that this setting can only be useful in cases when some writes are
// performed with Sync = false. Otherwise, the guarantee will already be met.
//
// Passing this option is functionally equivalent to calling
// DB.LogData(nil, Sync) right before DB.Checkpoint.
func WithFlushedWAL() CheckpointOption {
return func(opt *checkpointOptions) {
opt.flushWAL = true
}
}
// WithRestrictToSpans specifies spans of interest for the checkpoint. Any SSTs
// that don't overlap with any of these spans are excluded from the checkpoint.
//
// Note that the checkpoint can still surface keys outside of these spans (from
// the WAL and from SSTs that partially overlap with these spans). Moreover,
// these surface keys aren't necessarily "valid" in that they could have been
// modified but the SST containing the modification is excluded.
func WithRestrictToSpans(spans []CheckpointSpan) CheckpointOption {
return func(opt *checkpointOptions) {
opt.restrictToSpans = spans
}
}
// CheckpointSpan is a key range [Start, End) (inclusive on Start, exclusive on
// End) of interest for a checkpoint.
type CheckpointSpan struct {
Start []byte
End []byte
}
// excludeFromCheckpoint returns true if an SST file should be excluded from the
// checkpoint because it does not overlap with the spans of interest
// (opt.restrictToSpans).
func excludeFromCheckpoint(f *fileMetadata, opt *checkpointOptions, cmp Compare) bool {
if len(opt.restrictToSpans) == 0 {
// Option not set; don't exclude anything.
return false
}
for _, s := range opt.restrictToSpans {
if f.Overlaps(cmp, s.Start, s.End, true /* exclusiveEnd */) {
return false
}
}
// None of the restrictToSpans overlapped; we can exclude this file.
return true
}
// mkdirAllAndSyncParents creates destDir and any of its missing parents.
// Those missing parents, as well as the closest existing ancestor, are synced.
// Returns a handle to the directory created at destDir.
func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
// Collect paths for all directories between destDir (excluded) and its
// closest existing ancestor (included).
var parentPaths []string
foundExistingAncestor := false
for parentPath := fs.PathDir(destDir); parentPath != "."; parentPath = fs.PathDir(parentPath) {
parentPaths = append(parentPaths, parentPath)
_, err := fs.Stat(parentPath)
if err == nil {
// Exit loop at the closest existing ancestor.
foundExistingAncestor = true
break
}
if !oserror.IsNotExist(err) {
return nil, err
}
}
// Handle empty filesystem edge case.
if !foundExistingAncestor {
parentPaths = append(parentPaths, "")
}
// Create destDir and any of its missing parents.
if err := fs.MkdirAll(destDir, 0755); err != nil {
return nil, err
}
// Sync all the parent directories up to the closest existing ancestor,
// included.
for _, parentPath := range parentPaths {
parentDir, err := fs.OpenDir(parentPath)
if err != nil {
return nil, err
}
err = parentDir.Sync()
if err != nil {
_ = parentDir.Close()
return nil, err
}
err = parentDir.Close()
if err != nil {
return nil, err
}
}
return fs.OpenDir(destDir)
}
// Checkpoint constructs a snapshot of the DB instance in the specified
// directory. The WAL, MANIFEST, OPTIONS, and sstables will be copied into the
// snapshot. Hard links will be used when possible. Beware of the significant
// space overhead for a checkpoint if hard links are disabled. Also beware that
// even if hard links are used, the space overhead for the checkpoint will
// increase over time as the DB performs compactions.
//
// TODO(bananabrick): Test checkpointing of virtual sstables once virtual
// sstables is running e2e.
func (d *DB) Checkpoint(
destDir string, opts ...CheckpointOption,
) (
ckErr error, /* used in deferred cleanup */
) {
opt := &checkpointOptions{}
for _, fn := range opts {
fn(opt)
}
if _, err := d.opts.FS.Stat(destDir); !oserror.IsNotExist(err) {
if err == nil {
return &os.PathError{
Op: "checkpoint",
Path: destDir,
Err: oserror.ErrExist,
}
}
return err
}
if opt.flushWAL && !d.opts.DisableWAL {
// Write an empty log-data record to flush and sync the WAL.
if err := d.LogData(nil /* data */, Sync); err != nil {
return err
}
}
// Disable file deletions.
d.mu.Lock()
d.disableFileDeletions()
defer func() {
d.mu.Lock()
defer d.mu.Unlock()
d.enableFileDeletions()
}()
// TODO(peter): RocksDB provides the option to roll the manifest if the
// MANIFEST size is too large. Should we do this too?
// Lock the manifest before getting the current version. We need the
// length of the manifest that we read to match the current version that
// we read, otherwise we might copy a versionEdit not reflected in the
// sstables we copy/link.
d.mu.versions.logLock()
// Get the unflushed log files, the current version, and the current manifest
// file number.
memQueue := d.mu.mem.queue
current := d.mu.versions.currentVersion()
formatVers := d.FormatMajorVersion()
manifestFileNum := d.mu.versions.manifestFileNum
manifestSize := d.mu.versions.manifest.Size()
optionsFileNum := d.optionsFileNum
virtualBackingFiles := make(map[base.DiskFileNum]struct{})
for diskFileNum := range d.mu.versions.fileBackingMap {
virtualBackingFiles[diskFileNum] = struct{}{}
}
// Release the manifest and DB.mu so we don't block other operations on
// the database.
d.mu.versions.logUnlock()
d.mu.Unlock()
// Wrap the normal filesystem with one which wraps newly created files with
// vfs.NewSyncingFile.
fs := vfs.NewSyncingFS(d.opts.FS, vfs.SyncingFileOptions{
NoSyncOnClose: d.opts.NoSyncOnClose,
BytesPerSync: d.opts.BytesPerSync,
})
// Create the dir and its parents (if necessary), and sync them.
var dir vfs.File
defer func() {
if dir != nil {
_ = dir.Close()
}
if ckErr != nil {
// Attempt to cleanup on error.
_ = fs.RemoveAll(destDir)
}
}()
dir, ckErr = mkdirAllAndSyncParents(fs, destDir)
if ckErr != nil {
return ckErr
}
{
// Link or copy the OPTIONS.
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeOptions, optionsFileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
if ckErr != nil {
return ckErr
}
}
{
// Set the format major version in the destination directory.
var versionMarker *atomicfs.Marker
versionMarker, _, ckErr = atomicfs.LocateMarker(fs, destDir, formatVersionMarkerName)
if ckErr != nil {
return ckErr
}
// We use the marker to encode the active format version in the
// marker filename. Unlike other uses of the atomic marker,
// there is no file with the filename `formatVers.String()` on
// the filesystem.
ckErr = versionMarker.Move(formatVers.String())
if ckErr != nil {
return ckErr
}
ckErr = versionMarker.Close()
if ckErr != nil {
return ckErr
}
}
var excludedFiles map[deletedFileEntry]*fileMetadata
// Set of FileBacking.DiskFileNum which will be required by virtual sstables
// in the checkpoint.
requiredVirtualBackingFiles := make(map[base.DiskFileNum]struct{})
// Link or copy the sstables.
for l := range current.Levels {
iter := current.Levels[l].Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if excludeFromCheckpoint(f, opt, d.cmp) {
if excludedFiles == nil {
excludedFiles = make(map[deletedFileEntry]*fileMetadata)
}
excludedFiles[deletedFileEntry{
Level: l,
FileNum: f.FileNum,
}] = f
continue
}
fileBacking := f.FileBacking
if f.Virtual {
if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok {
continue
}
requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{}
}
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum)
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
if ckErr != nil {
return ckErr
}
}
}
var removeBackingTables []base.DiskFileNum
for diskFileNum := range virtualBackingFiles {
if _, ok := requiredVirtualBackingFiles[diskFileNum]; !ok {
// The backing sstable associated with fileNum is no longer
// required.
removeBackingTables = append(removeBackingTables, diskFileNum)
}
}
ckErr = d.writeCheckpointManifest(
fs, formatVers, destDir, dir, manifestFileNum.DiskFileNum(), manifestSize,
excludedFiles, removeBackingTables,
)
if ckErr != nil {
return ckErr
}
// Copy the WAL files. We copy rather than link because WAL file recycling
// will cause the WAL files to be reused which would invalidate the
// checkpoint.
for i := range memQueue {
logNum := memQueue[i].logNum
if logNum == 0 {
continue
}
srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum.DiskFileNum())
destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
ckErr = vfs.Copy(fs, srcPath, destPath)
if ckErr != nil {
return ckErr
}
}
// Sync and close the checkpoint directory.
ckErr = dir.Sync()
if ckErr != nil {
return ckErr
}
ckErr = dir.Close()
dir = nil
return ckErr
}
func (d *DB) writeCheckpointManifest(
fs vfs.FS,
formatVers FormatMajorVersion,
destDirPath string,
destDir vfs.File,
manifestFileNum base.DiskFileNum,
manifestSize int64,
excludedFiles map[deletedFileEntry]*fileMetadata,
removeBackingTables []base.DiskFileNum,
) error {
// Copy the MANIFEST, and create a pointer to it. We copy rather
// than link because additional version edits added to the
// MANIFEST after we took our snapshot of the sstables will
// reference sstables that aren't in our checkpoint. For a
// similar reason, we need to limit how much of the MANIFEST we
// copy.
// If some files are excluded from the checkpoint, also append a block that
// records those files as deleted.
if err := func() error {
srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum)
destPath := fs.PathJoin(destDirPath, fs.PathBase(srcPath))
src, err := fs.Open(srcPath, vfs.SequentialReadsOption)
if err != nil {
return err
}
defer src.Close()
dst, err := fs.Create(destPath)
if err != nil {
return err
}
defer dst.Close()
// Copy all existing records. We need to copy at the record level in case we
// need to append another record with the excluded files (we cannot simply
// append a record after a raw data copy; see
// https://github.com/cockroachdb/cockroach/issues/100935).
r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum.FileNum())
w := record.NewWriter(dst)
for {
rr, err := r.Next()
if err != nil {
if err == io.EOF {
break
}
return err
}
rw, err := w.Next()
if err != nil {
return err
}
if _, err := io.Copy(rw, rr); err != nil {
return err
}
}
if len(excludedFiles) > 0 {
// Write out an additional VersionEdit that deletes the excluded SST files.
ve := versionEdit{
DeletedFiles: excludedFiles,
RemovedBackingTables: removeBackingTables,
}
rw, err := w.Next()
if err != nil {
return err
}
if err := ve.Encode(rw); err != nil {
return err
}
}
if err := w.Close(); err != nil {
return err
}
return dst.Sync()
}(); err != nil {
return err
}
// Recent format versions use an atomic marker for setting the
// active manifest. Older versions use the CURRENT file. The
// setCurrentFunc function will return a closure that will
// take the appropriate action for the database's format
// version.
var manifestMarker *atomicfs.Marker
manifestMarker, _, err := atomicfs.LocateMarker(fs, destDirPath, manifestMarkerName)
if err != nil {
return err
}
if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum.FileNum()); err != nil {
return err
}
return manifestMarker.Close()
}

295
vendor/github.com/cockroachdb/pebble/cleaner.go generated vendored Normal file
View File

@@ -0,0 +1,295 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"context"
"runtime/pprof"
"sync"
"time"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/objstorage"
"github.com/cockroachdb/tokenbucket"
)
// Cleaner exports the base.Cleaner type.
type Cleaner = base.Cleaner
// DeleteCleaner exports the base.DeleteCleaner type.
type DeleteCleaner = base.DeleteCleaner
// ArchiveCleaner exports the base.ArchiveCleaner type.
type ArchiveCleaner = base.ArchiveCleaner
type cleanupManager struct {
opts *Options
objProvider objstorage.Provider
onTableDeleteFn func(fileSize uint64)
deletePacer *deletionPacer
// jobsCh is used as the cleanup job queue.
jobsCh chan *cleanupJob
// waitGroup is used to wait for the background goroutine to exit.
waitGroup sync.WaitGroup
mu struct {
sync.Mutex
// totalJobs is the total number of enqueued jobs (completed or in progress).
totalJobs int
completedJobs int
completedJobsCond sync.Cond
jobsQueueWarningIssued bool
}
}
// We can queue this many jobs before we have to block EnqueueJob.
const jobsQueueDepth = 1000
// obsoleteFile holds information about a file that needs to be deleted soon.
type obsoleteFile struct {
dir string
fileNum base.DiskFileNum
fileType fileType
fileSize uint64
}
type cleanupJob struct {
jobID int
obsoleteFiles []obsoleteFile
}
// openCleanupManager creates a cleanupManager and starts its background goroutine.
// The cleanupManager must be Close()d.
func openCleanupManager(
opts *Options,
objProvider objstorage.Provider,
onTableDeleteFn func(fileSize uint64),
getDeletePacerInfo func() deletionPacerInfo,
) *cleanupManager {
cm := &cleanupManager{
opts: opts,
objProvider: objProvider,
onTableDeleteFn: onTableDeleteFn,
deletePacer: newDeletionPacer(time.Now(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo),
jobsCh: make(chan *cleanupJob, jobsQueueDepth),
}
cm.mu.completedJobsCond.L = &cm.mu.Mutex
cm.waitGroup.Add(1)
go func() {
pprof.Do(context.Background(), gcLabels, func(context.Context) {
cm.mainLoop()
})
}()
return cm
}
// Close stops the background goroutine, waiting until all queued jobs are completed.
// Delete pacing is disabled for the remaining jobs.
func (cm *cleanupManager) Close() {
close(cm.jobsCh)
cm.waitGroup.Wait()
}
// EnqueueJob adds a cleanup job to the manager's queue.
func (cm *cleanupManager) EnqueueJob(jobID int, obsoleteFiles []obsoleteFile) {
job := &cleanupJob{
jobID: jobID,
obsoleteFiles: obsoleteFiles,
}
// Report deleted bytes to the pacer, which can use this data to potentially
// increase the deletion rate to keep up. We want to do this at enqueue time
// rather than when we get to the job, otherwise the reported bytes will be
// subject to the throttling rate which defeats the purpose.
var pacingBytes uint64
for _, of := range obsoleteFiles {
if cm.needsPacing(of.fileType, of.fileNum) {
pacingBytes += of.fileSize
}
}
if pacingBytes > 0 {
cm.deletePacer.ReportDeletion(time.Now(), pacingBytes)
}
cm.mu.Lock()
cm.mu.totalJobs++
cm.maybeLogLocked()
cm.mu.Unlock()
if invariants.Enabled && len(cm.jobsCh) >= cap(cm.jobsCh)-2 {
panic("cleanup jobs queue full")
}
cm.jobsCh <- job
}
// Wait until the completion of all jobs that were already queued.
//
// Does not wait for jobs that are enqueued during the call.
//
// Note that DB.mu should not be held while calling this method; the background
// goroutine needs to acquire DB.mu to update deleted table metrics.
func (cm *cleanupManager) Wait() {
cm.mu.Lock()
defer cm.mu.Unlock()
n := cm.mu.totalJobs
for cm.mu.completedJobs < n {
cm.mu.completedJobsCond.Wait()
}
}
// mainLoop runs the manager's background goroutine.
func (cm *cleanupManager) mainLoop() {
defer cm.waitGroup.Done()
var tb tokenbucket.TokenBucket
// Use a token bucket with 1 token / second refill rate and 1 token burst.
tb.Init(1.0, 1.0)
for job := range cm.jobsCh {
for _, of := range job.obsoleteFiles {
if of.fileType != fileTypeTable {
path := base.MakeFilepath(cm.opts.FS, of.dir, of.fileType, of.fileNum)
cm.deleteObsoleteFile(of.fileType, job.jobID, path, of.fileNum, of.fileSize)
} else {
cm.maybePace(&tb, of.fileType, of.fileNum, of.fileSize)
cm.onTableDeleteFn(of.fileSize)
cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.fileNum)
}
}
cm.mu.Lock()
cm.mu.completedJobs++
cm.mu.completedJobsCond.Broadcast()
cm.maybeLogLocked()
cm.mu.Unlock()
}
}
func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNum base.DiskFileNum) bool {
if fileType != fileTypeTable {
return false
}
meta, err := cm.objProvider.Lookup(fileType, fileNum)
if err != nil {
// The object was already removed from the provider; we won't actually
// delete anything, so we don't need to pace.
return false
}
// Don't throttle deletion of remote objects.
return !meta.IsRemote()
}
// maybePace sleeps before deleting an object if appropriate. It is always
// called from the background goroutine.
func (cm *cleanupManager) maybePace(
tb *tokenbucket.TokenBucket, fileType base.FileType, fileNum base.DiskFileNum, fileSize uint64,
) {
if !cm.needsPacing(fileType, fileNum) {
return
}
tokens := cm.deletePacer.PacingDelay(time.Now(), fileSize)
if tokens == 0.0 {
// The token bucket might be in debt; it could make us wait even for 0
// tokens. We don't want that if the pacer decided throttling should be
// disabled.
return
}
// Wait for tokens. We use a token bucket instead of sleeping outright because
// the token bucket accumulates up to one second of unused tokens.
for {
ok, d := tb.TryToFulfill(tokenbucket.Tokens(tokens))
if ok {
break
}
time.Sleep(d)
}
}
// deleteObsoleteFile deletes a (non-object) file that is no longer needed.
func (cm *cleanupManager) deleteObsoleteFile(
fileType fileType, jobID int, path string, fileNum base.DiskFileNum, fileSize uint64,
) {
// TODO(peter): need to handle this error, probably by re-adding the
// file that couldn't be deleted to one of the obsolete slices map.
err := cm.opts.Cleaner.Clean(cm.opts.FS, fileType, path)
if oserror.IsNotExist(err) {
return
}
switch fileType {
case fileTypeLog:
cm.opts.EventListener.WALDeleted(WALDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
case fileTypeManifest:
cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
case fileTypeTable:
panic("invalid deletion of object file")
}
}
func (cm *cleanupManager) deleteObsoleteObject(
fileType fileType, jobID int, fileNum base.DiskFileNum,
) {
if fileType != fileTypeTable {
panic("not an object")
}
var path string
meta, err := cm.objProvider.Lookup(fileType, fileNum)
if err != nil {
path = "<nil>"
} else {
path = cm.objProvider.Path(meta)
err = cm.objProvider.Remove(fileType, fileNum)
}
if cm.objProvider.IsNotExistError(err) {
return
}
switch fileType {
case fileTypeTable:
cm.opts.EventListener.TableDeleted(TableDeleteInfo{
JobID: jobID,
Path: path,
FileNum: fileNum.FileNum(),
Err: err,
})
}
}
// maybeLogLocked issues a log if the job queue gets 75% full and issues a log
// when the job queue gets back to less than 10% full.
//
// Must be called with cm.mu locked.
func (cm *cleanupManager) maybeLogLocked() {
const highThreshold = jobsQueueDepth * 3 / 4
const lowThreshold = jobsQueueDepth / 10
jobsInQueue := cm.mu.totalJobs - cm.mu.completedJobs
if !cm.mu.jobsQueueWarningIssued && jobsInQueue > highThreshold {
cm.mu.jobsQueueWarningIssued = true
cm.opts.Logger.Infof("cleanup falling behind; job queue has over %d jobs", highThreshold)
}
if cm.mu.jobsQueueWarningIssued && jobsInQueue < lowThreshold {
cm.mu.jobsQueueWarningIssued = false
cm.opts.Logger.Infof("cleanup back to normal; job queue has under %d jobs", lowThreshold)
}
}

517
vendor/github.com/cockroachdb/pebble/commit.go generated vendored Normal file
View File

@@ -0,0 +1,517 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/cockroachdb/pebble/record"
)
// commitQueue is a lock-free fixed-size single-producer, multi-consumer
// queue. The single producer can enqueue (push) to the head, and consumers can
// dequeue (pop) from the tail.
//
// It has the added feature that it nils out unused slots to avoid unnecessary
// retention of objects.
type commitQueue struct {
// headTail packs together a 32-bit head index and a 32-bit tail index. Both
// are indexes into slots modulo len(slots)-1.
//
// tail = index of oldest data in queue
// head = index of next slot to fill
//
// Slots in the range [tail, head) are owned by consumers. A consumer
// continues to own a slot outside this range until it nils the slot, at
// which point ownership passes to the producer.
//
// The head index is stored in the most-significant bits so that we can
// atomically add to it and the overflow is harmless.
headTail atomic.Uint64
// slots is a ring buffer of values stored in this queue. The size must be a
// power of 2. A slot is in use until *both* the tail index has moved beyond
// it and the slot value has been set to nil. The slot value is set to nil
// atomically by the consumer and read atomically by the producer.
slots [record.SyncConcurrency]atomic.Pointer[Batch]
}
const dequeueBits = 32
func (q *commitQueue) unpack(ptrs uint64) (head, tail uint32) {
const mask = 1<<dequeueBits - 1
head = uint32((ptrs >> dequeueBits) & mask)
tail = uint32(ptrs & mask)
return
}
func (q *commitQueue) pack(head, tail uint32) uint64 {
const mask = 1<<dequeueBits - 1
return (uint64(head) << dequeueBits) |
uint64(tail&mask)
}
func (q *commitQueue) enqueue(b *Batch) {
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
// Queue is full. This should never be reached because commitPipeline.commitQueueSem
// limits the number of concurrent operations.
panic("pebble: not reached")
}
slot := &q.slots[head&uint32(len(q.slots)-1)]
// Check if the head slot has been released by dequeueApplied.
for slot.Load() != nil {
// Another goroutine is still cleaning up the tail, so the queue is
// actually still full. We spin because this should resolve itself
// momentarily.
runtime.Gosched()
}
// The head slot is free, so we own it.
slot.Store(b)
// Increment head. This passes ownership of slot to dequeueApplied and acts as a
// store barrier for writing the slot.
q.headTail.Add(1 << dequeueBits)
}
// dequeueApplied removes the earliest enqueued Batch, if it is applied.
//
// Returns nil if the commit queue is empty or the earliest Batch is not yet
// applied.
func (q *commitQueue) dequeueApplied() *Batch {
for {
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if tail == head {
// Queue is empty.
return nil
}
slot := &q.slots[tail&uint32(len(q.slots)-1)]
b := slot.Load()
if b == nil || !b.applied.Load() {
// The batch is not ready to be dequeued, or another goroutine has
// already dequeued it.
return nil
}
// Confirm head and tail (for our speculative check above) and increment
// tail. If this succeeds, then we own the slot at tail.
ptrs2 := q.pack(head, tail+1)
if q.headTail.CompareAndSwap(ptrs, ptrs2) {
// We now own slot.
//
// Tell enqueue that we're done with this slot. Zeroing the slot is also
// important so we don't leave behind references that could keep this object
// live longer than necessary.
slot.Store(nil)
// At this point enqueue owns the slot.
return b
}
}
}
// commitEnv contains the environment that a commitPipeline interacts
// with. This allows fine-grained testing of commitPipeline behavior without
// construction of an entire DB.
type commitEnv struct {
// The next sequence number to give to a batch. Protected by
// commitPipeline.mu.
logSeqNum *atomic.Uint64
// The visible sequence number at which reads should be performed. Ratcheted
// upwards atomically as batches are applied to the memtable.
visibleSeqNum *atomic.Uint64
// Apply the batch to the specified memtable. Called concurrently.
apply func(b *Batch, mem *memTable) error
// Write the batch to the WAL. If wg != nil, the data will be persisted
// asynchronously and done will be called on wg upon completion. If wg != nil
// and err != nil, a failure to persist the WAL will populate *err. Returns
// the memtable the batch should be applied to. Serial execution enforced by
// commitPipeline.mu.
write func(b *Batch, wg *sync.WaitGroup, err *error) (*memTable, error)
}
// A commitPipeline manages the stages of committing a set of mutations
// (contained in a single Batch) atomically to the DB. The steps are
// conceptually:
//
// 1. Write the batch to the WAL and optionally sync the WAL
// 2. Apply the mutations in the batch to the memtable
//
// These two simple steps are made complicated by the desire for high
// performance. In the absence of concurrency, performance is limited by how
// fast a batch can be written (and synced) to the WAL and then added to the
// memtable, both of which are outside the purview of the commit
// pipeline. Performance under concurrency is the primary concern of the commit
// pipeline, though it also needs to maintain two invariants:
//
// 1. Batches need to be written to the WAL in sequence number order.
// 2. Batches need to be made visible for reads in sequence number order. This
// invariant arises from the use of a single sequence number which
// indicates which mutations are visible.
//
// Taking these invariants into account, let's revisit the work the commit
// pipeline needs to perform. Writing the batch to the WAL is necessarily
// serialized as there is a single WAL object. The order of the entries in the
// WAL defines the sequence number order. Note that writing to the WAL is
// extremely fast, usually just a memory copy. Applying the mutations in a
// batch to the memtable can occur concurrently as the underlying skiplist
// supports concurrent insertions. Publishing the visible sequence number is
// another serialization point, but one with a twist: the visible sequence
// number cannot be bumped until the mutations for earlier batches have
// finished applying to the memtable (the visible sequence number only ratchets
// up). Lastly, if requested, the commit waits for the WAL to sync. Note that
// waiting for the WAL sync after ratcheting the visible sequence number allows
// another goroutine to read committed data before the WAL has synced. This is
// similar behavior to RocksDB's manual WAL flush functionality. Application
// code needs to protect against this if necessary.
//
// The full outline of the commit pipeline operation is as follows:
//
// with commitPipeline mutex locked:
// assign batch sequence number
// write batch to WAL
// (optionally) add batch to WAL sync list
// apply batch to memtable (concurrently)
// wait for earlier batches to apply
// ratchet read sequence number
// (optionally) wait for the WAL to sync
//
// As soon as a batch has been written to the WAL, the commitPipeline mutex is
// released allowing another batch to write to the WAL. Each commit operation
// individually applies its batch to the memtable providing concurrency. The
// WAL sync happens concurrently with applying to the memtable (see
// commitPipeline.syncLoop).
//
// The "waits for earlier batches to apply" work is more complicated than might
// be expected. The obvious approach would be to keep a queue of pending
// batches and for each batch to wait for the previous batch to finish
// committing. This approach was tried initially and turned out to be too
// slow. The problem is that it causes excessive goroutine activity as each
// committing goroutine needs to wake up in order for the next goroutine to be
// unblocked. The approach taken in the current code is conceptually similar,
// though it avoids waking a goroutine to perform work that another goroutine
// can perform. A commitQueue (a single-producer, multiple-consumer queue)
// holds the ordered list of committing batches. Addition to the queue is done
// while holding commitPipeline.mutex ensuring the same ordering of batches in
// the queue as the ordering in the WAL. When a batch finishes applying to the
// memtable, it atomically updates its Batch.applied field. Ratcheting of the
// visible sequence number is done by commitPipeline.publish which loops
// dequeueing "applied" batches and ratcheting the visible sequence number. If
// we hit an unapplied batch at the head of the queue we can block as we know
// that committing of that unapplied batch will eventually find our (applied)
// batch in the queue. See commitPipeline.publish for additional commentary.
type commitPipeline struct {
// WARNING: The following struct `commitQueue` contains fields which will
// be accessed atomically.
//
// Go allocations are guaranteed to be 64-bit aligned which we take advantage
// of by placing the 64-bit fields which we access atomically at the beginning
// of the commitPipeline struct.
// For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
// Queue of pending batches to commit.
pending commitQueue
env commitEnv
// The commit path has two queues:
// - commitPipeline.pending contains batches whose seqnums have not yet been
// published. It is a lock-free single producer multi consumer queue.
// - LogWriter.flusher.syncQ contains state for batches that have asked for
// a sync. It is a lock-free single producer single consumer queue.
// These lock-free queues have a fixed capacity. And since they are
// lock-free, we cannot do blocking waits when pushing onto these queues, in
// case they are full. Additionally, adding to these queues happens while
// holding commitPipeline.mu, and we don't want to block while holding that
// mutex since it is also needed by other code.
//
// Popping from these queues is independent and for a particular batch can
// occur in either order, though it is more common that popping from the
// commitPipeline.pending will happen first.
//
// Due to these constraints, we reserve a unit of space in each queue before
// acquiring commitPipeline.mu, which also ensures that the push operation
// is guaranteed to have space in the queue. The commitQueueSem and
// logSyncQSem are used for this reservation.
commitQueueSem chan struct{}
logSyncQSem chan struct{}
ingestSem chan struct{}
// The mutex to use for synchronizing access to logSeqNum and serializing
// calls to commitEnv.write().
mu sync.Mutex
}
func newCommitPipeline(env commitEnv) *commitPipeline {
p := &commitPipeline{
env: env,
// The capacity of both commitQueue.slots and syncQueue.slots is set to
// record.SyncConcurrency, which also determines the value of these
// semaphores. We used to have a single semaphore, which required that the
// capacity of these queues be the same. Now that we have two semaphores,
// the capacity of these queues could be changed to be different. Say half
// of the batches asked to be synced, but syncing took 5x the latency of
// adding to the memtable and publishing. Then syncQueue.slots could be
// sized as 0.5*5 of the commitQueue.slots. We can explore this if we find
// that LogWriterMetrics.SyncQueueLen has high utilization under some
// workloads.
//
// NB: the commit concurrency is one less than SyncConcurrency because we
// have to allow one "slot" for a concurrent WAL rotation which will close
// and sync the WAL.
commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
logSyncQSem: make(chan struct{}, record.SyncConcurrency-1),
ingestSem: make(chan struct{}, 1),
}
return p
}
// directWrite is used to directly write to the WAL. commitPipeline.mu must be
// held while this is called. DB.mu must not be held. directWrite will only
// return once the WAL sync is complete. Note that DirectWrite is a special case
// function which is currently only used when ingesting sstables as a flushable.
// Reason carefully about the correctness argument when calling this function
// from any context.
func (p *commitPipeline) directWrite(b *Batch) error {
var syncWG sync.WaitGroup
var syncErr error
syncWG.Add(1)
p.logSyncQSem <- struct{}{}
_, err := p.env.write(b, &syncWG, &syncErr)
syncWG.Wait()
err = firstError(err, syncErr)
return err
}
// Commit the specified batch, writing it to the WAL, optionally syncing the
// WAL, and applying the batch to the memtable. Upon successful return the
// batch's mutations will be visible for reading.
// REQUIRES: noSyncWait => syncWAL
func (p *commitPipeline) Commit(b *Batch, syncWAL bool, noSyncWait bool) error {
if b.Empty() {
return nil
}
commitStartTime := time.Now()
// Acquire semaphores.
p.commitQueueSem <- struct{}{}
if syncWAL {
p.logSyncQSem <- struct{}{}
}
b.commitStats.SemaphoreWaitDuration = time.Since(commitStartTime)
// Prepare the batch for committing: enqueuing the batch in the pending
// queue, determining the batch sequence number and writing the data to the
// WAL.
//
// NB: We set Batch.commitErr on error so that the batch won't be a candidate
// for reuse. See Batch.release().
mem, err := p.prepare(b, syncWAL, noSyncWait)
if err != nil {
b.db = nil // prevent batch reuse on error
// NB: we are not doing <-p.commitQueueSem since the batch is still
// sitting in the pending queue. We should consider fixing this by also
// removing the batch from the pending queue.
return err
}
// Apply the batch to the memtable.
if err := p.env.apply(b, mem); err != nil {
b.db = nil // prevent batch reuse on error
// NB: we are not doing <-p.commitQueueSem since the batch is still
// sitting in the pending queue. We should consider fixing this by also
// removing the batch from the pending queue.
return err
}
// Publish the batch sequence number.
p.publish(b)
<-p.commitQueueSem
if !noSyncWait {
// Already waited for commit, so look at the error.
if b.commitErr != nil {
b.db = nil // prevent batch reuse on error
err = b.commitErr
}
}
// Else noSyncWait. The LogWriter can be concurrently writing to
// b.commitErr. We will read b.commitErr in Batch.SyncWait after the
// LogWriter is done writing.
b.commitStats.TotalDuration = time.Since(commitStartTime)
return err
}
// AllocateSeqNum allocates count sequence numbers, invokes the prepare
// callback, then the apply callback, and then publishes the sequence
// numbers. AllocateSeqNum does not write to the WAL or add entries to the
// memtable. AllocateSeqNum can be used to sequence an operation such as
// sstable ingestion within the commit pipeline. The prepare callback is
// invoked with commitPipeline.mu held, but note that DB.mu is not held and
// must be locked if necessary.
func (p *commitPipeline) AllocateSeqNum(
count int, prepare func(seqNum uint64), apply func(seqNum uint64),
) {
// This method is similar to Commit and prepare. Be careful about trying to
// share additional code with those methods because Commit and prepare are
// performance critical code paths.
b := newBatch(nil)
defer b.release()
// Give the batch a count of 1 so that the log and visible sequence number
// are incremented correctly.
b.data = make([]byte, batchHeaderLen)
b.setCount(uint32(count))
b.commit.Add(1)
p.commitQueueSem <- struct{}{}
p.mu.Lock()
// Enqueue the batch in the pending queue. Note that while the pending queue
// is lock-free, we want the order of batches to be the same as the sequence
// number order.
p.pending.enqueue(b)
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
logSeqNum := p.env.logSeqNum.Add(uint64(count)) - uint64(count)
seqNum := logSeqNum
if seqNum == 0 {
// We can't use the value 0 for the global seqnum during ingestion, because
// 0 indicates no global seqnum. So allocate one more seqnum.
p.env.logSeqNum.Add(1)
seqNum++
}
b.setSeqNum(seqNum)
// Wait for any outstanding writes to the memtable to complete. This is
// necessary for ingestion so that the check for memtable overlap can see any
// writes that were sequenced before the ingestion. The spin loop is
// unfortunate, but obviates the need for additional synchronization.
for {
visibleSeqNum := p.env.visibleSeqNum.Load()
if visibleSeqNum == logSeqNum {
break
}
runtime.Gosched()
}
// Invoke the prepare callback. Note the lack of error reporting. Even if the
// callback internally fails, the sequence number needs to be published in
// order to allow the commit pipeline to proceed.
prepare(b.SeqNum())
p.mu.Unlock()
// Invoke the apply callback.
apply(b.SeqNum())
// Publish the sequence number.
p.publish(b)
<-p.commitQueueSem
}
func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memTable, error) {
n := uint64(b.Count())
if n == invalidBatchCount {
return nil, ErrInvalidBatch
}
var syncWG *sync.WaitGroup
var syncErr *error
switch {
case !syncWAL:
// Only need to wait for the publish.
b.commit.Add(1)
// Remaining cases represent syncWAL=true.
case noSyncWait:
syncErr = &b.commitErr
syncWG = &b.fsyncWait
// Only need to wait synchronously for the publish. The user will
// (asynchronously) wait on the batch's fsyncWait.
b.commit.Add(1)
b.fsyncWait.Add(1)
case !noSyncWait:
syncErr = &b.commitErr
syncWG = &b.commit
// Must wait for both the publish and the WAL fsync.
b.commit.Add(2)
}
p.mu.Lock()
// Enqueue the batch in the pending queue. Note that while the pending queue
// is lock-free, we want the order of batches to be the same as the sequence
// number order.
p.pending.enqueue(b)
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
b.setSeqNum(p.env.logSeqNum.Add(n) - n)
// Write the data to the WAL.
mem, err := p.env.write(b, syncWG, syncErr)
p.mu.Unlock()
return mem, err
}
func (p *commitPipeline) publish(b *Batch) {
// Mark the batch as applied.
b.applied.Store(true)
// Loop dequeuing applied batches from the pending queue. If our batch was
// the head of the pending queue we are guaranteed that either we'll publish
// it or someone else will dequeueApplied and publish it. If our batch is not the
// head of the queue then either we'll dequeueApplied applied batches and reach our
// batch or there is an unapplied batch blocking us. When that unapplied
// batch applies it will go through the same process and publish our batch
// for us.
for {
t := p.pending.dequeueApplied()
if t == nil {
// Wait for another goroutine to publish us. We might also be waiting for
// the WAL sync to finish.
now := time.Now()
b.commit.Wait()
b.commitStats.CommitWaitDuration += time.Since(now)
break
}
if !t.applied.Load() {
panic("not reached")
}
// We're responsible for publishing the sequence number for batch t, but
// another concurrent goroutine might sneak in and publish the sequence
// number for a subsequent batch. That's ok as all we're guaranteeing is
// that the sequence number ratchets up.
for {
curSeqNum := p.env.visibleSeqNum.Load()
newSeqNum := t.SeqNum() + uint64(t.Count())
if newSeqNum <= curSeqNum {
// t's sequence number has already been published.
break
}
if p.env.visibleSeqNum.CompareAndSwap(curSeqNum, newSeqNum) {
// We successfully published t's sequence number.
break
}
}
t.commit.Done()
}
}

3738
vendor/github.com/cockroachdb/pebble/compaction.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1338
vendor/github.com/cockroachdb/pebble/compaction_iter.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1900
vendor/github.com/cockroachdb/pebble/compaction_picker.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

31
vendor/github.com/cockroachdb/pebble/comparer.go generated vendored Normal file
View File

@@ -0,0 +1,31 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import "github.com/cockroachdb/pebble/internal/base"
// Compare exports the base.Compare type.
type Compare = base.Compare
// Equal exports the base.Equal type.
type Equal = base.Equal
// AbbreviatedKey exports the base.AbbreviatedKey type.
type AbbreviatedKey = base.AbbreviatedKey
// Separator exports the base.Separator type.
type Separator = base.Separator
// Successor exports the base.Successor type.
type Successor = base.Successor
// Split exports the base.Split type.
type Split = base.Split
// Comparer exports the base.Comparer type.
type Comparer = base.Comparer
// DefaultComparer exports the base.DefaultComparer variable.
var DefaultComparer = base.DefaultComparer

2944
vendor/github.com/cockroachdb/pebble/db.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

90
vendor/github.com/cockroachdb/pebble/error_iter.go generated vendored Normal file
View File

@@ -0,0 +1,90 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
type errorIter struct {
err error
}
// errorIter implements the base.InternalIterator interface.
var _ internalIterator = (*errorIter)(nil)
func newErrorIter(err error) *errorIter {
return &errorIter{err: err}
}
func (c *errorIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) First() (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) Last() (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) Next() (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) Prev() (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) {
return nil, base.LazyValue{}
}
func (c *errorIter) Error() error {
return c.err
}
func (c *errorIter) Close() error {
return c.err
}
func (c *errorIter) String() string {
return "error"
}
func (c *errorIter) SetBounds(lower, upper []byte) {}
type errorKeyspanIter struct {
err error
}
// errorKeyspanIter implements the keyspan.FragmentIterator interface.
var _ keyspan.FragmentIterator = (*errorKeyspanIter)(nil)
func newErrorKeyspanIter(err error) *errorKeyspanIter {
return &errorKeyspanIter{err: err}
}
func (*errorKeyspanIter) SeekGE(key []byte) *keyspan.Span { return nil }
func (*errorKeyspanIter) SeekLT(key []byte) *keyspan.Span { return nil }
func (*errorKeyspanIter) First() *keyspan.Span { return nil }
func (*errorKeyspanIter) Last() *keyspan.Span { return nil }
func (*errorKeyspanIter) Next() *keyspan.Span { return nil }
func (*errorKeyspanIter) Prev() *keyspan.Span { return nil }
func (i *errorKeyspanIter) Error() error { return i.err }
func (i *errorKeyspanIter) Close() error { return i.err }
func (*errorKeyspanIter) String() string { return "error" }

736
vendor/github.com/cockroachdb/pebble/event.go generated vendored Normal file
View File

@@ -0,0 +1,736 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"strings"
"time"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/humanize"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/redact"
)
// TableInfo exports the manifest.TableInfo type.
type TableInfo = manifest.TableInfo
func tablesTotalSize(tables []TableInfo) uint64 {
var size uint64
for i := range tables {
size += tables[i].Size
}
return size
}
func formatFileNums(tables []TableInfo) string {
var buf strings.Builder
for i := range tables {
if i > 0 {
buf.WriteString(" ")
}
buf.WriteString(tables[i].FileNum.String())
}
return buf.String()
}
// LevelInfo contains info pertaining to a partificular level.
type LevelInfo struct {
Level int
Tables []TableInfo
}
func (i LevelInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))))
}
// CompactionInfo contains the info for a compaction event.
type CompactionInfo struct {
// JobID is the ID of the compaction job.
JobID int
// Reason is the reason for the compaction.
Reason string
// Input contains the input tables for the compaction organized by level.
Input []LevelInfo
// Output contains the output tables generated by the compaction. The output
// tables are empty for the compaction begin event.
Output LevelInfo
// Duration is the time spent compacting, including reading and writing
// sstables.
Duration time.Duration
// TotalDuration is the total wall-time duration of the compaction,
// including applying the compaction to the database. TotalDuration is
// always ≥ Duration.
TotalDuration time.Duration
Done bool
Err error
}
func (i CompactionInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] compaction(%s) to L%d error: %s",
redact.Safe(i.JobID), redact.SafeString(i.Reason), redact.Safe(i.Output.Level), i.Err)
return
}
if !i.Done {
w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Print(levelInfos(i.Input))
return
}
outputSize := tablesTotalSize(i.Output.Tables)
w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Print(levelInfos(i.Input))
w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Output.Level),
redact.Safe(formatFileNums(i.Output.Tables)),
redact.Safe(humanize.Bytes.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
type levelInfos []LevelInfo
func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) {
for j, levelInfo := range i {
if j > 0 {
w.Printf(" + ")
}
w.Print(levelInfo)
}
}
// DiskSlowInfo contains the info for a disk slowness event when writing to a
// file.
type DiskSlowInfo = vfs.DiskSlowInfo
// FlushInfo contains the info for a flush event.
type FlushInfo struct {
// JobID is the ID of the flush job.
JobID int
// Reason is the reason for the flush.
Reason string
// Input contains the count of input memtables that were flushed.
Input int
// InputBytes contains the total in-memory size of the memtable(s) that were
// flushed. This size includes skiplist indexing data structures.
InputBytes uint64
// Output contains the ouptut table generated by the flush. The output info
// is empty for the flush begin event.
Output []TableInfo
// Duration is the time spent flushing. This duration includes writing and
// syncing all of the flushed keys to sstables.
Duration time.Duration
// TotalDuration is the total wall-time duration of the flush, including
// applying the flush to the database. TotalDuration is always ≥ Duration.
TotalDuration time.Duration
// Ingest is set to true if the flush is handling tables that were added to
// the flushable queue via an ingestion operation.
Ingest bool
// IngestLevels are the output levels for each ingested table in the flush.
// This field is only populated when Ingest is true.
IngestLevels []int
Done bool
Err error
}
func (i FlushInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] flush error: %s", redact.Safe(i.JobID), i.Err)
return
}
plural := redact.SafeString("s")
if i.Input == 1 {
plural = ""
}
if !i.Done {
w.Printf("[JOB %d] ", redact.Safe(i.JobID))
if !i.Ingest {
w.Printf("flushing %d memtable", redact.Safe(i.Input))
w.SafeString(plural)
w.Printf(" (%s) to L0", redact.Safe(humanize.Bytes.Uint64(i.InputBytes)))
} else {
w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural)
}
return
}
outputSize := tablesTotalSize(i.Output)
if !i.Ingest {
if invariants.Enabled && len(i.IngestLevels) > 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0"))
}
w.Printf("[JOB %d] flushed %d memtable%s (%s) to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.JobID), redact.Safe(i.Input), plural,
redact.Safe(humanize.Bytes.Uint64(i.InputBytes)),
redact.Safe(formatFileNums(i.Output)),
redact.Safe(humanize.Bytes.Uint64(outputSize)),
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
} else {
if invariants.Enabled && len(i.IngestLevels) == 0 {
panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0"))
}
w.Printf("[JOB %d] flushed %d ingested flushable%s",
redact.Safe(i.JobID), redact.Safe(len(i.Output)), plural)
for j, level := range i.IngestLevels {
file := i.Output[j]
if j > 0 {
w.Printf(" +")
}
w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.Bytes.Uint64(file.Size))
}
w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Duration.Seconds()),
redact.Safe(i.TotalDuration.Seconds()),
redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
}
}
// ManifestCreateInfo contains info about a manifest creation event.
type ManifestCreateInfo struct {
// JobID is the ID of the job the caused the manifest to be created.
JobID int
Path string
// The file number of the new Manifest.
FileNum FileNum
Err error
}
func (i ManifestCreateInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// ManifestDeleteInfo contains the info for a Manifest deletion event.
type ManifestDeleteInfo struct {
// JobID is the ID of the job the caused the Manifest to be deleted.
JobID int
Path string
FileNum FileNum
Err error
}
func (i ManifestDeleteInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// TableCreateInfo contains the info for a table creation event.
type TableCreateInfo struct {
JobID int
// Reason is the reason for the table creation: "compacting", "flushing", or
// "ingesting".
Reason string
Path string
FileNum FileNum
}
func (i TableCreateInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] %s: sstable created %s",
redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
}
// TableDeleteInfo contains the info for a table deletion event.
type TableDeleteInfo struct {
JobID int
Path string
FileNum FileNum
Err error
}
func (i TableDeleteInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] sstable delete error %s: %s",
redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
return
}
w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// TableIngestInfo contains the info for a table ingestion event.
type TableIngestInfo struct {
// JobID is the ID of the job the caused the table to be ingested.
JobID int
Tables []struct {
TableInfo
Level int
}
// GlobalSeqNum is the sequence number that was assigned to all entries in
// the ingested table.
GlobalSeqNum uint64
// flushable indicates whether the ingested sstable was treated as a
// flushable.
flushable bool
Err error
}
func (i TableIngestInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] ingest error: %s", redact.Safe(i.JobID), i.Err)
return
}
if i.flushable {
w.Printf("[JOB %d] ingested as flushable", redact.Safe(i.JobID))
} else {
w.Printf("[JOB %d] ingested", redact.Safe(i.JobID))
}
for j := range i.Tables {
t := &i.Tables[j]
if j > 0 {
w.Printf(",")
}
levelStr := ""
if !i.flushable {
levelStr = fmt.Sprintf("L%d:", t.Level)
}
w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum),
redact.Safe(humanize.Bytes.Uint64(t.Size)))
}
}
// TableStatsInfo contains the info for a table stats loaded event.
type TableStatsInfo struct {
// JobID is the ID of the job that finished loading the initial tables'
// stats.
JobID int
}
func (i TableStatsInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableStatsInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] all initial table stats loaded", redact.Safe(i.JobID))
}
// TableValidatedInfo contains information on the result of a validation run
// on an sstable.
type TableValidatedInfo struct {
JobID int
Meta *fileMetadata
}
func (i TableValidatedInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i TableValidatedInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] validated table: %s", redact.Safe(i.JobID), i.Meta)
}
// WALCreateInfo contains info about a WAL creation event.
type WALCreateInfo struct {
// JobID is the ID of the job the caused the WAL to be created.
JobID int
Path string
// The file number of the new WAL.
FileNum FileNum
// The file number of a previous WAL which was recycled to create this
// one. Zero if recycling did not take place.
RecycledFileNum FileNum
Err error
}
func (i WALCreateInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] WAL create error: %s", redact.Safe(i.JobID), i.Err)
return
}
if i.RecycledFileNum == 0 {
w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
return
}
w.Printf("[JOB %d] WAL created %s (recycled %s)",
redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
}
// WALDeleteInfo contains the info for a WAL deletion event.
type WALDeleteInfo struct {
// JobID is the ID of the job the caused the WAL to be deleted.
JobID int
Path string
FileNum FileNum
Err error
}
func (i WALDeleteInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
if i.Err != nil {
w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
return
}
w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
}
// WriteStallBeginInfo contains the info for a write stall begin event.
type WriteStallBeginInfo struct {
Reason string
}
func (i WriteStallBeginInfo) String() string {
return redact.StringWithoutMarkers(i)
}
// SafeFormat implements redact.SafeFormatter.
func (i WriteStallBeginInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("write stall beginning: %s", redact.Safe(i.Reason))
}
// EventListener contains a set of functions that will be invoked when various
// significant DB events occur. Note that the functions should not run for an
// excessive amount of time as they are invoked synchronously by the DB and may
// block continued DB work. For a similar reason it is advisable to not perform
// any synchronous calls back into the DB.
type EventListener struct {
// BackgroundError is invoked whenever an error occurs during a background
// operation such as flush or compaction.
BackgroundError func(error)
// CompactionBegin is invoked after the inputs to a compaction have been
// determined, but before the compaction has produced any output.
CompactionBegin func(CompactionInfo)
// CompactionEnd is invoked after a compaction has completed and the result
// has been installed.
CompactionEnd func(CompactionInfo)
// DiskSlow is invoked after a disk write operation on a file created with a
// disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks) is
// observed to exceed the specified disk slowness threshold duration. DiskSlow
// is called on a goroutine that is monitoring slowness/stuckness. The callee
// MUST return without doing any IO, or blocking on anything (like a mutex)
// that is waiting on IO. This is imperative in order to reliably monitor for
// slowness, since if this goroutine gets stuck, the monitoring will stop
// working.
DiskSlow func(DiskSlowInfo)
// FlushBegin is invoked after the inputs to a flush have been determined,
// but before the flush has produced any output.
FlushBegin func(FlushInfo)
// FlushEnd is invoked after a flush has complated and the result has been
// installed.
FlushEnd func(FlushInfo)
// FormatUpgrade is invoked after the database's FormatMajorVersion
// is upgraded.
FormatUpgrade func(FormatMajorVersion)
// ManifestCreated is invoked after a manifest has been created.
ManifestCreated func(ManifestCreateInfo)
// ManifestDeleted is invoked after a manifest has been deleted.
ManifestDeleted func(ManifestDeleteInfo)
// TableCreated is invoked when a table has been created.
TableCreated func(TableCreateInfo)
// TableDeleted is invoked after a table has been deleted.
TableDeleted func(TableDeleteInfo)
// TableIngested is invoked after an externally created table has been
// ingested via a call to DB.Ingest().
TableIngested func(TableIngestInfo)
// TableStatsLoaded is invoked at most once, when the table stats
// collector has loaded statistics for all tables that existed at Open.
TableStatsLoaded func(TableStatsInfo)
// TableValidated is invoked after validation runs on an sstable.
TableValidated func(TableValidatedInfo)
// WALCreated is invoked after a WAL has been created.
WALCreated func(WALCreateInfo)
// WALDeleted is invoked after a WAL has been deleted.
WALDeleted func(WALDeleteInfo)
// WriteStallBegin is invoked when writes are intentionally delayed.
WriteStallBegin func(WriteStallBeginInfo)
// WriteStallEnd is invoked when delayed writes are released.
WriteStallEnd func()
}
// EnsureDefaults ensures that background error events are logged to the
// specified logger if a handler for those events hasn't been otherwise
// specified. Ensure all handlers are non-nil so that we don't have to check
// for nil-ness before invoking.
func (l *EventListener) EnsureDefaults(logger Logger) {
if l.BackgroundError == nil {
if logger != nil {
l.BackgroundError = func(err error) {
logger.Infof("background error: %s", err)
}
} else {
l.BackgroundError = func(error) {}
}
}
if l.CompactionBegin == nil {
l.CompactionBegin = func(info CompactionInfo) {}
}
if l.CompactionEnd == nil {
l.CompactionEnd = func(info CompactionInfo) {}
}
if l.DiskSlow == nil {
l.DiskSlow = func(info DiskSlowInfo) {}
}
if l.FlushBegin == nil {
l.FlushBegin = func(info FlushInfo) {}
}
if l.FlushEnd == nil {
l.FlushEnd = func(info FlushInfo) {}
}
if l.FormatUpgrade == nil {
l.FormatUpgrade = func(v FormatMajorVersion) {}
}
if l.ManifestCreated == nil {
l.ManifestCreated = func(info ManifestCreateInfo) {}
}
if l.ManifestDeleted == nil {
l.ManifestDeleted = func(info ManifestDeleteInfo) {}
}
if l.TableCreated == nil {
l.TableCreated = func(info TableCreateInfo) {}
}
if l.TableDeleted == nil {
l.TableDeleted = func(info TableDeleteInfo) {}
}
if l.TableIngested == nil {
l.TableIngested = func(info TableIngestInfo) {}
}
if l.TableStatsLoaded == nil {
l.TableStatsLoaded = func(info TableStatsInfo) {}
}
if l.TableValidated == nil {
l.TableValidated = func(validated TableValidatedInfo) {}
}
if l.WALCreated == nil {
l.WALCreated = func(info WALCreateInfo) {}
}
if l.WALDeleted == nil {
l.WALDeleted = func(info WALDeleteInfo) {}
}
if l.WriteStallBegin == nil {
l.WriteStallBegin = func(info WriteStallBeginInfo) {}
}
if l.WriteStallEnd == nil {
l.WriteStallEnd = func() {}
}
}
// MakeLoggingEventListener creates an EventListener that logs all events to the
// specified logger.
func MakeLoggingEventListener(logger Logger) EventListener {
if logger == nil {
logger = DefaultLogger
}
return EventListener{
BackgroundError: func(err error) {
logger.Infof("background error: %s", err)
},
CompactionBegin: func(info CompactionInfo) {
logger.Infof("%s", info)
},
CompactionEnd: func(info CompactionInfo) {
logger.Infof("%s", info)
},
DiskSlow: func(info DiskSlowInfo) {
logger.Infof("%s", info)
},
FlushBegin: func(info FlushInfo) {
logger.Infof("%s", info)
},
FlushEnd: func(info FlushInfo) {
logger.Infof("%s", info)
},
FormatUpgrade: func(v FormatMajorVersion) {
logger.Infof("upgraded to format version: %s", v)
},
ManifestCreated: func(info ManifestCreateInfo) {
logger.Infof("%s", info)
},
ManifestDeleted: func(info ManifestDeleteInfo) {
logger.Infof("%s", info)
},
TableCreated: func(info TableCreateInfo) {
logger.Infof("%s", info)
},
TableDeleted: func(info TableDeleteInfo) {
logger.Infof("%s", info)
},
TableIngested: func(info TableIngestInfo) {
logger.Infof("%s", info)
},
TableStatsLoaded: func(info TableStatsInfo) {
logger.Infof("%s", info)
},
TableValidated: func(info TableValidatedInfo) {
logger.Infof("%s", info)
},
WALCreated: func(info WALCreateInfo) {
logger.Infof("%s", info)
},
WALDeleted: func(info WALDeleteInfo) {
logger.Infof("%s", info)
},
WriteStallBegin: func(info WriteStallBeginInfo) {
logger.Infof("%s", info)
},
WriteStallEnd: func() {
logger.Infof("write stall ending")
},
}
}
// TeeEventListener wraps two EventListeners, forwarding all events to both.
func TeeEventListener(a, b EventListener) EventListener {
a.EnsureDefaults(nil)
b.EnsureDefaults(nil)
return EventListener{
BackgroundError: func(err error) {
a.BackgroundError(err)
b.BackgroundError(err)
},
CompactionBegin: func(info CompactionInfo) {
a.CompactionBegin(info)
b.CompactionBegin(info)
},
CompactionEnd: func(info CompactionInfo) {
a.CompactionEnd(info)
b.CompactionEnd(info)
},
DiskSlow: func(info DiskSlowInfo) {
a.DiskSlow(info)
b.DiskSlow(info)
},
FlushBegin: func(info FlushInfo) {
a.FlushBegin(info)
b.FlushBegin(info)
},
FlushEnd: func(info FlushInfo) {
a.FlushEnd(info)
b.FlushEnd(info)
},
FormatUpgrade: func(v FormatMajorVersion) {
a.FormatUpgrade(v)
b.FormatUpgrade(v)
},
ManifestCreated: func(info ManifestCreateInfo) {
a.ManifestCreated(info)
b.ManifestCreated(info)
},
ManifestDeleted: func(info ManifestDeleteInfo) {
a.ManifestDeleted(info)
b.ManifestDeleted(info)
},
TableCreated: func(info TableCreateInfo) {
a.TableCreated(info)
b.TableCreated(info)
},
TableDeleted: func(info TableDeleteInfo) {
a.TableDeleted(info)
b.TableDeleted(info)
},
TableIngested: func(info TableIngestInfo) {
a.TableIngested(info)
b.TableIngested(info)
},
TableStatsLoaded: func(info TableStatsInfo) {
a.TableStatsLoaded(info)
b.TableStatsLoaded(info)
},
TableValidated: func(info TableValidatedInfo) {
a.TableValidated(info)
b.TableValidated(info)
},
WALCreated: func(info WALCreateInfo) {
a.WALCreated(info)
b.WALCreated(info)
},
WALDeleted: func(info WALDeleteInfo) {
a.WALDeleted(info)
b.WALDeleted(info)
},
WriteStallBegin: func(info WriteStallBeginInfo) {
a.WriteStallBegin(info)
b.WriteStallBegin(info)
},
WriteStallEnd: func() {
a.WriteStallEnd()
b.WriteStallEnd()
},
}
}

555
vendor/github.com/cockroachdb/pebble/external_iterator.go generated vendored Normal file
View File

@@ -0,0 +1,555 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"context"
"fmt"
"sort"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/sstable"
)
// ExternalIterOption provide an interface to specify open-time options to
// NewExternalIter.
type ExternalIterOption interface {
// iterApply is called on the iterator during opening in order to set internal
// parameters.
iterApply(*Iterator)
// readerOptions returns any reader options added by this iter option.
readerOptions() []sstable.ReaderOption
}
type externalIterReaderOptions struct {
opts []sstable.ReaderOption
}
func (e *externalIterReaderOptions) iterApply(iterator *Iterator) {
// Do nothing.
}
func (e *externalIterReaderOptions) readerOptions() []sstable.ReaderOption {
return e.opts
}
// ExternalIterReaderOptions returns an ExternalIterOption that specifies
// sstable.ReaderOptions to be applied on sstable readers in NewExternalIter.
func ExternalIterReaderOptions(opts ...sstable.ReaderOption) ExternalIterOption {
return &externalIterReaderOptions{opts: opts}
}
// ExternalIterForwardOnly is an ExternalIterOption that specifies this iterator
// will only be used for forward positioning operations (First, SeekGE, Next).
// This could enable optimizations that take advantage of this invariant.
// Behaviour when a reverse positioning operation is done on an iterator
// opened with this option is unpredictable, though in most cases it should.
type ExternalIterForwardOnly struct{}
func (e ExternalIterForwardOnly) iterApply(iter *Iterator) {
iter.forwardOnly = true
}
func (e ExternalIterForwardOnly) readerOptions() []sstable.ReaderOption {
return nil
}
// NewExternalIter takes an input 2d array of sstable files which may overlap
// across subarrays but not within a subarray (at least as far as points are
// concerned; range keys are allowed to overlap arbitrarily even within a
// subarray), and returns an Iterator over the merged contents of the sstables.
// Input sstables may contain point keys, range keys, range deletions, etc. The
// input files slice must be sorted in reverse chronological ordering. A key in a
// file at a lower index subarray will shadow a key with an identical user key
// contained within a file at a higher index subarray. Each subarray must be
// sorted in internal key order, where lower index files contain keys that sort
// left of files with higher indexes.
//
// Input sstables must only contain keys with the zero sequence number.
//
// Iterators constructed through NewExternalIter do not support all iterator
// options, including block-property and table filters. NewExternalIter errors
// if an incompatible option is set.
func NewExternalIter(
o *Options,
iterOpts *IterOptions,
files [][]sstable.ReadableFile,
extraOpts ...ExternalIterOption,
) (it *Iterator, err error) {
return NewExternalIterWithContext(context.Background(), o, iterOpts, files, extraOpts...)
}
// NewExternalIterWithContext is like NewExternalIter, and additionally
// accepts a context for tracing.
func NewExternalIterWithContext(
ctx context.Context,
o *Options,
iterOpts *IterOptions,
files [][]sstable.ReadableFile,
extraOpts ...ExternalIterOption,
) (it *Iterator, err error) {
if iterOpts != nil {
if err := validateExternalIterOpts(iterOpts); err != nil {
return nil, err
}
}
var readers [][]*sstable.Reader
// Ensure we close all the opened readers if we error out.
defer func() {
if err != nil {
for i := range readers {
for j := range readers[i] {
_ = readers[i][j].Close()
}
}
}
}()
seqNumOffset := 0
var extraReaderOpts []sstable.ReaderOption
for i := range extraOpts {
extraReaderOpts = append(extraReaderOpts, extraOpts[i].readerOptions()...)
}
for _, levelFiles := range files {
seqNumOffset += len(levelFiles)
}
for _, levelFiles := range files {
var subReaders []*sstable.Reader
seqNumOffset -= len(levelFiles)
subReaders, err = openExternalTables(o, levelFiles, seqNumOffset, o.MakeReaderOptions(), extraReaderOpts...)
readers = append(readers, subReaders)
}
if err != nil {
return nil, err
}
buf := iterAllocPool.Get().(*iterAlloc)
dbi := &buf.dbi
*dbi = Iterator{
ctx: ctx,
alloc: buf,
merge: o.Merger.Merge,
comparer: *o.Comparer,
readState: nil,
keyBuf: buf.keyBuf,
prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
boundsBuf: buf.boundsBuf,
batch: nil,
// Add the readers to the Iterator so that Close closes them, and
// SetOptions can re-construct iterators from them.
externalReaders: readers,
newIters: func(
ctx context.Context, f *manifest.FileMetadata, opts *IterOptions,
internalOpts internalIterOpts) (internalIterator, keyspan.FragmentIterator, error) {
// NB: External iterators are currently constructed without any
// `levelIters`. newIters should never be called. When we support
// organizing multiple non-overlapping files into a single level
// (see TODO below), we'll need to adjust this tableNewIters
// implementation to open iterators by looking up f in a map
// of readers indexed by *fileMetadata.
panic("unreachable")
},
seqNum: base.InternalKeySeqNumMax,
}
if iterOpts != nil {
dbi.opts = *iterOpts
dbi.processBounds(iterOpts.LowerBound, iterOpts.UpperBound)
}
for i := range extraOpts {
extraOpts[i].iterApply(dbi)
}
finishInitializingExternal(ctx, dbi)
return dbi, nil
}
func validateExternalIterOpts(iterOpts *IterOptions) error {
switch {
case iterOpts.TableFilter != nil:
return errors.Errorf("pebble: external iterator: TableFilter unsupported")
case iterOpts.PointKeyFilters != nil:
return errors.Errorf("pebble: external iterator: PointKeyFilters unsupported")
case iterOpts.RangeKeyFilters != nil:
return errors.Errorf("pebble: external iterator: RangeKeyFilters unsupported")
case iterOpts.OnlyReadGuaranteedDurable:
return errors.Errorf("pebble: external iterator: OnlyReadGuaranteedDurable unsupported")
case iterOpts.UseL6Filters:
return errors.Errorf("pebble: external iterator: UseL6Filters unsupported")
}
return nil
}
func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterator, error) {
// TODO(jackson): In some instances we could generate fewer levels by using
// L0Sublevels code to organize nonoverlapping files into the same level.
// This would allow us to use levelIters and keep a smaller set of data and
// files in-memory. However, it would also require us to identify the bounds
// of all the files upfront.
if !it.opts.pointKeys() {
return emptyIter, nil
} else if it.pointIter != nil {
return it.pointIter, nil
}
mlevels := it.alloc.mlevels[:0]
if len(it.externalReaders) > cap(mlevels) {
mlevels = make([]mergingIterLevel, 0, len(it.externalReaders))
}
for _, readers := range it.externalReaders {
var combinedIters []internalIterator
for _, r := range readers {
var (
rangeDelIter keyspan.FragmentIterator
pointIter internalIterator
err error
)
// We could set hideObsoletePoints=true, since we are reading at
// InternalKeySeqNumMax, but we don't bother since these sstables should
// not have obsolete points (so the performance optimization is
// unnecessary), and we don't want to bother constructing a
// BlockPropertiesFilterer that includes obsoleteKeyBlockPropertyFilter.
pointIter, err = r.NewIterWithBlockPropertyFiltersAndContextEtc(
ctx, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */
false /* hideObsoletePoints */, false, /* useFilterBlock */
&it.stats.InternalStats, sstable.TrivialReaderProvider{Reader: r})
if err != nil {
return nil, err
}
rangeDelIter, err = r.NewRawRangeDelIter()
if err != nil {
return nil, err
}
if rangeDelIter == nil && pointIter != nil && it.forwardOnly {
// TODO(bilal): Consider implementing range key pausing in
// simpleLevelIter so we can reduce mergingIterLevels even more by
// sending all sstable iterators to combinedIters, not just those
// corresponding to sstables without range deletes.
combinedIters = append(combinedIters, pointIter)
continue
}
mlevels = append(mlevels, mergingIterLevel{
iter: pointIter,
rangeDelIter: rangeDelIter,
})
}
if len(combinedIters) == 1 {
mlevels = append(mlevels, mergingIterLevel{
iter: combinedIters[0],
})
} else if len(combinedIters) > 1 {
sli := &simpleLevelIter{
cmp: it.cmp,
iters: combinedIters,
}
sli.init(it.opts)
mlevels = append(mlevels, mergingIterLevel{
iter: sli,
rangeDelIter: nil,
})
}
}
if len(mlevels) == 1 && mlevels[0].rangeDelIter == nil {
// Set closePointIterOnce to true. This is because we're bypassing the
// merging iter, which turns Close()s on it idempotent for any child
// iterators. The outer Iterator could call Close() on a point iter twice,
// which sstable iterators do not support (as they release themselves to
// a pool).
it.closePointIterOnce = true
return mlevels[0].iter, nil
}
it.alloc.merging.init(&it.opts, &it.stats.InternalStats, it.comparer.Compare, it.comparer.Split, mlevels...)
it.alloc.merging.snapshot = base.InternalKeySeqNumMax
if len(mlevels) <= cap(it.alloc.levelsPositioned) {
it.alloc.merging.levelsPositioned = it.alloc.levelsPositioned[:len(mlevels)]
}
return &it.alloc.merging, nil
}
func finishInitializingExternal(ctx context.Context, it *Iterator) {
pointIter, err := createExternalPointIter(ctx, it)
if err != nil {
it.pointIter = &errorIter{err: err}
} else {
it.pointIter = pointIter
}
it.iter = it.pointIter
if it.opts.rangeKeys() {
it.rangeKeyMasking.init(it, it.comparer.Compare, it.comparer.Split)
var rangeKeyIters []keyspan.FragmentIterator
if it.rangeKey == nil {
// We could take advantage of the lack of overlaps in range keys within
// each slice in it.externalReaders, and generate keyspan.LevelIters
// out of those. However, since range keys are expected to be sparse to
// begin with, the performance gain might not be significant enough to
// warrant it.
//
// TODO(bilal): Explore adding a simpleRangeKeyLevelIter that does not
// operate on FileMetadatas (similar to simpleLevelIter), and implements
// this optimization.
for _, readers := range it.externalReaders {
for _, r := range readers {
if rki, err := r.NewRawRangeKeyIter(); err != nil {
rangeKeyIters = append(rangeKeyIters, &errorKeyspanIter{err: err})
} else if rki != nil {
rangeKeyIters = append(rangeKeyIters, rki)
}
}
}
if len(rangeKeyIters) > 0 {
it.rangeKey = iterRangeKeyStateAllocPool.Get().(*iteratorRangeKeyState)
it.rangeKey.init(it.comparer.Compare, it.comparer.Split, &it.opts)
it.rangeKey.rangeKeyIter = it.rangeKey.iterConfig.Init(
&it.comparer,
base.InternalKeySeqNumMax,
it.opts.LowerBound, it.opts.UpperBound,
&it.hasPrefix, &it.prefixOrFullSeekKey,
true /* onlySets */, &it.rangeKey.internal,
)
for i := range rangeKeyIters {
it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])
}
}
}
if it.rangeKey != nil {
it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter,
keyspan.InterleavingIterOpts{
Mask: &it.rangeKeyMasking,
LowerBound: it.opts.LowerBound,
UpperBound: it.opts.UpperBound,
})
it.iter = &it.rangeKey.iiter
}
}
}
func openExternalTables(
o *Options,
files []sstable.ReadableFile,
seqNumOffset int,
readerOpts sstable.ReaderOptions,
extraReaderOpts ...sstable.ReaderOption,
) (readers []*sstable.Reader, err error) {
readers = make([]*sstable.Reader, 0, len(files))
for i := range files {
readable, err := sstable.NewSimpleReadable(files[i])
if err != nil {
return readers, err
}
r, err := sstable.NewReader(readable, readerOpts, extraReaderOpts...)
if err != nil {
return readers, err
}
// Use the index of the file in files as the sequence number for all of
// its keys.
r.Properties.GlobalSeqNum = uint64(len(files) - i + seqNumOffset)
readers = append(readers, r)
}
return readers, err
}
// simpleLevelIter is similar to a levelIter in that it merges the points
// from multiple point iterators that are non-overlapping in the key ranges
// they return. It is only expected to support forward iteration and forward
// regular seeking; reverse iteration and prefix seeking is not supported.
// Intended to be a low-overhead, non-FileMetadata dependent option for
// NewExternalIter. To optimize seeking and forward iteration, it maintains
// two slices of child iterators; one of all iterators, and a subset of it that
// contains just the iterators that contain point keys within the current
// bounds.
//
// Note that this levelIter does not support pausing at file boundaries
// in case of range tombstones in this file that could apply to points outside
// of this file (and outside of this level). This is sufficient for optimizing
// the main use cases of NewExternalIter, however for completeness it would make
// sense to build this pausing functionality in.
type simpleLevelIter struct {
cmp Compare
err error
lowerBound []byte
iters []internalIterator
filtered []internalIterator
firstKeys [][]byte
firstKeysBuf []byte
currentIdx int
}
var _ internalIterator = &simpleLevelIter{}
// init initializes this simpleLevelIter.
func (s *simpleLevelIter) init(opts IterOptions) {
s.currentIdx = 0
s.lowerBound = opts.LowerBound
s.resetFilteredIters()
}
func (s *simpleLevelIter) resetFilteredIters() {
s.filtered = s.filtered[:0]
s.firstKeys = s.firstKeys[:0]
s.firstKeysBuf = s.firstKeysBuf[:0]
s.err = nil
for i := range s.iters {
var iterKey *base.InternalKey
if s.lowerBound != nil {
iterKey, _ = s.iters[i].SeekGE(s.lowerBound, base.SeekGEFlagsNone)
} else {
iterKey, _ = s.iters[i].First()
}
if iterKey != nil {
s.filtered = append(s.filtered, s.iters[i])
bufStart := len(s.firstKeysBuf)
s.firstKeysBuf = append(s.firstKeysBuf, iterKey.UserKey...)
s.firstKeys = append(s.firstKeys, s.firstKeysBuf[bufStart:bufStart+len(iterKey.UserKey)])
} else if err := s.iters[i].Error(); err != nil {
s.err = err
}
}
}
func (s *simpleLevelIter) SeekGE(
key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
if s.err != nil {
return nil, base.LazyValue{}
}
// Find the first file that is entirely >= key. The file before that could
// contain the key we're looking for.
n := sort.Search(len(s.firstKeys), func(i int) bool {
return s.cmp(key, s.firstKeys[i]) <= 0
})
if n > 0 {
s.currentIdx = n - 1
} else {
s.currentIdx = n
}
if s.currentIdx < len(s.filtered) {
if iterKey, val := s.filtered[s.currentIdx].SeekGE(key, flags); iterKey != nil {
return iterKey, val
}
if err := s.filtered[s.currentIdx].Error(); err != nil {
s.err = err
}
s.currentIdx++
}
return s.skipEmptyFileForward(key, flags)
}
func (s *simpleLevelIter) skipEmptyFileForward(
seekKey []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
var iterKey *base.InternalKey
var val base.LazyValue
for s.currentIdx >= 0 && s.currentIdx < len(s.filtered) && s.err == nil {
if seekKey != nil {
iterKey, val = s.filtered[s.currentIdx].SeekGE(seekKey, flags)
} else if s.lowerBound != nil {
iterKey, val = s.filtered[s.currentIdx].SeekGE(s.lowerBound, flags)
} else {
iterKey, val = s.filtered[s.currentIdx].First()
}
if iterKey != nil {
return iterKey, val
}
if err := s.filtered[s.currentIdx].Error(); err != nil {
s.err = err
}
s.currentIdx++
}
return nil, base.LazyValue{}
}
func (s *simpleLevelIter) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
func (s *simpleLevelIter) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
func (s *simpleLevelIter) First() (*base.InternalKey, base.LazyValue) {
if s.err != nil {
return nil, base.LazyValue{}
}
s.currentIdx = 0
return s.skipEmptyFileForward(nil /* seekKey */, base.SeekGEFlagsNone)
}
func (s *simpleLevelIter) Last() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
func (s *simpleLevelIter) Next() (*base.InternalKey, base.LazyValue) {
if s.err != nil {
return nil, base.LazyValue{}
}
if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) {
return nil, base.LazyValue{}
}
if iterKey, val := s.filtered[s.currentIdx].Next(); iterKey != nil {
return iterKey, val
}
s.currentIdx++
return s.skipEmptyFileForward(nil /* seekKey */, base.SeekGEFlagsNone)
}
func (s *simpleLevelIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
if s.err != nil {
return nil, base.LazyValue{}
}
if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) {
return nil, base.LazyValue{}
}
if iterKey, val := s.filtered[s.currentIdx].NextPrefix(succKey); iterKey != nil {
return iterKey, val
}
s.currentIdx++
return s.skipEmptyFileForward(succKey /* seekKey */, base.SeekGEFlagsNone)
}
func (s *simpleLevelIter) Prev() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
func (s *simpleLevelIter) Error() error {
if s.currentIdx >= 0 && s.currentIdx < len(s.filtered) {
s.err = firstError(s.err, s.filtered[s.currentIdx].Error())
}
return s.err
}
func (s *simpleLevelIter) Close() error {
var err error
for i := range s.iters {
err = firstError(err, s.iters[i].Close())
}
return err
}
func (s *simpleLevelIter) SetBounds(lower, upper []byte) {
s.currentIdx = -1
s.lowerBound = lower
for i := range s.iters {
s.iters[i].SetBounds(lower, upper)
}
s.resetFilteredIters()
}
func (s *simpleLevelIter) String() string {
if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) {
return "simpleLevelIter: current=<nil>"
}
return fmt.Sprintf("simpleLevelIter: current=%s", s.filtered[s.currentIdx])
}
var _ internalIterator = &simpleLevelIter{}

54
vendor/github.com/cockroachdb/pebble/filenames.go generated vendored Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/vfs"
)
type fileType = base.FileType
// FileNum is an identifier for a file within a database.
type FileNum = base.FileNum
const (
fileTypeLog = base.FileTypeLog
fileTypeLock = base.FileTypeLock
fileTypeTable = base.FileTypeTable
fileTypeManifest = base.FileTypeManifest
fileTypeCurrent = base.FileTypeCurrent
fileTypeOptions = base.FileTypeOptions
fileTypeTemp = base.FileTypeTemp
fileTypeOldTemp = base.FileTypeOldTemp
)
// setCurrentFile sets the CURRENT file to point to the manifest with
// provided file number.
//
// NB: This is a low-level routine and typically not what you want to
// use. Newer versions of Pebble running newer format major versions do
// not use the CURRENT file. See setCurrentFunc in version_set.go.
func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error {
newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum)
oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum)
fs.Remove(oldFilename)
f, err := fs.Create(oldFilename)
if err != nil {
return err
}
if _, err := fmt.Fprintf(f, "MANIFEST-%s\n", fileNum); err != nil {
return err
}
if err := f.Sync(); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
return fs.Rename(oldFilename, newFilename)
}

252
vendor/github.com/cockroachdb/pebble/flushable.go generated vendored Normal file
View File

@@ -0,0 +1,252 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"context"
"fmt"
"sync/atomic"
"time"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
)
// flushable defines the interface for immutable memtables.
type flushable interface {
newIter(o *IterOptions) internalIterator
newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator
newRangeDelIter(o *IterOptions) keyspan.FragmentIterator
newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator
containsRangeKeys() bool
// inuseBytes returns the number of inuse bytes by the flushable.
inuseBytes() uint64
// totalBytes returns the total number of bytes allocated by the flushable.
totalBytes() uint64
// readyForFlush returns true when the flushable is ready for flushing. See
// memTable.readyForFlush for one implementation which needs to check whether
// there are any outstanding write references.
readyForFlush() bool
}
// flushableEntry wraps a flushable and adds additional metadata and
// functionality that is common to all flushables.
type flushableEntry struct {
flushable
// Channel which is closed when the flushable has been flushed.
flushed chan struct{}
// flushForced indicates whether a flush was forced on this memtable (either
// manual, or due to ingestion). Protected by DB.mu.
flushForced bool
// delayedFlushForcedAt indicates whether a timer has been set to force a
// flush on this memtable at some point in the future. Protected by DB.mu.
// Holds the timestamp of when the flush will be issued.
delayedFlushForcedAt time.Time
// logNum corresponds to the WAL that contains the records present in the
// receiver.
logNum FileNum
// logSize is the size in bytes of the associated WAL. Protected by DB.mu.
logSize uint64
// The current logSeqNum at the time the memtable was created. This is
// guaranteed to be less than or equal to any seqnum stored in the memtable.
logSeqNum uint64
// readerRefs tracks the read references on the flushable. The two sources of
// reader references are DB.mu.mem.queue and readState.memtables. The memory
// reserved by the flushable in the cache is released when the reader refs
// drop to zero. If the flushable is referencing sstables, then the file
// refount is also decreased once the reader refs drops to 0. If the
// flushable is a memTable, when the reader refs drops to zero, the writer
// refs will already be zero because the memtable will have been flushed and
// that only occurs once the writer refs drops to zero.
readerRefs atomic.Int32
// Closure to invoke to release memory accounting.
releaseMemAccounting func()
// unrefFiles, if not nil, should be invoked to decrease the ref count of
// files which are backing the flushable.
unrefFiles func() []*fileBacking
// deleteFnLocked should be called if the caller is holding DB.mu.
deleteFnLocked func(obsolete []*fileBacking)
// deleteFn should be called if the caller is not holding DB.mu.
deleteFn func(obsolete []*fileBacking)
}
func (e *flushableEntry) readerRef() {
switch v := e.readerRefs.Add(1); {
case v <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
}
// db.mu must not be held when this is called.
func (e *flushableEntry) readerUnref(deleteFiles bool) {
e.readerUnrefHelper(deleteFiles, e.deleteFn)
}
// db.mu must be held when this is called.
func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) {
e.readerUnrefHelper(deleteFiles, e.deleteFnLocked)
}
func (e *flushableEntry) readerUnrefHelper(
deleteFiles bool, deleteFn func(obsolete []*fileBacking),
) {
switch v := e.readerRefs.Add(-1); {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
if e.releaseMemAccounting == nil {
panic("pebble: memtable reservation already released")
}
e.releaseMemAccounting()
e.releaseMemAccounting = nil
if e.unrefFiles != nil {
obsolete := e.unrefFiles()
e.unrefFiles = nil
if deleteFiles {
deleteFn(obsolete)
}
}
}
}
type flushableList []*flushableEntry
// ingestedFlushable is the implementation of the flushable interface for the
// ingesting sstables which are added to the flushable list.
type ingestedFlushable struct {
files []physicalMeta
cmp Compare
split Split
newIters tableNewIters
newRangeKeyIters keyspan.TableNewSpanIter
// Since the level slice is immutable, we construct and set it once. It
// should be safe to read from slice in future reads.
slice manifest.LevelSlice
// hasRangeKeys is set on ingestedFlushable construction.
hasRangeKeys bool
}
func newIngestedFlushable(
files []*fileMetadata,
cmp Compare,
split Split,
newIters tableNewIters,
newRangeKeyIters keyspan.TableNewSpanIter,
) *ingestedFlushable {
var physicalFiles []physicalMeta
var hasRangeKeys bool
for _, f := range files {
if f.HasRangeKeys {
hasRangeKeys = true
}
physicalFiles = append(physicalFiles, f.PhysicalMeta())
}
ret := &ingestedFlushable{
files: physicalFiles,
cmp: cmp,
split: split,
newIters: newIters,
newRangeKeyIters: newRangeKeyIters,
// slice is immutable and can be set once and used many times.
slice: manifest.NewLevelSliceKeySorted(cmp, files),
hasRangeKeys: hasRangeKeys,
}
return ret
}
// TODO(sumeer): ingestedFlushable iters also need to plumb context for
// tracing.
// newIter is part of the flushable interface.
func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
var opts IterOptions
if o != nil {
opts = *o
}
// TODO(bananabrick): The manifest.Level in newLevelIter is only used for
// logging. Update the manifest.Level encoding to account for levels which
// aren't truly levels in the lsm. Right now, the encoding only supports
// L0 sublevels, and the rest of the levels in the lsm.
return newLevelIter(
opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
)
}
// newFlushIter is part of the flushable interface.
func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
// newFlushIter is only used for writing memtables to disk as sstables.
// Since ingested sstables are already present on disk, they don't need to
// make use of a flush iter.
panic("pebble: not implemented")
}
func (s *ingestedFlushable) constructRangeDelIter(
file *manifest.FileMetadata, _ keyspan.SpanIterOptions,
) (keyspan.FragmentIterator, error) {
// Note that the keyspan level iter expects a non-nil iterator to be
// returned even if there is an error. So, we return the emptyKeyspanIter.
iter, rangeDelIter, err := s.newIters(context.Background(), file, nil, internalIterOpts{})
if err != nil {
return emptyKeyspanIter, err
}
iter.Close()
if rangeDelIter == nil {
return emptyKeyspanIter, nil
}
return rangeDelIter, nil
}
// newRangeDelIter is part of the flushable interface.
// TODO(bananabrick): Using a level iter instead of a keyspan level iter to
// surface range deletes is more efficient.
func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp,
s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
manifest.KeyTypePoint,
)
}
// newRangeKeyIter is part of the flushable interface.
func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
if !s.containsRangeKeys() {
return nil
}
return keyspan.NewLevelIter(
keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters,
s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
)
}
// containsRangeKeys is part of the flushable interface.
func (s *ingestedFlushable) containsRangeKeys() bool {
return s.hasRangeKeys
}
// inuseBytes is part of the flushable interface.
func (s *ingestedFlushable) inuseBytes() uint64 {
// inuseBytes is only used when memtables are flushed to disk as sstables.
panic("pebble: not implemented")
}
// totalBytes is part of the flushable interface.
func (s *ingestedFlushable) totalBytes() uint64 {
// We don't allocate additional bytes for the ingestedFlushable.
return 0
}
// readyForFlush is part of the flushable interface.
func (s *ingestedFlushable) readyForFlush() bool {
// ingestedFlushable should always be ready to flush. However, note that
// memtables before the ingested sstables in the memtable queue must be
// flushed before an ingestedFlushable can be flushed. This is because the
// ingested sstables need an updated view of the Version to
// determine where to place the files in the lsm.
return true
}

664
vendor/github.com/cockroachdb/pebble/format_major_version.go generated vendored Normal file
View File

@@ -0,0 +1,664 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"fmt"
"strconv"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/manifest"
"github.com/cockroachdb/pebble/sstable"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/pebble/vfs/atomicfs"
)
// FormatMajorVersion is a constant controlling the format of persisted
// data. Backwards incompatible changes to durable formats are gated
// behind new format major versions.
//
// At any point, a database's format major version may be bumped.
// However, once a database's format major version is increased,
// previous versions of Pebble will refuse to open the database.
//
// The zero value format is the FormatDefault constant. The exact
// FormatVersion that the default corresponds to may change with time.
type FormatMajorVersion uint64
// String implements fmt.Stringer.
func (v FormatMajorVersion) String() string {
// NB: This must not change. It's used as the value for the the
// on-disk version marker file.
//
// Specifically, this value must always parse as a base 10 integer
// that fits in a uint64. We format it as zero-padded, 3-digit
// number today, but the padding may change.
return fmt.Sprintf("%03d", v)
}
const (
// 21.2 versions.
// FormatDefault leaves the format version unspecified. The
// FormatDefault constant may be ratcheted upwards over time.
FormatDefault FormatMajorVersion = iota
// FormatMostCompatible maintains the most backwards compatibility,
// maintaining bi-directional compatibility with RocksDB 6.2.1 in
// the particular configuration described in the Pebble README.
FormatMostCompatible
// formatVersionedManifestMarker is the first
// backwards-incompatible change made to Pebble, introducing the
// format-version marker file for handling backwards-incompatible
// changes more broadly, and replacing the `CURRENT` file with a
// marker file.
//
// This format version is intended as an intermediary version state.
// It is deliberately unexported to discourage direct use of this
// format major version. Clients should use FormatVersioned which
// also ensures earlier versions of Pebble fail to open a database
// written in a future format major version.
formatVersionedManifestMarker
// FormatVersioned is a new format major version that replaces the
// old `CURRENT` file with a new 'marker' file scheme. Previous
// Pebble versions will be unable to open the database unless
// they're aware of format versions.
FormatVersioned
// FormatSetWithDelete is a format major version that introduces a new key
// kind, base.InternalKeyKindSetWithDelete. Previous Pebble versions will be
// unable to open this database.
FormatSetWithDelete
// 22.1 versions.
// FormatBlockPropertyCollector is a format major version that introduces
// BlockPropertyCollectors.
FormatBlockPropertyCollector
// FormatSplitUserKeysMarked is a format major version that guarantees that
// all files that share user keys with neighbors are marked for compaction
// in the manifest. Ratcheting to FormatSplitUserKeysMarked will block
// (without holding mutexes) until the scan of the LSM is complete and the
// manifest has been rotated.
FormatSplitUserKeysMarked
// 22.2 versions.
// FormatSplitUserKeysMarkedCompacted is a format major version that
// guarantees that all files explicitly marked for compaction in the manifest
// have been compacted. Combined with the FormatSplitUserKeysMarked format
// major version, this version guarantees that there are no user keys split
// across multiple files within a level L1+. Ratcheting to this format version
// will block (without holding mutexes) until all necessary compactions for
// files marked for compaction are complete.
FormatSplitUserKeysMarkedCompacted
// FormatRangeKeys is a format major version that introduces range keys.
FormatRangeKeys
// FormatMinTableFormatPebblev1 is a format major version that guarantees that
// tables created by or ingested into the DB at or above this format major
// version will have a table format version of at least Pebblev1 (Block
// Properties).
FormatMinTableFormatPebblev1
// FormatPrePebblev1Marked is a format major version that guarantees that all
// sstables with a table format version pre-Pebblev1 (i.e. those that are
// guaranteed to not contain block properties) are marked for compaction in
// the manifest. Ratcheting to FormatPrePebblev1Marked will block (without
// holding mutexes) until the scan of the LSM is complete and the manifest has
// been rotated.
FormatPrePebblev1Marked
// 23.1 versions.
// FormatUnusedPrePebblev1MarkedCompacted is an unused format major version.
// This format major version was originally intended to ship in the 23.1
// release. It was later decided that this should be deferred until a
// subsequent release. The original ordering is preserved so as not to
// introduce breaking changes in Cockroach.
FormatUnusedPrePebblev1MarkedCompacted
// FormatSSTableValueBlocks is a format major version that adds support for
// storing values in value blocks in the sstable. Value block support is not
// necessarily enabled when writing sstables, when running with this format
// major version.
//
// WARNING: In development, so no production code should upgrade to this
// format, since a DB with this format major version will not actually
// interoperate correctly with another DB with the same format major
// version. This format major version is introduced so that tests can start
// being executed up to this version. Note that these tests succeed despite
// the incomplete support since they do not enable value blocks and use
// TableFormatPebblev2.
FormatSSTableValueBlocks
// FormatFlushableIngest is a format major version that enables lazy
// addition of ingested sstables into the LSM structure. When an ingest
// overlaps with a memtable, a record of the ingest is written to the WAL
// without waiting for a flush. Subsequent reads treat the ingested files as
// a level above the overlapping memtable. Once the memtable is flushed, the
// ingested files are moved into the lowest possible levels.
//
// This feature is behind a format major version because it required
// breaking changes to the WAL format.
FormatFlushableIngest
// 23.2 versions.
// FormatPrePebblev1MarkedCompacted is a format major version that guarantees
// that all sstables explicitly marked for compaction in the manifest (see
// FormatPrePebblev1Marked) have been compacted. Ratcheting to this format
// version will block (without holding mutexes) until all necessary
// compactions for files marked for compaction are complete.
FormatPrePebblev1MarkedCompacted
// ExperimentalFormatDeleteSizedAndObsolete is a format major version that adds support
// for deletion tombstones that encode the size of the value they're
// expected to delete. This format major version is required before the
// associated key kind may be committed through batch applications or
// ingests. It also adds support for keys that are marked obsolete (see
// sstable/format.go for details).
ExperimentalFormatDeleteSizedAndObsolete
// ExperimentalFormatVirtualSSTables is a format major version that adds support for
// virtual sstables that can reference a sub-range of keys in an underlying
// physical sstable. This information is persisted through new,
// backward-incompatible fields in the Manifest, and therefore requires
// a format major version.
ExperimentalFormatVirtualSSTables
// internalFormatNewest holds the newest format major version, including
// experimental ones excluded from the exported FormatNewest constant until
// they've stabilized. Used in tests.
internalFormatNewest FormatMajorVersion = iota - 1
// FormatNewest always contains the most recent format major version.
// TODO(jackson): restore FormatNewest to internalFormatNewest.
FormatNewest FormatMajorVersion = FormatPrePebblev1MarkedCompacted
)
// MaxTableFormat returns the maximum sstable.TableFormat that can be used at
// this FormatMajorVersion.
func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
switch v {
case FormatDefault, FormatMostCompatible, formatVersionedManifestMarker,
FormatVersioned, FormatSetWithDelete:
return sstable.TableFormatRocksDBv2
case FormatBlockPropertyCollector, FormatSplitUserKeysMarked,
FormatSplitUserKeysMarkedCompacted:
return sstable.TableFormatPebblev1
case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatUnusedPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev2
case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
return sstable.TableFormatPebblev3
case ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
return sstable.TableFormatPebblev4
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
}
}
// MinTableFormat returns the minimum sstable.TableFormat that can be used at
// this FormatMajorVersion.
func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
switch v {
case FormatDefault, FormatMostCompatible, formatVersionedManifestMarker,
FormatVersioned, FormatSetWithDelete, FormatBlockPropertyCollector,
FormatSplitUserKeysMarked, FormatSplitUserKeysMarkedCompacted,
FormatRangeKeys:
return sstable.TableFormatLevelDB
case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
return sstable.TableFormatPebblev1
default:
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
}
}
// formatMajorVersionMigrations defines the migrations from one format
// major version to the next. Each migration is defined as a closure
// which will be invoked on the database before the new format major
// version is committed. Migrations must be idempotent. Migrations are
// invoked with d.mu locked.
//
// Each migration is responsible for invoking finalizeFormatVersUpgrade
// to set the new format major version. RatchetFormatMajorVersion will
// panic if a migration returns a nil error but fails to finalize the
// new format major version.
var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
FormatMostCompatible: func(d *DB) error { return nil },
formatVersionedManifestMarker: func(d *DB) error {
// formatVersionedManifestMarker introduces the use of a marker
// file for pointing to the current MANIFEST file.
// Lock the manifest.
d.mu.versions.logLock()
defer d.mu.versions.logUnlock()
// Construct the filename of the currently active manifest and
// move the manifest marker to that filename. The marker is
// guaranteed to exist, because we unconditionally locate it
// during Open.
manifestFileNum := d.mu.versions.manifestFileNum
filename := base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum())
if err := d.mu.versions.manifestMarker.Move(filename); err != nil {
return errors.Wrap(err, "moving manifest marker")
}
// Now that we have a manifest marker file in place and pointing
// to the current MANIFEST, finalize the upgrade. If we fail for
// some reason, a retry of this migration is guaranteed to again
// move the manifest marker file to the latest manifest. If
// we're unable to finalize the upgrade, a subsequent call to
// Open will ignore the manifest marker.
if err := d.finalizeFormatVersUpgrade(formatVersionedManifestMarker); err != nil {
return err
}
// We've finalized the upgrade. All subsequent Open calls will
// ignore the CURRENT file and instead read the manifest marker.
// Before we unlock the manifest, we need to update versionSet
// to use the manifest marker on future rotations.
d.mu.versions.setCurrent = setCurrentFuncMarker(
d.mu.versions.manifestMarker,
d.mu.versions.fs,
d.mu.versions.dirname)
return nil
},
// The FormatVersioned version is split into two, each with their
// own migration to ensure the post-migration cleanup happens even
// if there's a crash immediately after finalizing the version. Once
// a new format major version is finalized, its migration will never
// run again. Post-migration cleanup like the one in the migration
// below must be performed in a separate migration or every time the
// database opens.
FormatVersioned: func(d *DB) error {
// Replace the `CURRENT` file with one that points to the
// nonexistent `MANIFEST-000000` file. If an earlier Pebble
// version that does not know about format major versions
// attempts to open the database, it will error avoiding
// accidental corruption.
if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatVersioned)
},
// As SetWithDelete is a new key kind, there is nothing to migrate. We can
// simply finalize the format version and we're done.
FormatSetWithDelete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatSetWithDelete)
},
FormatBlockPropertyCollector: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatBlockPropertyCollector)
},
FormatSplitUserKeysMarked: func(d *DB) error {
// Mark any unmarked files with split-user keys. Note all format major
// versions migrations are invoked with DB.mu locked.
if err := d.markFilesLocked(markFilesWithSplitUserKeys(d.opts.Comparer.Equal)); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatSplitUserKeysMarked)
},
FormatSplitUserKeysMarkedCompacted: func(d *DB) error {
// Before finalizing the format major version, rewrite any sstables
// still marked for compaction. Note all format major versions
// migrations are invoked with DB.mu locked.
if err := d.compactMarkedFilesLocked(); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatSplitUserKeysMarkedCompacted)
},
FormatRangeKeys: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatRangeKeys)
},
FormatMinTableFormatPebblev1: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatMinTableFormatPebblev1)
},
FormatPrePebblev1Marked: func(d *DB) error {
// Mark any unmarked files that contain only table properties. Note all
// format major versions migrations are invoked with DB.mu locked.
if err := d.markFilesLocked(markFilesPrePebblev1(d.tableCache)); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
},
FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
// Intentional no-op.
return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted)
},
FormatSSTableValueBlocks: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
},
FormatFlushableIngest: func(d *DB) error {
return d.finalizeFormatVersUpgrade(FormatFlushableIngest)
},
FormatPrePebblev1MarkedCompacted: func(d *DB) error {
// Before finalizing the format major version, rewrite any sstables
// still marked for compaction. Note all format major versions
// migrations are invoked with DB.mu locked.
if err := d.compactMarkedFilesLocked(); err != nil {
return err
}
return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
},
ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error {
return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete)
},
ExperimentalFormatVirtualSSTables: func(d *DB) error {
return d.finalizeFormatVersUpgrade(ExperimentalFormatVirtualSSTables)
},
}
const formatVersionMarkerName = `format-version`
func lookupFormatMajorVersion(
fs vfs.FS, dirname string,
) (FormatMajorVersion, *atomicfs.Marker, error) {
m, versString, err := atomicfs.LocateMarker(fs, dirname, formatVersionMarkerName)
if err != nil {
return 0, nil, err
}
if versString == "" {
return FormatMostCompatible, m, nil
}
v, err := strconv.ParseUint(versString, 10, 64)
if err != nil {
return 0, nil, errors.Wrap(err, "parsing format major version")
}
vers := FormatMajorVersion(v)
if vers == FormatDefault {
return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers)
}
if vers > internalFormatNewest {
return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers)
}
return vers, m, nil
}
// FormatMajorVersion returns the database's active format major
// version. The format major version may be higher than the one
// provided in Options when the database was opened if the existing
// database was written with a higher format version.
func (d *DB) FormatMajorVersion() FormatMajorVersion {
return FormatMajorVersion(d.mu.formatVers.vers.Load())
}
// RatchetFormatMajorVersion ratchets the opened database's format major
// version to the provided version. It errors if the provided format
// major version is below the database's current version. Once a
// database's format major version is upgraded, previous Pebble versions
// that do not know of the format version will be unable to open the
// database.
func (d *DB) RatchetFormatMajorVersion(fmv FormatMajorVersion) error {
if err := d.closed.Load(); err != nil {
panic(err)
}
d.mu.Lock()
defer d.mu.Unlock()
return d.ratchetFormatMajorVersionLocked(fmv)
}
func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) error {
if d.opts.ReadOnly {
return ErrReadOnly
}
if formatVers > internalFormatNewest {
// Guard against accidentally forgetting to update internalFormatNewest.
return errors.Errorf("pebble: unknown format version %d", formatVers)
}
if currentVers := d.FormatMajorVersion(); currentVers > formatVers {
return errors.Newf("pebble: database already at format major version %d; cannot reduce to %d",
currentVers, formatVers)
}
if d.mu.formatVers.ratcheting {
return errors.Newf("pebble: database format major version upgrade is in-progress")
}
d.mu.formatVers.ratcheting = true
defer func() { d.mu.formatVers.ratcheting = false }()
for nextVers := d.FormatMajorVersion() + 1; nextVers <= formatVers; nextVers++ {
if err := formatMajorVersionMigrations[nextVers](d); err != nil {
return errors.Wrapf(err, "migrating to version %d", nextVers)
}
// NB: The migration is responsible for calling
// finalizeFormatVersUpgrade to finalize the upgrade. This
// structure is necessary because some migrations may need to
// update in-memory state (without ever dropping locks) after
// the upgrade is finalized. Here we assert that the upgrade
// did occur.
if d.FormatMajorVersion() != nextVers {
d.opts.Logger.Fatalf("pebble: successful migration to format version %d never finalized the upgrade", nextVers)
}
}
return nil
}
// finalizeFormatVersUpgrade is typically only be called from within a
// format major version migration.
//
// See formatMajorVersionMigrations.
func (d *DB) finalizeFormatVersUpgrade(formatVers FormatMajorVersion) error {
// We use the marker to encode the active format version in the
// marker filename. Unlike other uses of the atomic marker, there is
// no file with the filename `formatVers.String()` on the
// filesystem.
if err := d.mu.formatVers.marker.Move(formatVers.String()); err != nil {
return err
}
d.mu.formatVers.vers.Store(uint64(formatVers))
d.opts.EventListener.FormatUpgrade(formatVers)
return nil
}
// compactMarkedFilesLocked performs a migration that schedules rewrite
// compactions to compact away any sstables marked for compaction.
// compactMarkedFilesLocked is run while ratcheting the database's format major
// version to FormatSplitUserKeysMarkedCompacted.
//
// Note that while this method is called with the DB.mu held, and will not
// return until all marked files have been compacted, the mutex is dropped while
// waiting for compactions to complete (or for slots to free up).
func (d *DB) compactMarkedFilesLocked() error {
curr := d.mu.versions.currentVersion()
for curr.Stats.MarkedForCompaction > 0 {
// Attempt to schedule a compaction to rewrite a file marked for
// compaction.
d.maybeScheduleCompactionPicker(func(picker compactionPicker, env compactionEnv) *pickedCompaction {
return picker.pickRewriteCompaction(env)
})
// The above attempt might succeed and schedule a rewrite compaction. Or
// there might not be available compaction concurrency to schedule the
// compaction. Or compaction of the file might have already been in
// progress. In any scenario, wait until there's some change in the
// state of active compactions.
// Before waiting, check that the database hasn't been closed. Trying to
// schedule the compaction may have dropped d.mu while waiting for a
// manifest write to complete. In that dropped interim, the database may
// have been closed.
if err := d.closed.Load(); err != nil {
return err.(error)
}
// Some flush or compaction may have scheduled or completed while we waited
// for the manifest lock in maybeScheduleCompactionPicker. Get the latest
// Version before waiting on a compaction.
curr = d.mu.versions.currentVersion()
// Only wait on compactions if there are files still marked for compaction.
// NB: Waiting on this condition variable drops d.mu while blocked.
if curr.Stats.MarkedForCompaction > 0 {
if d.mu.compact.compactingCount == 0 {
panic("expected a compaction of marked files in progress")
}
d.mu.compact.cond.Wait()
// Refresh the current version again.
curr = d.mu.versions.currentVersion()
}
}
return nil
}
// findFilesFunc scans the LSM for files, returning true if at least one
// file was found. The returned array contains the matched files, if any, per
// level.
type findFilesFunc func(v *version) (found bool, files [numLevels][]*fileMetadata, _ error)
// markFilesWithSplitUserKeys scans the LSM's levels 1 through 6 for adjacent
// files that contain the same user key. Such arrangements of files were
// permitted in RocksDB and in Pebble up to SHA a860bbad.
var markFilesWithSplitUserKeys = func(equal Equal) findFilesFunc {
return func(v *version) (found bool, files [numLevels][]*fileMetadata, _ error) {
// Files with split user keys are expected to be rare and performing key
// comparisons for every file within the LSM is expensive, so drop the
// database lock while scanning the file metadata.
for l := numLevels - 1; l > 0; l-- {
iter := v.Levels[l].Iter()
var prevFile *fileMetadata
var prevUserKey []byte
for f := iter.First(); f != nil; f = iter.Next() {
if prevUserKey != nil && equal(prevUserKey, f.Smallest.UserKey) {
// NB: We may append a file twice, once as prevFile and once
// as f. That's okay, and handled below.
files[l] = append(files[l], prevFile, f)
found = true
}
if f.Largest.IsExclusiveSentinel() {
prevUserKey = nil
prevFile = nil
} else {
prevUserKey = f.Largest.UserKey
prevFile = f
}
}
}
return
}
}
// markFilesPrePebblev1 scans the LSM for files that do not support block
// properties (i.e. a table format version pre-Pebblev1).
var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc {
return func(v *version) (found bool, files [numLevels][]*fileMetadata, err error) {
for l := numLevels - 1; l > 0; l-- {
iter := v.Levels[l].Iter()
for f := iter.First(); f != nil; f = iter.Next() {
if f.Virtual {
// Any physical sstable which has been virtualized must
// have already undergone this migration, and we don't
// need to worry about the virtual sstable themselves.
panic("pebble: unexpected virtual sstable during migration")
}
err = tc.withReader(
f.PhysicalMeta(), func(r *sstable.Reader) error {
tf, err := r.TableFormat()
if err != nil {
return err
}
if tf < sstable.TableFormatPebblev1 {
found = true
files[l] = append(files[l], f)
}
return nil
})
if err != nil {
return
}
}
}
return
}
}
// markFilesLock durably marks the files that match the given findFilesFunc for
// compaction.
func (d *DB) markFilesLocked(findFn findFilesFunc) error {
jobID := d.mu.nextJobID
d.mu.nextJobID++
// Acquire a read state to have a view of the LSM and a guarantee that none
// of the referenced files will be deleted until we've unreferenced the read
// state. Some findFilesFuncs may read the files, requiring they not be
// deleted.
rs := d.loadReadState()
var (
found bool
files [numLevels][]*fileMetadata
err error
)
func() {
defer rs.unrefLocked()
// Note the unusual locking: unlock, defer Lock(). The scan of the files in
// the version does not need to block other operations that require the
// DB.mu. Drop it for the scan, before re-acquiring it.
d.mu.Unlock()
defer d.mu.Lock()
found, files, err = findFn(rs.current)
}()
if err != nil {
return err
}
// The database lock has been acquired again by the defer within the above
// anonymous function.
if !found {
// Nothing to do.
return nil
}
// After scanning, if we found files to mark, we fetch the current state of
// the LSM (which may have changed) and set MarkedForCompaction on the files,
// and update the version's Stats.MarkedForCompaction count, which are both
// protected by d.mu.
// Lock the manifest for a coherent view of the LSM. The database lock has
// been re-acquired by the defer within the above anonymous function.
d.mu.versions.logLock()
vers := d.mu.versions.currentVersion()
for l, filesToMark := range files {
if len(filesToMark) == 0 {
continue
}
for _, f := range filesToMark {
// Ignore files to be marked that have already been compacted or marked.
if f.CompactionState == manifest.CompactionStateCompacted ||
f.MarkedForCompaction {
continue
}
// Else, mark the file for compaction in this version.
vers.Stats.MarkedForCompaction++
f.MarkedForCompaction = true
}
// The compaction picker uses the markedForCompactionAnnotator to
// quickly find files marked for compaction, or to quickly determine
// that there are no such files marked for compaction within a level.
// A b-tree node may be annotated with an annotation recording that
// there are no files marked for compaction within the node's subtree,
// based on the assumption that it's static.
//
// Since we're marking files for compaction, these b-tree nodes'
// annotations will be out of date. Clear the compaction-picking
// annotation, so that it's recomputed the next time the compaction
// picker looks for a file marked for compaction.
vers.Levels[l].InvalidateAnnotation(markedForCompactionAnnotator{})
}
// The 'marked-for-compaction' bit is persisted in the MANIFEST file
// metadata. We've already modified the in-memory file metadata, but the
// manifest hasn't been updated. Force rotation to a new MANIFEST file,
// which will write every file metadata to the new manifest file and ensure
// that the now marked-for-compaction file metadata are persisted as marked.
// NB: This call to logAndApply will unlockthe MANIFEST, which we locked up
// above before obtaining `vers`.
return d.mu.versions.logAndApply(
jobID,
&manifest.VersionEdit{},
map[int]*LevelMetrics{},
true, /* forceRotation */
func() []compactionInfo { return d.getInProgressCompactionInfoLocked(nil) })
}

245
vendor/github.com/cockroachdb/pebble/get_iter.go generated vendored Normal file
View File

@@ -0,0 +1,245 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import (
"context"
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
)
// getIter is an internal iterator used to perform gets. It iterates through
// the values for a particular key, level by level. It is not a general purpose
// internalIterator, but specialized for Get operations so that it loads data
// lazily.
type getIter struct {
logger Logger
cmp Compare
equal Equal
split Split
newIters tableNewIters
snapshot uint64
key []byte
iter internalIterator
rangeDelIter keyspan.FragmentIterator
tombstone *keyspan.Span
levelIter levelIter
level int
batch *Batch
mem flushableList
l0 []manifest.LevelSlice
version *version
iterKey *InternalKey
iterValue base.LazyValue
err error
}
// TODO(sumeer): CockroachDB code doesn't use getIter, but, for completeness,
// make this implement InternalIteratorWithStats.
// getIter implements the base.InternalIterator interface.
var _ base.InternalIterator = (*getIter)(nil)
func (g *getIter) String() string {
return fmt.Sprintf("len(l0)=%d, len(mem)=%d, level=%d", len(g.l0), len(g.mem), g.level)
}
func (g *getIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) {
panic("pebble: SeekGE unimplemented")
}
func (g *getIter) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekPrefixGE unimplemented")
}
func (g *getIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, base.LazyValue) {
panic("pebble: SeekLT unimplemented")
}
func (g *getIter) First() (*InternalKey, base.LazyValue) {
return g.Next()
}
func (g *getIter) Last() (*InternalKey, base.LazyValue) {
panic("pebble: Last unimplemented")
}
func (g *getIter) Next() (*InternalKey, base.LazyValue) {
if g.iter != nil {
g.iterKey, g.iterValue = g.iter.Next()
}
for {
if g.iter != nil {
// We have to check rangeDelIter on each iteration because a single
// user-key can be spread across multiple tables in a level. A range
// tombstone will appear in the table corresponding to its start
// key. Every call to levelIter.Next() potentially switches to a new
// table and thus reinitializes rangeDelIter.
if g.rangeDelIter != nil {
g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key)
if g.err = g.rangeDelIter.Close(); g.err != nil {
return nil, base.LazyValue{}
}
g.rangeDelIter = nil
}
if g.iterKey != nil {
key := g.iterKey
if g.tombstone != nil && g.tombstone.CoversAt(g.snapshot, key.SeqNum()) {
// We have a range tombstone covering this key. Rather than return a
// point or range deletion here, we return false and close our
// internal iterator which will make Valid() return false,
// effectively stopping iteration.
g.err = g.iter.Close()
g.iter = nil
return nil, base.LazyValue{}
}
if g.equal(g.key, key.UserKey) {
if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) {
g.iterKey, g.iterValue = g.iter.Next()
continue
}
return g.iterKey, g.iterValue
}
}
// We've advanced the iterator passed the desired key. Move on to the
// next memtable / level.
g.err = g.iter.Close()
g.iter = nil
if g.err != nil {
return nil, base.LazyValue{}
}
}
// Create an iterator from the batch.
if g.batch != nil {
if g.batch.index == nil {
g.err = ErrNotIndexed
g.iterKey, g.iterValue = nil, base.LazyValue{}
return nil, base.LazyValue{}
}
g.iter = g.batch.newInternalIter(nil)
g.rangeDelIter = g.batch.newRangeDelIter(
nil,
// Get always reads the entirety of the batch's history, so no
// batch keys should be filtered.
base.InternalKeySeqNumMax,
)
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
g.batch = nil
continue
}
// If we have a tombstone from a previous level it is guaranteed to delete
// keys in lower levels.
if g.tombstone != nil && g.tombstone.VisibleAt(g.snapshot) {
return nil, base.LazyValue{}
}
// Create iterators from memtables from newest to oldest.
if n := len(g.mem); n > 0 {
m := g.mem[n-1]
g.iter = m.newIter(nil)
g.rangeDelIter = m.newRangeDelIter(nil)
g.mem = g.mem[:n-1]
g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
continue
}
if g.level == 0 {
// Create iterators from L0 from newest to oldest.
if n := len(g.l0); n > 0 {
files := g.l0[n-1].Iter()
g.l0 = g.l0[:n-1]
iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
g.levelIter.init(context.Background(), iterOpts, g.cmp, g.split, g.newIters,
files, manifest.L0Sublevel(n), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
bc := levelIterBoundaryContext{}
g.levelIter.initBoundaryContext(&bc)
g.iter = &g.levelIter
// Compute the key prefix for bloom filtering if split function is
// specified, or use the user key as default.
prefix := g.key
if g.split != nil {
prefix = g.key[:g.split(g.key)]
}
g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
g.iterKey = nil
g.iterValue = base.LazyValue{}
}
continue
}
g.level++
}
if g.level >= numLevels {
return nil, base.LazyValue{}
}
if g.version.Levels[g.level].Empty() {
g.level++
continue
}
iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
g.levelIter.init(context.Background(), iterOpts, g.cmp, g.split, g.newIters,
g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
g.levelIter.initRangeDel(&g.rangeDelIter)
bc := levelIterBoundaryContext{}
g.levelIter.initBoundaryContext(&bc)
g.level++
g.iter = &g.levelIter
// Compute the key prefix for bloom filtering if split function is
// specified, or use the user key as default.
prefix := g.key
if g.split != nil {
prefix = g.key[:g.split(g.key)]
}
g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
g.iterKey = nil
g.iterValue = base.LazyValue{}
}
}
}
func (g *getIter) Prev() (*InternalKey, base.LazyValue) {
panic("pebble: Prev unimplemented")
}
func (g *getIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) {
panic("pebble: NextPrefix unimplemented")
}
func (g *getIter) Valid() bool {
return g.iterKey != nil && g.err == nil
}
func (g *getIter) Error() error {
return g.err
}
func (g *getIter) Close() error {
if g.iter != nil {
if err := g.iter.Close(); err != nil && g.err == nil {
g.err = err
}
g.iter = nil
}
return g.err
}
func (g *getIter) SetBounds(lower, upper []byte) {
panic("pebble: SetBounds unimplemented")
}

2112
vendor/github.com/cockroachdb/pebble/ingest.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

51
vendor/github.com/cockroachdb/pebble/internal.go generated vendored Normal file
View File

@@ -0,0 +1,51 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package pebble
import "github.com/cockroachdb/pebble/internal/base"
// InternalKeyKind exports the base.InternalKeyKind type.
type InternalKeyKind = base.InternalKeyKind
// These constants are part of the file format, and should not be changed.
const (
InternalKeyKindDelete = base.InternalKeyKindDelete
InternalKeyKindSet = base.InternalKeyKindSet
InternalKeyKindMerge = base.InternalKeyKindMerge
InternalKeyKindLogData = base.InternalKeyKindLogData
InternalKeyKindSingleDelete = base.InternalKeyKindSingleDelete
InternalKeyKindRangeDelete = base.InternalKeyKindRangeDelete
InternalKeyKindMax = base.InternalKeyKindMax
InternalKeyKindSetWithDelete = base.InternalKeyKindSetWithDelete
InternalKeyKindRangeKeySet = base.InternalKeyKindRangeKeySet
InternalKeyKindRangeKeyUnset = base.InternalKeyKindRangeKeyUnset
InternalKeyKindRangeKeyDelete = base.InternalKeyKindRangeKeyDelete
InternalKeyKindIngestSST = base.InternalKeyKindIngestSST
InternalKeyKindDeleteSized = base.InternalKeyKindDeleteSized
InternalKeyKindInvalid = base.InternalKeyKindInvalid
InternalKeySeqNumBatch = base.InternalKeySeqNumBatch
InternalKeySeqNumMax = base.InternalKeySeqNumMax
InternalKeyRangeDeleteSentinel = base.InternalKeyRangeDeleteSentinel
)
// InternalKey exports the base.InternalKey type.
type InternalKey = base.InternalKey
type internalIterator = base.InternalIterator
// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST,
// sstable) isn't in the expected format.
var ErrCorruption = base.ErrCorruption
// AttributeAndLen exports the base.AttributeAndLen type.
type AttributeAndLen = base.AttributeAndLen
// ShortAttribute exports the base.ShortAttribute type.
type ShortAttribute = base.ShortAttribute
// LazyFetcher exports the base.LazyFetcher type. This export is needed since
// LazyValue.Clone requires a pointer to a LazyFetcher struct to avoid
// allocations. No code outside Pebble needs to peer into a LazyFetcher.
type LazyFetcher = base.LazyFetcher

201
vendor/github.com/cockroachdb/pebble/internal/arenaskl/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

93
vendor/github.com/cockroachdb/pebble/internal/arenaskl/README.md generated vendored Normal file
View File

@@ -0,0 +1,93 @@
# arenaskl
Fast, lock-free, arena-based Skiplist implementation in Go that supports iteration
in both directions.
## Advantages
Arenaskl offers several advantages over other skiplist implementations:
* High performance that linearly scales with the number of cores. This is
achieved by allocating from a fixed-size arena and by avoiding locks.
* Iterators that can be allocated on the stack and easily cloned by value.
* Simple-to-use and low overhead model for detecting and handling race conditions
with other threads.
* Support for iterating in reverse (i.e. previous links).
## Limitations
The advantages come at a cost that prevents arenaskl from being a general-purpose
skiplist implementation:
* The size of the arena sets a hard upper bound on the combined size of skiplist
nodes, keys, and values. This limit includes even the size of deleted nodes,
keys, and values.
* Deletion is not supported. Instead, higher-level code is expected to
add deletion tombstones and needs to process those tombstones
appropriately.
## Pedigree
This code is based on Andy Kimball's arenaskl code:
https://github.com/andy-kimball/arenaskl
The arenaskl code is based on the skiplist found in Badger, a Go-based
KV store:
https://github.com/dgraph-io/badger/tree/master/skl
The skiplist in Badger is itself based on a C++ skiplist built for
Facebook's RocksDB:
https://github.com/facebook/rocksdb/tree/master/memtable
## Benchmarks
The benchmarks consist of a mix of reads and writes executed in parallel. The
fraction of reads is indicated in the run name: "frac_X" indicates a run where
X percent of the operations are reads.
The results are much better than `skiplist` and `slist`.
```
name time/op
ReadWrite/frac_0-8 470ns ±11%
ReadWrite/frac_10-8 462ns ± 3%
ReadWrite/frac_20-8 436ns ± 2%
ReadWrite/frac_30-8 410ns ± 2%
ReadWrite/frac_40-8 385ns ± 2%
ReadWrite/frac_50-8 360ns ± 4%
ReadWrite/frac_60-8 386ns ± 1%
ReadWrite/frac_70-8 352ns ± 2%
ReadWrite/frac_80-8 306ns ± 3%
ReadWrite/frac_90-8 253ns ± 4%
ReadWrite/frac_100-8 28.1ns ± 2%
```
Note that the above numbers are for concurrent operations using 8x
parallelism. The same benchmarks without concurrency (use these
numbers when comparing vs batchskl):
```
name time/op
ReadWrite/frac_0 1.53µs ± 1%
ReadWrite/frac_10 1.46µs ± 2%
ReadWrite/frac_20 1.39µs ± 3%
ReadWrite/frac_30 1.28µs ± 3%
ReadWrite/frac_40 1.21µs ± 2%
ReadWrite/frac_50 1.11µs ± 3%
ReadWrite/frac_60 1.23µs ±17%
ReadWrite/frac_70 1.16µs ± 4%
ReadWrite/frac_80 959ns ± 3%
ReadWrite/frac_90 738ns ± 5%
ReadWrite/frac_100 81.9ns ± 2%
```
Forward and backward iteration are also fast:
```
name time/op
IterNext 3.97ns ± 5%
IterPrev 3.88ns ± 3%
```

125
vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go generated vendored Normal file
View File

@@ -0,0 +1,125 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/constants"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Arena is lock-free.
type Arena struct {
n atomic.Uint64
buf []byte
}
const nodeAlignment = 4
var (
// ErrArenaFull indicates that the arena is full and cannot perform any more
// allocations.
ErrArenaFull = errors.New("allocation failed because arena is full")
)
// NewArena allocates a new arena using the specified buffer as the backing
// store.
func NewArena(buf []byte) *Arena {
if len(buf) > constants.MaxUint32OrInt {
if invariants.Enabled {
panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
}
buf = buf[:constants.MaxUint32OrInt]
}
a := &Arena{
buf: buf,
}
// We don't store data at position 0 in order to reserve offset=0 as a kind of
// nil pointer.
a.n.Store(1)
return a
}
// Size returns the number of bytes allocated by the arena.
func (a *Arena) Size() uint32 {
s := a.n.Load()
if s > constants.MaxUint32OrInt {
// The last failed allocation can push the size higher than len(a.buf).
// Saturate at the maximum representable offset.
return constants.MaxUint32OrInt
}
return uint32(s)
}
// Capacity returns the capacity of the arena.
func (a *Arena) Capacity() uint32 {
return uint32(len(a.buf))
}
// alloc allocates a buffer of the given size and with the given alignment
// (which must be a power of 2).
//
// If overflow is not 0, it also ensures that many bytes after the buffer are
// inside the arena (this is used for structures that are larger than the
// requested size but don't use those extra bytes).
func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
if invariants.Enabled && (alignment&(alignment-1)) != 0 {
panic(errors.AssertionFailedf("invalid alignment %d", alignment))
}
// Verify that the arena isn't already full.
origSize := a.n.Load()
if int(origSize) > len(a.buf) {
return 0, 0, ErrArenaFull
}
// Pad the allocation with enough bytes to ensure the requested alignment.
padded := uint64(size) + uint64(alignment) - 1
newSize := a.n.Add(padded)
if newSize+uint64(overflow) > uint64(len(a.buf)) {
return 0, 0, ErrArenaFull
}
// Return the aligned offset.
offset := (uint32(newSize) - size) & ^(alignment - 1)
return offset, uint32(padded), nil
}
func (a *Arena) getBytes(offset uint32, size uint32) []byte {
if offset == 0 {
return nil
}
return a.buf[offset : offset+size : offset+size]
}
func (a *Arena) getPointer(offset uint32) unsafe.Pointer {
if offset == 0 {
return nil
}
return unsafe.Pointer(&a.buf[offset])
}
func (a *Arena) getPointerOffset(ptr unsafe.Pointer) uint32 {
if ptr == nil {
return 0
}
return uint32(uintptr(ptr) - uintptr(unsafe.Pointer(&a.buf[0])))
}

88
vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go generated vendored Normal file
View File

@@ -0,0 +1,88 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import "github.com/cockroachdb/pebble/internal/base"
// flushIterator is an iterator over the skiplist object. Use Skiplist.NewFlushIter
// to construct an iterator. The current state of the iterator can be cloned by
// simply value copying the struct.
type flushIterator struct {
Iterator
bytesIterated *uint64
}
// flushIterator implements the base.InternalIterator interface.
var _ base.InternalIterator = (*flushIterator)(nil)
func (it *flushIterator) String() string {
return "memtable"
}
func (it *flushIterator) SeekGE(
key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekGE unimplemented")
}
func (it *flushIterator) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekPrefixGE unimplemented")
}
func (it *flushIterator) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekLT unimplemented")
}
// First seeks position at the first entry in list. Returns the key and value
// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that First only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound.
func (it *flushIterator) First() (*base.InternalKey, base.LazyValue) {
key, val := it.Iterator.First()
if key == nil {
return nil, base.LazyValue{}
}
*it.bytesIterated += uint64(it.nd.allocSize)
return key, val
}
// Next advances to the next position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
// Note: flushIterator.Next mirrors the implementation of Iterator.Next
// due to performance. Keep the two in sync.
func (it *flushIterator) Next() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
*it.bytesIterated += uint64(it.nd.allocSize)
return &it.key, base.MakeInPlaceValue(it.value())
}
func (it *flushIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
panic("pebble: NextPrefix unimplemented")
}
func (it *flushIterator) Prev() (*base.InternalKey, base.LazyValue) {
panic("pebble: Prev unimplemented")
}

271
vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,271 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"sync"
"github.com/cockroachdb/pebble/internal/base"
)
type splice struct {
prev *node
next *node
}
func (s *splice) init(prev, next *node) {
s.prev = prev
s.next = next
}
// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
// to construct an iterator. The current state of the iterator can be cloned by
// simply value copying the struct. All iterator methods are thread-safe.
type Iterator struct {
list *Skiplist
nd *node
key base.InternalKey
lower []byte
upper []byte
}
// Iterator implements the base.InternalIterator interface.
var _ base.InternalIterator = (*Iterator)(nil)
var iterPool = sync.Pool{
New: func() interface{} {
return &Iterator{}
},
}
// Close resets the iterator.
func (it *Iterator) Close() error {
it.list = nil
it.nd = nil
it.lower = nil
it.upper = nil
iterPool.Put(it)
return nil
}
func (it *Iterator) String() string {
return "memtable"
}
// Error returns any accumulated error.
func (it *Iterator) Error() error {
return nil
}
// SeekGE moves the iterator to the first entry whose key is greater than or
// equal to the given key. Returns the key and value if the iterator is
// pointing at a valid entry, and (nil, nil) otherwise. Note that SeekGE only
// checks the upper bound. It is up to the caller to ensure that key is greater
// than or equal to the lower bound.
func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
if flags.TrySeekUsingNext() {
if it.nd == it.list.tail {
// Iterator is done.
return nil, base.LazyValue{}
}
less := it.list.cmp(it.key.UserKey, key) < 0
// Arbitrary constant. By measuring the seek cost as a function of the
// number of elements in the skip list, and fitting to a model, we
// could adjust the number of nexts based on the current size of the
// skip list.
const numNexts = 5
for i := 0; less && i < numNexts; i++ {
k, _ := it.Next()
if k == nil {
// Iterator is done.
return nil, base.LazyValue{}
}
less = it.list.cmp(it.key.UserKey, key) < 0
}
if !less {
return &it.key, base.MakeInPlaceValue(it.value())
}
}
_, it.nd, _ = it.seekForBaseSplice(key)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// SeekPrefixGE moves the iterator to the first entry whose key is greater than
// or equal to the given key. This method is equivalent to SeekGE and is
// provided so that an arenaskl.Iterator implements the
// internal/base.InternalIterator interface.
func (it *Iterator) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
return it.SeekGE(key, flags)
}
// SeekLT moves the iterator to the last entry whose key is less than the given
// key. Returns the key and value if the iterator is pointing at a valid entry,
// and (nil, nil) otherwise. Note that SeekLT only checks the lower bound. It
// is up to the caller to ensure that key is less than the upper bound.
func (it *Iterator) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
// NB: the top-level Iterator has already adjusted key based on
// the upper-bound.
it.nd, _, _ = it.seekForBaseSplice(key)
if it.nd == it.list.head {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// First seeks position at the first entry in list. Returns the key and value
// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that First only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound (e.g. via a call to SeekGE(lower)).
func (it *Iterator) First() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getNext(it.list.head, 0)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// Last seeks position at the last entry in list. Returns the key and value if
// the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that Last only checks the lower bound. It is up to the caller to ensure that
// key is less than the upper bound (e.g. via a call to SeekLT(upper)).
func (it *Iterator) Last() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getPrev(it.list.tail, 0)
if it.nd == it.list.head {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// Next advances to the next position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
// Note: flushIterator.Next mirrors the implementation of Iterator.Next
// due to performance. Keep the two in sync.
func (it *Iterator) Next() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// NextPrefix advances to the next position with a new prefix. Returns the key
// and value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise.
func (it *Iterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
return it.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext())
}
// Prev moves to the previous position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
func (it *Iterator) Prev() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getPrev(it.nd, 0)
if it.nd == it.list.head {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// value returns the value at the current position.
func (it *Iterator) value() []byte {
return it.nd.getValue(it.list.arena)
}
// Head true iff the iterator is positioned at the sentinel head node.
func (it *Iterator) Head() bool {
return it.nd == it.list.head
}
// Tail true iff the iterator is positioned at the sentinel tail node.
func (it *Iterator) Tail() bool {
return it.nd == it.list.tail
}
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
func (it *Iterator) SetBounds(lower, upper []byte) {
it.lower = lower
it.upper = upper
}
func (it *Iterator) decodeKey() {
it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
it.key.Trailer = it.nd.keyTrailer
}
func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {
ikey := base.MakeSearchKey(key)
level := int(it.list.Height() - 1)
prev = it.list.head
for {
prev, next, found = it.list.findSpliceForLevel(ikey, level, prev)
if found {
if level != 0 {
// next is pointing at the target node, but we need to find previous on
// the bottom level.
prev = it.list.getPrev(next, 0)
}
break
}
if level == 0 {
break
}
level--
}
return
}

133
vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go generated vendored Normal file
View File

@@ -0,0 +1,133 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"math"
"sync/atomic"
"github.com/cockroachdb/pebble/internal/base"
)
// MaxNodeSize returns the maximum space needed for a node with the specified
// key and value sizes. This could overflow a uint32, which is why a uint64
// is used here. If a key/value overflows a uint32, it should not be added to
// the skiplist.
func MaxNodeSize(keySize, valueSize uint32) uint64 {
const maxPadding = nodeAlignment - 1
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
}
type links struct {
nextOffset atomic.Uint32
prevOffset atomic.Uint32
}
func (l *links) init(prevOffset, nextOffset uint32) {
l.nextOffset.Store(nextOffset)
l.prevOffset.Store(prevOffset)
}
type node struct {
// Immutable fields, so no need to lock to access key.
keyOffset uint32
keySize uint32
keyTrailer uint64
valueSize uint32
allocSize uint32
// Most nodes do not need to use the full height of the tower, since the
// probability of each successive level decreases exponentially. Because
// these elements are never accessed, they do not need to be allocated.
// Therefore, when a node is allocated in the arena, its memory footprint
// is deliberately truncated to not include unneeded tower elements.
//
// All accesses to elements should use CAS operations, with no need to lock.
tower [maxHeight]links
}
func newNode(
arena *Arena, height uint32, key base.InternalKey, value []byte,
) (nd *node, err error) {
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
keySize := len(key.UserKey)
if int64(keySize) > math.MaxUint32 {
panic("key is too large")
}
valueSize := len(value)
if int64(len(value)) > math.MaxUint32 {
panic("value is too large")
}
if int64(len(value))+int64(keySize)+int64(maxNodeSize) > math.MaxUint32 {
panic("combined key and value size is too large")
}
nd, err = newRawNode(arena, height, uint32(keySize), uint32(valueSize))
if err != nil {
return
}
nd.keyTrailer = key.Trailer
copy(nd.getKeyBytes(arena), key.UserKey)
copy(nd.getValue(arena), value)
return
}
func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *node, err error) {
// Compute the amount of the tower that will never be used, since the height
// is less than maxHeight.
unusedSize := uint32((maxHeight - int(height)) * linksSize)
nodeSize := uint32(maxNodeSize) - unusedSize
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
if err != nil {
return
}
nd = (*node)(arena.getPointer(nodeOffset))
nd.keyOffset = nodeOffset + nodeSize
nd.keySize = keySize
nd.valueSize = valueSize
nd.allocSize = allocSize
return
}
func (n *node) getKeyBytes(arena *Arena) []byte {
return arena.getBytes(n.keyOffset, n.keySize)
}
func (n *node) getValue(arena *Arena) []byte {
return arena.getBytes(n.keyOffset+n.keySize, uint32(n.valueSize))
}
func (n *node) nextOffset(h int) uint32 {
return n.tower[h].nextOffset.Load()
}
func (n *node) prevOffset(h int) uint32 {
return n.tower[h].prevOffset.Load()
}
func (n *node) casNextOffset(h int, old, val uint32) bool {
return n.tower[h].nextOffset.CompareAndSwap(old, val)
}
func (n *node) casPrevOffset(h int, old, val uint32) bool {
return n.tower[h].prevOffset.CompareAndSwap(old, val)
}

464
vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go generated vendored Normal file
View File

@@ -0,0 +1,464 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Adapted from RocksDB inline skiplist.
Key differences:
- No optimization for sequential inserts (no "prev").
- No custom comparator.
- Support overwrites. This requires care when we see the same key when inserting.
For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
there is no need for values. We don't intend to support versioning. In-place updates of values
would be more efficient.
- We discard all non-concurrent code.
- We do not support Splices. This simplifies the code a lot.
- No AllocateNode or other pointer arithmetic.
- We combine the findLessThan, findGreaterOrEqual, etc into one function.
*/
/*
Further adapted from Badger: https://github.com/dgraph-io/badger.
Key differences:
- Support for previous pointers - doubly linked lists. Note that it's up to higher
level code to deal with the intermediate state that occurs during insertion,
where node A is linked to node B, but node B is not yet linked back to node A.
- Iterator includes mutator functions.
*/
package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
import (
"math"
"runtime"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/fastrand"
)
const (
maxHeight = 20
maxNodeSize = int(unsafe.Sizeof(node{}))
linksSize = int(unsafe.Sizeof(links{}))
pValue = 1 / math.E
)
// ErrRecordExists indicates that an entry with the specified key already
// exists in the skiplist. Duplicate entries are not directly supported and
// instead must be handled by the user by appending a unique version suffix to
// keys.
var ErrRecordExists = errors.New("record with this key already exists")
// Skiplist is a fast, concurrent skiplist implementation that supports forward
// and backward iteration. See batchskl.Skiplist for a non-concurrent
// skiplist. Keys and values are immutable once added to the skiplist and
// deletion is not supported. Instead, higher-level code is expected to add new
// entries that shadow existing entries and perform deletion via tombstones. It
// is up to the user to process these shadow entries and tombstones
// appropriately during retrieval.
type Skiplist struct {
arena *Arena
cmp base.Compare
head *node
tail *node
height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
// If set to true by tests, then extra delays are added to make it easier to
// detect unusual race conditions.
testing bool
}
// Inserter TODO(peter)
type Inserter struct {
spl [maxHeight]splice
height uint32
}
// Add TODO(peter)
func (ins *Inserter) Add(list *Skiplist, key base.InternalKey, value []byte) error {
return list.addInternal(key, value, ins)
}
var (
probabilities [maxHeight]uint32
)
func init() {
// Precompute the skiplist probabilities so that only a single random number
// needs to be generated and so that the optimal pvalue can be used (inverse
// of Euler's number).
p := float64(1.0)
for i := 0; i < maxHeight; i++ {
probabilities[i] = uint32(float64(math.MaxUint32) * p)
p *= pValue
}
}
// NewSkiplist constructs and initializes a new, empty skiplist. All nodes, keys,
// and values in the skiplist will be allocated from the given arena.
func NewSkiplist(arena *Arena, cmp base.Compare) *Skiplist {
skl := &Skiplist{}
skl.Reset(arena, cmp)
return skl
}
// Reset the skiplist to empty and re-initialize.
func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
// Allocate head and tail nodes.
head, err := newRawNode(arena, maxHeight, 0, 0)
if err != nil {
panic("arenaSize is not large enough to hold the head node")
}
head.keyOffset = 0
tail, err := newRawNode(arena, maxHeight, 0, 0)
if err != nil {
panic("arenaSize is not large enough to hold the tail node")
}
tail.keyOffset = 0
// Link all head/tail levels together.
headOffset := arena.getPointerOffset(unsafe.Pointer(head))
tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
for i := 0; i < maxHeight; i++ {
head.tower[i].nextOffset.Store(tailOffset)
tail.tower[i].prevOffset.Store(headOffset)
}
*s = Skiplist{
arena: arena,
cmp: cmp,
head: head,
tail: tail,
}
s.height.Store(1)
}
// Height returns the height of the highest tower within any of the nodes that
// have ever been allocated as part of this skiplist.
func (s *Skiplist) Height() uint32 { return s.height.Load() }
// Arena returns the arena backing this skiplist.
func (s *Skiplist) Arena() *Arena { return s.arena }
// Size returns the number of bytes that have allocated from the arena.
func (s *Skiplist) Size() uint32 { return s.arena.Size() }
// Add adds a new key if it does not yet exist. If the key already exists, then
// Add returns ErrRecordExists. If there isn't enough room in the arena, then
// Add returns ErrArenaFull.
func (s *Skiplist) Add(key base.InternalKey, value []byte) error {
var ins Inserter
return s.addInternal(key, value, &ins)
}
func (s *Skiplist) addInternal(key base.InternalKey, value []byte, ins *Inserter) error {
if s.findSplice(key, ins) {
// Found a matching node, but handle case where it's been deleted.
return ErrRecordExists
}
if s.testing {
// Add delay to make it easier to test race between this thread
// and another thread that sees the intermediate state between
// finding the splice and using it.
runtime.Gosched()
}
nd, height, err := s.newNode(key, value)
if err != nil {
return err
}
ndOffset := s.arena.getPointerOffset(unsafe.Pointer(nd))
// We always insert from the base level and up. After you add a node in base
// level, we cannot create a node in the level above because it would have
// discovered the node in the base level.
var found bool
var invalidateSplice bool
for i := 0; i < int(height); i++ {
prev := ins.spl[i].prev
next := ins.spl[i].next
if prev == nil {
// New node increased the height of the skiplist, so assume that the
// new level has not yet been populated.
if next != nil {
panic("next is expected to be nil, since prev is nil")
}
prev = s.head
next = s.tail
}
// +----------------+ +------------+ +----------------+
// | prev | | nd | | next |
// | prevNextOffset |---->| | | |
// | |<----| prevOffset | | |
// | | | nextOffset |---->| |
// | | | |<----| nextPrevOffset |
// +----------------+ +------------+ +----------------+
//
// 1. Initialize prevOffset and nextOffset to point to prev and next.
// 2. CAS prevNextOffset to repoint from next to nd.
// 3. CAS nextPrevOffset to repoint from prev to nd.
for {
prevOffset := s.arena.getPointerOffset(unsafe.Pointer(prev))
nextOffset := s.arena.getPointerOffset(unsafe.Pointer(next))
nd.tower[i].init(prevOffset, nextOffset)
// Check whether next has an updated link to prev. If it does not,
// that can mean one of two things:
// 1. The thread that added the next node hasn't yet had a chance
// to add the prev link (but will shortly).
// 2. Another thread has added a new node between prev and next.
nextPrevOffset := next.prevOffset(i)
if nextPrevOffset != prevOffset {
// Determine whether #1 or #2 is true by checking whether prev
// is still pointing to next. As long as the atomic operations
// have at least acquire/release semantics (no need for
// sequential consistency), this works, as it is equivalent to
// the "publication safety" pattern.
prevNextOffset := prev.nextOffset(i)
if prevNextOffset == nextOffset {
// Ok, case #1 is true, so help the other thread along by
// updating the next node's prev link.
next.casPrevOffset(i, nextPrevOffset, prevOffset)
}
}
if prev.casNextOffset(i, nextOffset, ndOffset) {
// Managed to insert nd between prev and next, so update the next
// node's prev link and go to the next level.
if s.testing {
// Add delay to make it easier to test race between this thread
// and another thread that sees the intermediate state between
// setting next and setting prev.
runtime.Gosched()
}
next.casPrevOffset(i, prevOffset, ndOffset)
break
}
// CAS failed. We need to recompute prev and next. It is unlikely to
// be helpful to try to use a different level as we redo the search,
// because it is unlikely that lots of nodes are inserted between prev
// and next.
prev, next, found = s.findSpliceForLevel(key, i, prev)
if found {
if i != 0 {
panic("how can another thread have inserted a node at a non-base level?")
}
return ErrRecordExists
}
invalidateSplice = true
}
}
// If we had to recompute the splice for a level, invalidate the entire
// cached splice.
if invalidateSplice {
ins.height = 0
} else {
// The splice was valid. We inserted a node between spl[i].prev and
// spl[i].next. Optimistically update spl[i].prev for use in a subsequent
// call to add.
for i := uint32(0); i < height; i++ {
ins.spl[i].prev = nd
}
}
return nil
}
// NewIter returns a new Iterator object. The lower and upper bound parameters
// control the range of keys the iterator will return. Specifying for nil for
// lower or upper bound disables the check for that boundary. Note that lower
// bound is not checked on {SeekGE,First} and upper bound is not check on
// {SeekLT,Last}. The user is expected to perform that check. Note that it is
// safe for an iterator to be copied by value.
func (s *Skiplist) NewIter(lower, upper []byte) *Iterator {
it := iterPool.Get().(*Iterator)
*it = Iterator{list: s, nd: s.head, lower: lower, upper: upper}
return it
}
// NewFlushIter returns a new flushIterator, which is similar to an Iterator
// but also sets the current number of the bytes that have been iterated
// through.
func (s *Skiplist) NewFlushIter(bytesFlushed *uint64) base.InternalIterator {
return &flushIterator{
Iterator: Iterator{list: s, nd: s.head},
bytesIterated: bytesFlushed,
}
}
func (s *Skiplist) newNode(
key base.InternalKey, value []byte,
) (nd *node, height uint32, err error) {
height = s.randomHeight()
nd, err = newNode(s.arena, height, key, value)
if err != nil {
return
}
// Try to increase s.height via CAS.
listHeight := s.Height()
for height > listHeight {
if s.height.CompareAndSwap(listHeight, height) {
// Successfully increased skiplist.height.
break
}
listHeight = s.Height()
}
return
}
func (s *Skiplist) randomHeight() uint32 {
rnd := fastrand.Uint32()
h := uint32(1)
for h < maxHeight && rnd <= probabilities[h] {
h++
}
return h
}
func (s *Skiplist) findSplice(key base.InternalKey, ins *Inserter) (found bool) {
listHeight := s.Height()
var level int
prev := s.head
if ins.height < listHeight {
// Our cached height is less than the list height, which means there were
// inserts that increased the height of the list. Recompute the splice from
// scratch.
ins.height = listHeight
level = int(ins.height)
} else {
// Our cached height is equal to the list height.
for ; level < int(listHeight); level++ {
spl := &ins.spl[level]
if s.getNext(spl.prev, level) != spl.next {
// One or more nodes have been inserted between the splice at this
// level.
continue
}
if spl.prev != s.head && !s.keyIsAfterNode(spl.prev, key) {
// Key lies before splice.
level = int(listHeight)
break
}
if spl.next != s.tail && s.keyIsAfterNode(spl.next, key) {
// Key lies after splice.
level = int(listHeight)
break
}
// The splice brackets the key!
prev = spl.prev
break
}
}
for level = level - 1; level >= 0; level-- {
var next *node
prev, next, found = s.findSpliceForLevel(key, level, prev)
if next == nil {
next = s.tail
}
ins.spl[level].init(prev, next)
}
return
}
func (s *Skiplist) findSpliceForLevel(
key base.InternalKey, level int, start *node,
) (prev, next *node, found bool) {
prev = start
for {
// Assume prev.key < key.
next = s.getNext(prev, level)
if next == s.tail {
// Tail node, so done.
break
}
offset, size := next.keyOffset, next.keySize
nextKey := s.arena.buf[offset : offset+size]
cmp := s.cmp(key.UserKey, nextKey)
if cmp < 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
if cmp == 0 {
// User-key equality.
if key.Trailer == next.keyTrailer {
// Internal key equality.
found = true
break
}
if key.Trailer > next.keyTrailer {
// We are done for this level, since prev.key < key < next.key.
break
}
}
// Keep moving right on this level.
prev = next
}
return
}
func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
cmp := s.cmp(ndKey, key.UserKey)
if cmp < 0 {
return true
}
if cmp > 0 {
return false
}
// User-key equality.
if key.Trailer == nd.keyTrailer {
// Internal key equality.
return false
}
return key.Trailer < nd.keyTrailer
}
func (s *Skiplist) getNext(nd *node, h int) *node {
offset := nd.tower[h].nextOffset.Load()
return (*node)(s.arena.getPointer(offset))
}
func (s *Skiplist) getPrev(nd *node, h int) *node {
offset := nd.tower[h].prevOffset.Load()
return (*node)(s.arena.getPointer(offset))
}

60
vendor/github.com/cockroachdb/pebble/internal/base/cleaner.go generated vendored Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/pebble/vfs"
// Cleaner cleans obsolete files.
type Cleaner interface {
Clean(fs vfs.FS, fileType FileType, path string) error
}
// NeedsFileContents is implemented by a cleaner that needs the contents of the
// files that it is being asked to clean.
type NeedsFileContents interface {
needsFileContents()
}
// DeleteCleaner deletes file.
type DeleteCleaner struct{}
// Clean removes file.
func (DeleteCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
return fs.Remove(path)
}
func (DeleteCleaner) String() string {
return "delete"
}
// ArchiveCleaner archives file instead delete.
type ArchiveCleaner struct{}
var _ NeedsFileContents = ArchiveCleaner{}
// Clean archives file.
func (ArchiveCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
switch fileType {
case FileTypeLog, FileTypeManifest, FileTypeTable:
destDir := fs.PathJoin(fs.PathDir(path), "archive")
if err := fs.MkdirAll(destDir, 0755); err != nil {
return err
}
destPath := fs.PathJoin(destDir, fs.PathBase(path))
return fs.Rename(path, destPath)
default:
return fs.Remove(path)
}
}
func (ArchiveCleaner) String() string {
return "archive"
}
func (ArchiveCleaner) needsFileContents() {
}

260
vendor/github.com/cockroachdb/pebble/internal/base/comparer.go generated vendored Normal file
View File

@@ -0,0 +1,260 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"bytes"
"encoding/binary"
"fmt"
"strconv"
"unicode/utf8"
)
// Compare returns -1, 0, or +1 depending on whether a is 'less than', 'equal
// to' or 'greater than' b. The two arguments can only be 'equal' if their
// contents are exactly equal. Furthermore, the empty slice must be 'less than'
// any non-empty slice. Compare is used to compare user keys, such as those
// passed as arguments to the various DB methods, as well as those returned
// from Separator, Successor, and Split.
type Compare func(a, b []byte) int
// Equal returns true if a and b are equivalent. For a given Compare,
// Equal(a,b) must return true iff Compare(a,b) returns zero, that is,
// Equal is a (potentially faster) specialization of Compare.
type Equal func(a, b []byte) bool
// AbbreviatedKey returns a fixed length prefix of a user key such that AbbreviatedKey(a)
// < AbbreviatedKey(b) iff a < b and AbbreviatedKey(a) > AbbreviatedKey(b) iff a > b. If
// AbbreviatedKey(a) == AbbreviatedKey(b) an additional comparison is required to
// determine if the two keys are actually equal.
//
// This helps optimize indexed batch comparisons for cache locality. If a Split
// function is specified, AbbreviatedKey usually returns the first eight bytes
// of the user key prefix in the order that gives the correct ordering.
type AbbreviatedKey func(key []byte) uint64
// FormatKey returns a formatter for the user key.
type FormatKey func(key []byte) fmt.Formatter
// FormatValue returns a formatter for the user value. The key is also
// specified for the value formatter in order to support value formatting that
// is dependent on the key.
type FormatValue func(key, value []byte) fmt.Formatter
// Separator is used to construct SSTable index blocks. A trivial implementation
// is `return a`, but appending fewer bytes leads to smaller SSTables.
//
// Given keys a, b for which Compare(a, b) < 0, Separator returns a key k such
// that:
//
// 1. Compare(a, k) <= 0, and
// 2. Compare(k, b) < 0.
//
// As a special case, b may be nil in which case the second condition is dropped.
//
// For example, if dst, a and b are the []byte equivalents of the strings
// "aqua", "black" and "blue", then the result may be "aquablb".
// Similarly, if the arguments were "aqua", "green" and "", then the result
// may be "aquah".
type Separator func(dst, a, b []byte) []byte
// Successor returns a shortened key given a key a, such that Compare(k, a) >=
// 0. A simple implementation may return a unchanged. The dst parameter may be
// used to store the returned key, though it is valid to pass nil. The returned
// key must be valid to pass to Compare.
type Successor func(dst, a []byte) []byte
// ImmediateSuccessor is invoked with a prefix key ([Split(a) == len(a)]) and
// returns the smallest key that is larger than the given prefix a.
// ImmediateSuccessor must return a prefix key k such that:
//
// Split(k) == len(k) and Compare(k, a) > 0
//
// and there exists no representable k2 such that:
//
// Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
//
// As an example, an implementation built on the natural byte ordering using
// bytes.Compare could append a `\0` to `a`.
//
// The dst parameter may be used to store the returned key, though it is valid
// to pass nil. The returned key must be valid to pass to Compare.
type ImmediateSuccessor func(dst, a []byte) []byte
// Split returns the length of the prefix of the user key that corresponds to
// the key portion of an MVCC encoding scheme to enable the use of prefix bloom
// filters.
//
// The method will only ever be called with valid MVCC keys, that is, keys that
// the user could potentially store in the database. Pebble does not know which
// keys are MVCC keys and which are not, and may call Split on both MVCC keys
// and non-MVCC keys.
//
// A trivial MVCC scheme is one in which Split() returns len(a). This
// corresponds to assigning a constant version to each key in the database. For
// performance reasons, it is preferable to use a `nil` split in this case.
//
// The returned prefix must have the following properties:
//
// 1. The prefix must be a byte prefix:
//
// bytes.HasPrefix(a, prefix(a))
//
// 2. A key consisting of just a prefix must sort before all other keys with
// that prefix:
//
// Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
//
// 3. Prefixes must be used to order keys before suffixes:
//
// If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
//
// 4. Suffixes themselves must be valid keys and comparable, respecting the same
// ordering as within a key.
//
// If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
type Split func(a []byte) int
// Comparer defines a total ordering over the space of []byte keys: a 'less
// than' relationship.
type Comparer struct {
Compare Compare
Equal Equal
AbbreviatedKey AbbreviatedKey
FormatKey FormatKey
FormatValue FormatValue
Separator Separator
Split Split
Successor Successor
ImmediateSuccessor ImmediateSuccessor
// Name is the name of the comparer.
//
// The Level-DB on-disk format stores the comparer name, and opening a
// database with a different comparer from the one it was created with
// will result in an error.
Name string
}
// DefaultFormatter is the default implementation of user key formatting:
// non-ASCII data is formatted as escaped hexadecimal values.
var DefaultFormatter = func(key []byte) fmt.Formatter {
return FormatBytes(key)
}
// DefaultComparer is the default implementation of the Comparer interface.
// It uses the natural ordering, consistent with bytes.Compare.
var DefaultComparer = &Comparer{
Compare: bytes.Compare,
Equal: bytes.Equal,
AbbreviatedKey: func(key []byte) uint64 {
if len(key) >= 8 {
return binary.BigEndian.Uint64(key)
}
var v uint64
for _, b := range key {
v <<= 8
v |= uint64(b)
}
return v << uint(8*(8-len(key)))
},
FormatKey: DefaultFormatter,
Separator: func(dst, a, b []byte) []byte {
i, n := SharedPrefixLen(a, b), len(dst)
dst = append(dst, a...)
min := len(a)
if min > len(b) {
min = len(b)
}
if i >= min {
// Do not shorten if one string is a prefix of the other.
return dst
}
if a[i] >= b[i] {
// b is smaller than a or a is already the shortest possible.
return dst
}
if i < len(b)-1 || a[i]+1 < b[i] {
i += n
dst[i]++
return dst[:i+1]
}
i += n + 1
for ; i < len(dst); i++ {
if dst[i] != 0xff {
dst[i]++
return dst[:i+1]
}
}
return dst
},
Successor: func(dst, a []byte) (ret []byte) {
for i := 0; i < len(a); i++ {
if a[i] != 0xff {
dst = append(dst, a[:i+1]...)
dst[len(dst)-1]++
return dst
}
}
// a is a run of 0xffs, leave it alone.
return append(dst, a...)
},
ImmediateSuccessor: func(dst, a []byte) (ret []byte) {
return append(append(dst, a...), 0x00)
},
// This name is part of the C++ Level-DB implementation's default file
// format, and should not be changed.
Name: "leveldb.BytewiseComparator",
}
// SharedPrefixLen returns the largest i such that a[:i] equals b[:i].
// This function can be useful in implementing the Comparer interface.
func SharedPrefixLen(a, b []byte) int {
i, n := 0, len(a)
if n > len(b) {
n = len(b)
}
asUint64 := func(c []byte, i int) uint64 {
return binary.LittleEndian.Uint64(c[i:])
}
for i < n-7 && asUint64(a, i) == asUint64(b, i) {
i += 8
}
for i < n && a[i] == b[i] {
i++
}
return i
}
// FormatBytes formats a byte slice using hexadecimal escapes for non-ASCII
// data.
type FormatBytes []byte
const lowerhex = "0123456789abcdef"
// Format implements the fmt.Formatter interface.
func (p FormatBytes) Format(s fmt.State, c rune) {
buf := make([]byte, 0, len(p))
for _, b := range p {
if b < utf8.RuneSelf && strconv.IsPrint(rune(b)) {
buf = append(buf, b)
continue
}
buf = append(buf, `\x`...)
buf = append(buf, lowerhex[b>>4])
buf = append(buf, lowerhex[b&0xF])
}
s.Write(buf)
}

28
vendor/github.com/cockroachdb/pebble/internal/base/error.go generated vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/errors"
// ErrNotFound means that a get or delete call did not find the requested key.
var ErrNotFound = errors.New("pebble: not found")
// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST,
// sstable) isn't in the expected format.
var ErrCorruption = errors.New("pebble: corruption")
// MarkCorruptionError marks given error as a corruption error.
func MarkCorruptionError(err error) error {
if errors.Is(err, ErrCorruption) {
return err
}
return errors.Mark(err, ErrCorruption)
}
// CorruptionErrorf formats according to a format specifier and returns
// the string as an error value that is marked as a corruption error.
func CorruptionErrorf(format string, args ...interface{}) error {
return errors.Mark(errors.Newf(format, args...), ErrCorruption)
}

194
vendor/github.com/cockroachdb/pebble/internal/base/filenames.go generated vendored Normal file
View File

@@ -0,0 +1,194 @@
// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"fmt"
"strconv"
"strings"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/redact"
)
// FileNum is an internal DB identifier for a file.
type FileNum uint64
// String returns a string representation of the file number.
func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
// called if the caller can ensure that the FileNum belongs to a physical file
// on disk. These could be manifests, log files, physical sstables on disk, the
// options file, but not virtual sstables.
func (fn FileNum) DiskFileNum() DiskFileNum {
return DiskFileNum{fn}
}
// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
// Note that a FileNum is an internal DB identifier and it could belong to files
// which don't exist on disk. An example would be virtual sstable FileNums.
// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
// FileNum to DiskFileNum may not be valid and care should be taken to prove
// that the FileNum actually exists on disk.
type DiskFileNum struct {
fn FileNum
}
func (dfn DiskFileNum) String() string { return dfn.fn.String() }
// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
func (dfn DiskFileNum) FileNum() FileNum {
return dfn.fn
}
// FileType enumerates the types of files found in a DB.
type FileType int
// The FileType enumeration.
const (
FileTypeLog FileType = iota
FileTypeLock
FileTypeTable
FileTypeManifest
FileTypeCurrent
FileTypeOptions
FileTypeOldTemp
FileTypeTemp
)
// MakeFilename builds a filename from components.
func MakeFilename(fileType FileType, dfn DiskFileNum) string {
switch fileType {
case FileTypeLog:
return fmt.Sprintf("%s.log", dfn)
case FileTypeLock:
return "LOCK"
case FileTypeTable:
return fmt.Sprintf("%s.sst", dfn)
case FileTypeManifest:
return fmt.Sprintf("MANIFEST-%s", dfn)
case FileTypeCurrent:
return "CURRENT"
case FileTypeOptions:
return fmt.Sprintf("OPTIONS-%s", dfn)
case FileTypeOldTemp:
return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
case FileTypeTemp:
return fmt.Sprintf("temporary.%s.dbtmp", dfn)
}
panic("unreachable")
}
// MakeFilepath builds a filepath from components.
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
}
// ParseFilename parses the components from a filename.
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
filename = fs.PathBase(filename)
switch {
case filename == "CURRENT":
return FileTypeCurrent, DiskFileNum{0}, true
case filename == "LOCK":
return FileTypeLock, DiskFileNum{0}, true
case strings.HasPrefix(filename, "MANIFEST-"):
dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
if !ok {
break
}
return FileTypeManifest, dfn, true
case strings.HasPrefix(filename, "OPTIONS-"):
dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
if !ok {
break
}
return FileTypeOptions, dfn, ok
case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeOldTemp, dfn, ok
case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeTemp, dfn, ok
default:
i := strings.IndexByte(filename, '.')
if i < 0 {
break
}
dfn, ok = parseDiskFileNum(filename[:i])
if !ok {
break
}
switch filename[i+1:] {
case "sst":
return FileTypeTable, dfn, true
case "log":
return FileTypeLog, dfn, true
}
}
return 0, dfn, false
}
func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
u, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return dfn, false
}
return DiskFileNum{FileNum(u)}, true
}
// A Fataler fatals a process with a message when called.
type Fataler interface {
Fatalf(format string, args ...interface{})
}
// MustExist checks if err is an error indicating a file does not exist.
// If it is, it lists the containing directory's files to annotate the error
// with counts of the various types of files and invokes the provided fataler.
// See cockroachdb/cockroach#56490.
func MustExist(fs vfs.FS, filename string, fataler Fataler, err error) {
if err == nil || !oserror.IsNotExist(err) {
return
}
ls, lsErr := fs.List(fs.PathDir(filename))
if lsErr != nil {
// TODO(jackson): if oserror.IsNotExist(lsErr), the the data directory
// doesn't exist anymore. Another process likely deleted it before
// killing the process. We want to fatal the process, but without
// triggering error reporting like Sentry.
fataler.Fatalf("%s:\norig err: %s\nlist err: %s", redact.Safe(fs.PathBase(filename)), err, lsErr)
}
var total, unknown, tables, logs, manifests int
total = len(ls)
for _, f := range ls {
typ, _, ok := ParseFilename(fs, f)
if !ok {
unknown++
continue
}
switch typ {
case FileTypeTable:
tables++
case FileTypeLog:
logs++
case FileTypeManifest:
manifests++
}
}
fataler.Fatalf("%s:\n%s\ndirectory contains %d files, %d unknown, %d tables, %d logs, %d manifests",
fs.PathBase(filename), err, total, unknown, tables, logs, manifests)
}

494
vendor/github.com/cockroachdb/pebble/internal/base/internal.go generated vendored Normal file
View File

@@ -0,0 +1,494 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base // import "github.com/cockroachdb/pebble/internal/base"
import (
"encoding/binary"
"fmt"
"strconv"
"strings"
)
const (
// SeqNumZero is the zero sequence number, set by compactions if they can
// guarantee there are no keys underneath an internal key.
SeqNumZero = uint64(0)
// SeqNumStart is the first sequence number assigned to a key. Sequence
// numbers 1-9 are reserved for potential future use.
SeqNumStart = uint64(10)
)
// InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
// value, a merged value, etc.
type InternalKeyKind uint8
// These constants are part of the file format, and should not be changed.
const (
InternalKeyKindDelete InternalKeyKind = 0
InternalKeyKindSet InternalKeyKind = 1
InternalKeyKindMerge InternalKeyKind = 2
InternalKeyKindLogData InternalKeyKind = 3
//InternalKeyKindColumnFamilyDeletion InternalKeyKind = 4
//InternalKeyKindColumnFamilyValue InternalKeyKind = 5
//InternalKeyKindColumnFamilyMerge InternalKeyKind = 6
// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
// solely for compactions (to reduce write amp and space amp). Readers other
// than compactions should treat SINGLEDEL as equivalent to a DEL.
// Historically, it was simpler for readers other than compactions to treat
// SINGLEDEL as equivalent to DEL, but as of the introduction of
// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
// correctness.
InternalKeyKindSingleDelete InternalKeyKind = 7
//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
//InternalKeyKindBeginPrepareXID InternalKeyKind = 9
//InternalKeyKindEndPrepareXID InternalKeyKind = 10
//InternalKeyKindCommitXID InternalKeyKind = 11
//InternalKeyKindRollbackXID InternalKeyKind = 12
//InternalKeyKindNoop InternalKeyKind = 13
//InternalKeyKindColumnFamilyRangeDelete InternalKeyKind = 14
InternalKeyKindRangeDelete InternalKeyKind = 15
//InternalKeyKindColumnFamilyBlobIndex InternalKeyKind = 16
//InternalKeyKindBlobIndex InternalKeyKind = 17
// InternalKeyKindSeparator is a key used for separator / successor keys
// written to sstable block indexes.
//
// NOTE: the RocksDB value has been repurposed. This was done to ensure that
// keys written to block indexes with value "17" (when 17 happened to be the
// max value, and InternalKeyKindMax was therefore set to 17), remain stable
// when new key kinds are supported in Pebble.
InternalKeyKindSeparator InternalKeyKind = 17
// InternalKeyKindSetWithDelete keys are SET keys that have met with a
// DELETE or SINGLEDEL key in a prior compaction. This key kind is
// specific to Pebble. See
// https://github.com/cockroachdb/pebble/issues/1255.
InternalKeyKindSetWithDelete InternalKeyKind = 18
// InternalKeyKindRangeKeyDelete removes all range keys within a key range.
// See the internal/rangekey package for more details.
InternalKeyKindRangeKeyDelete InternalKeyKind = 19
// InternalKeyKindRangeKeySet and InternalKeyKindRangeUnset represent
// keys that set and unset values associated with ranges of key
// space. See the internal/rangekey package for more details.
InternalKeyKindRangeKeyUnset InternalKeyKind = 20
InternalKeyKindRangeKeySet InternalKeyKind = 21
// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
// the WAL entry for ingested sstables that are added to the flushable
// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
// batch, or in an sstable.
InternalKeyKindIngestSST InternalKeyKind = 22
// InternalKeyKindDeleteSized keys behave identically to
// InternalKeyKindDelete keys, except that they hold an associated uint64
// value indicating the (len(key)+len(value)) of the shadowed entry the
// tombstone is expected to delete. This value is used to inform compaction
// heuristics, but is not required to be accurate for correctness.
InternalKeyKindDeleteSized InternalKeyKind = 23
// This maximum value isn't part of the file format. Future extensions may
// increase this value.
//
// When constructing an internal key to pass to DB.Seek{GE,LE},
// internalKeyComparer sorts decreasing by kind (after sorting increasing by
// user key and decreasing by sequence number). Thus, use InternalKeyKindMax,
// which sorts 'less than or equal to' any other valid internalKeyKind, when
// searching for any kind of internal key formed by a certain user key and
// seqNum.
InternalKeyKindMax InternalKeyKind = 23
// Internal to the sstable format. Not exposed by any sstable iterator.
// Declared here to prevent definition of valid key kinds that set this bit.
InternalKeyKindSSTableInternalObsoleteBit InternalKeyKind = 64
InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
// zero sequence number.
InternalKeyZeroSeqnumMaxTrailer = uint64(255)
// A marker for an invalid key.
InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
// which prevents those entries from being excluded from iteration.
InternalKeySeqNumBatch = uint64(1 << 55)
// InternalKeySeqNumMax is the largest valid sequence number.
InternalKeySeqNumMax = uint64(1<<56 - 1)
// InternalKeyRangeDeleteSentinel is the marker for a range delete sentinel
// key. This sequence number and kind are used for the upper stable boundary
// when a range deletion tombstone is the largest key in an sstable. This is
// necessary because sstable boundaries are inclusive, while the end key of a
// range deletion tombstone is exclusive.
InternalKeyRangeDeleteSentinel = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeDelete)
// InternalKeyBoundaryRangeKey is the marker for a range key boundary. This
// sequence number and kind are used during interleaved range key and point
// iteration to allow an iterator to stop at range key start keys where
// there exists no point key.
InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
)
// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
var internalKeyKindNames = []string{
InternalKeyKindDelete: "DEL",
InternalKeyKindSet: "SET",
InternalKeyKindMerge: "MERGE",
InternalKeyKindLogData: "LOGDATA",
InternalKeyKindSingleDelete: "SINGLEDEL",
InternalKeyKindRangeDelete: "RANGEDEL",
InternalKeyKindSeparator: "SEPARATOR",
InternalKeyKindSetWithDelete: "SETWITHDEL",
InternalKeyKindRangeKeySet: "RANGEKEYSET",
InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET",
InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
InternalKeyKindIngestSST: "INGESTSST",
InternalKeyKindDeleteSized: "DELSIZED",
InternalKeyKindInvalid: "INVALID",
}
func (k InternalKeyKind) String() string {
if int(k) < len(internalKeyKindNames) {
return internalKeyKindNames[k]
}
return fmt.Sprintf("UNKNOWN:%d", k)
}
// InternalKey is a key used for the in-memory and on-disk partial DBs that
// make up a pebble DB.
//
// It consists of the user key (as given by the code that uses package pebble)
// followed by 8-bytes of metadata:
// - 1 byte for the type of internal key: delete or set,
// - 7 bytes for a uint56 sequence number, in little-endian format.
type InternalKey struct {
UserKey []byte
Trailer uint64
}
// InvalidInternalKey is an invalid internal key for which Valid() will return
// false.
var InvalidInternalKey = MakeInternalKey(nil, 0, InternalKeyKindInvalid)
// MakeInternalKey constructs an internal key from a specified user key,
// sequence number and kind.
func MakeInternalKey(userKey []byte, seqNum uint64, kind InternalKeyKind) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (seqNum << 8) | uint64(kind),
}
}
// MakeTrailer constructs an internal key trailer from the specified sequence
// number and kind.
func MakeTrailer(seqNum uint64, kind InternalKeyKind) uint64 {
return (seqNum << 8) | uint64(kind)
}
// MakeSearchKey constructs an internal key that is appropriate for searching
// for a the specified user key. The search key contain the maximal sequence
// number and kind ensuring that it sorts before any other internal keys for
// the same user key.
func MakeSearchKey(userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindMax),
}
}
// MakeRangeDeleteSentinelKey constructs an internal key that is a range
// deletion sentinel key, used as the upper boundary for an sstable when a
// range deletion is the largest key in an sstable.
func MakeRangeDeleteSentinelKey(userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: InternalKeyRangeDeleteSentinel,
}
}
// MakeExclusiveSentinelKey constructs an internal key that is an
// exclusive sentinel key, used as the upper boundary for an sstable
// when a ranged key is the largest key in an sstable.
func MakeExclusiveSentinelKey(kind InternalKeyKind, userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (InternalKeySeqNumMax << 8) | uint64(kind),
}
}
var kindsMap = map[string]InternalKeyKind{
"DEL": InternalKeyKindDelete,
"SINGLEDEL": InternalKeyKindSingleDelete,
"RANGEDEL": InternalKeyKindRangeDelete,
"LOGDATA": InternalKeyKindLogData,
"SET": InternalKeyKindSet,
"MERGE": InternalKeyKindMerge,
"INVALID": InternalKeyKindInvalid,
"SEPARATOR": InternalKeyKindSeparator,
"SETWITHDEL": InternalKeyKindSetWithDelete,
"RANGEKEYSET": InternalKeyKindRangeKeySet,
"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
"RANGEKEYDEL": InternalKeyKindRangeKeyDelete,
"INGESTSST": InternalKeyKindIngestSST,
"DELSIZED": InternalKeyKindDeleteSized,
}
// ParseInternalKey parses the string representation of an internal key. The
// format is <user-key>.<kind>.<seq-num>. If the seq-num starts with a "b" it
// is marked as a batch-seq-num (i.e. the InternalKeySeqNumBatch bit is set).
func ParseInternalKey(s string) InternalKey {
x := strings.Split(s, ".")
ukey := x[0]
kind, ok := kindsMap[x[1]]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", x[1]))
}
j := 0
if x[2][0] == 'b' {
j = 1
}
seqNum, _ := strconv.ParseUint(x[2][j:], 10, 64)
if x[2][0] == 'b' {
seqNum |= InternalKeySeqNumBatch
}
return MakeInternalKey([]byte(ukey), seqNum, kind)
}
// ParseKind parses the string representation of an internal key kind.
func ParseKind(s string) InternalKeyKind {
kind, ok := kindsMap[s]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", s))
}
return kind
}
// InternalTrailerLen is the number of bytes used to encode InternalKey.Trailer.
const InternalTrailerLen = 8
// DecodeInternalKey decodes an encoded internal key. See InternalKey.Encode().
func DecodeInternalKey(encodedKey []byte) InternalKey {
n := len(encodedKey) - InternalTrailerLen
var trailer uint64
if n >= 0 {
trailer = binary.LittleEndian.Uint64(encodedKey[n:])
encodedKey = encodedKey[:n:n]
} else {
trailer = uint64(InternalKeyKindInvalid)
encodedKey = nil
}
return InternalKey{
UserKey: encodedKey,
Trailer: trailer,
}
}
// InternalCompare compares two internal keys using the specified comparison
// function. For equal user keys, internal keys compare in descending sequence
// number order. For equal user keys and sequence numbers, internal keys
// compare in descending kind order (this may happen in practice among range
// keys).
func InternalCompare(userCmp Compare, a, b InternalKey) int {
if x := userCmp(a.UserKey, b.UserKey); x != 0 {
return x
}
if a.Trailer > b.Trailer {
return -1
}
if a.Trailer < b.Trailer {
return 1
}
return 0
}
// Encode encodes the receiver into the buffer. The buffer must be large enough
// to hold the encoded data. See InternalKey.Size().
func (k InternalKey) Encode(buf []byte) {
i := copy(buf, k.UserKey)
binary.LittleEndian.PutUint64(buf[i:], k.Trailer)
}
// EncodeTrailer returns the trailer encoded to an 8-byte array.
func (k InternalKey) EncodeTrailer() [8]byte {
var buf [8]byte
binary.LittleEndian.PutUint64(buf[:], k.Trailer)
return buf
}
// Separator returns a separator key such that k <= x && x < other, where less
// than is consistent with the Compare function. The buf parameter may be used
// to store the returned InternalKey.UserKey, though it is valid to pass a
// nil. See the Separator type for details on separator keys.
func (k InternalKey) Separator(
cmp Compare, sep Separator, buf []byte, other InternalKey,
) InternalKey {
buf = sep(buf, k.UserKey, other.UserKey)
if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
// The separator user key is physically shorter than k.UserKey (if it is
// longer, we'll continue to use "k"), but logically after. Tack on the max
// sequence number to the shortened user key. Note that we could tack on
// any sequence number and kind here to create a valid separator key. We
// use the max sequence number to match the behavior of LevelDB and
// RocksDB.
return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
}
return k
}
// Successor returns a successor key such that k <= x. A simple implementation
// may return k unchanged. The buf parameter may be used to store the returned
// InternalKey.UserKey, though it is valid to pass a nil.
func (k InternalKey) Successor(cmp Compare, succ Successor, buf []byte) InternalKey {
buf = succ(buf, k.UserKey)
if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
// The successor user key is physically shorter that k.UserKey (if it is
// longer, we'll continue to use "k"), but logically after. Tack on the max
// sequence number to the shortened user key. Note that we could tack on
// any sequence number and kind here to create a valid separator key. We
// use the max sequence number to match the behavior of LevelDB and
// RocksDB.
return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
}
return k
}
// Size returns the encoded size of the key.
func (k InternalKey) Size() int {
return len(k.UserKey) + 8
}
// SetSeqNum sets the sequence number component of the key.
func (k *InternalKey) SetSeqNum(seqNum uint64) {
k.Trailer = (seqNum << 8) | (k.Trailer & 0xff)
}
// SeqNum returns the sequence number component of the key.
func (k InternalKey) SeqNum() uint64 {
return k.Trailer >> 8
}
// Visible returns true if the key is visible at the specified snapshot
// sequence number.
func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
return Visible(k.SeqNum(), snapshot, batchSnapshot)
}
// Visible returns true if a key with the provided sequence number is visible at
// the specified snapshot sequence numbers.
func Visible(seqNum uint64, snapshot, batchSnapshot uint64) bool {
// There are two snapshot sequence numbers, one for committed keys and one
// for batch keys. If a seqNum is less than `snapshot`, then seqNum
// corresponds to a committed key that is visible. If seqNum has its batch
// bit set, then seqNum corresponds to an uncommitted batch key. Its
// visible if its snapshot is less than batchSnapshot.
//
// There's one complication. The maximal sequence number
// (`InternalKeySeqNumMax`) is used across Pebble for exclusive sentinel
// keys and other purposes. The maximal sequence number has its batch bit
// set, but it can never be < `batchSnapshot`, since there is no expressible
// larger snapshot. We dictate that the maximal sequence number is always
// visible.
return seqNum < snapshot ||
((seqNum&InternalKeySeqNumBatch) != 0 && seqNum < batchSnapshot) ||
seqNum == InternalKeySeqNumMax
}
// SetKind sets the kind component of the key.
func (k *InternalKey) SetKind(kind InternalKeyKind) {
k.Trailer = (k.Trailer &^ 0xff) | uint64(kind)
}
// Kind returns the kind component of the key.
func (k InternalKey) Kind() InternalKeyKind {
return TrailerKind(k.Trailer)
}
// TrailerKind returns the key kind of the key trailer.
func TrailerKind(trailer uint64) InternalKeyKind {
return InternalKeyKind(trailer & 0xff)
}
// Valid returns true if the key has a valid kind.
func (k InternalKey) Valid() bool {
return k.Kind() <= InternalKeyKindMax
}
// Clone clones the storage for the UserKey component of the key.
func (k InternalKey) Clone() InternalKey {
if len(k.UserKey) == 0 {
return k
}
return InternalKey{
UserKey: append([]byte(nil), k.UserKey...),
Trailer: k.Trailer,
}
}
// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
// reusing any space already used for the current UserKey.
func (k *InternalKey) CopyFrom(k2 InternalKey) {
k.UserKey = append(k.UserKey[:0], k2.UserKey...)
k.Trailer = k2.Trailer
}
// String returns a string representation of the key.
func (k InternalKey) String() string {
return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())
}
// Pretty returns a formatter for the key.
func (k InternalKey) Pretty(f FormatKey) fmt.Formatter {
return prettyInternalKey{k, f}
}
// IsExclusiveSentinel returns whether this internal key excludes point keys
// with the same user key if used as an end boundary. See the comment on
// InternalKeyRangeDeletionSentinel.
func (k InternalKey) IsExclusiveSentinel() bool {
switch kind := k.Kind(); kind {
case InternalKeyKindRangeDelete:
return k.Trailer == InternalKeyRangeDeleteSentinel
case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
return (k.Trailer >> 8) == InternalKeySeqNumMax
default:
return false
}
}
type prettyInternalKey struct {
InternalKey
formatKey FormatKey
}
func (k prettyInternalKey) Format(s fmt.State, c rune) {
if seqNum := k.SeqNum(); seqNum == InternalKeySeqNumMax {
fmt.Fprintf(s, "%s#inf,%s", k.formatKey(k.UserKey), k.Kind())
} else {
fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
}
}
// ParsePrettyInternalKey parses the pretty string representation of an
// internal key. The format is <user-key>#<seq-num>,<kind>.
func ParsePrettyInternalKey(s string) InternalKey {
x := strings.FieldsFunc(s, func(c rune) bool { return c == '#' || c == ',' })
ukey := x[0]
kind, ok := kindsMap[x[2]]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", x[2]))
}
var seqNum uint64
if x[1] == "max" || x[1] == "inf" {
seqNum = InternalKeySeqNumMax
} else {
seqNum, _ = strconv.ParseUint(x[1], 10, 64)
}
return MakeInternalKey([]byte(ukey), seqNum, kind)
}

409
vendor/github.com/cockroachdb/pebble/internal/base/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,409 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"fmt"
"time"
)
// InternalIterator iterates over a DB's key/value pairs in key order. Unlike
// the Iterator interface, the returned keys are InternalKeys composed of the
// user-key, a sequence number and a key kind. In forward iteration, key/value
// pairs for identical user-keys are returned in descending sequence order. In
// reverse iteration, key/value pairs for identical user-keys are returned in
// ascending sequence order.
//
// InternalIterators provide 5 absolute positioning methods and 2 relative
// positioning methods. The absolute positioning methods are:
//
// - SeekGE
// - SeekPrefixGE
// - SeekLT
// - First
// - Last
//
// The relative positioning methods are:
//
// - Next
// - Prev
//
// The relative positioning methods can be used in conjunction with any of the
// absolute positioning methods with one exception: SeekPrefixGE does not
// support reverse iteration via Prev. It is undefined to call relative
// positioning methods without ever calling an absolute positioning method.
//
// InternalIterators can optionally implement a prefix iteration mode. This
// mode is entered by calling SeekPrefixGE and exited by any other absolute
// positioning method (SeekGE, SeekLT, First, Last). When in prefix iteration
// mode, a call to Next will advance to the next key which has the same
// "prefix" as the one supplied to SeekPrefixGE. Note that "prefix" in this
// context is not a strict byte prefix, but defined by byte equality for the
// result of the Comparer.Split method. An InternalIterator is not required to
// support prefix iteration mode, and can implement SeekPrefixGE by forwarding
// to SeekGE. When the iteration prefix is exhausted, it is not valid to call
// Next on an internal iterator that's already returned (nil,nilv) or a key
// beyond the prefix.
//
// Bounds, [lower, upper), can be set on iterators, either using the SetBounds()
// function in the interface, or in implementation specific ways during iterator
// creation. The forward positioning routines (SeekGE, First, and Next) only
// check the upper bound. The reverse positioning routines (SeekLT, Last, and
// Prev) only check the lower bound. It is up to the caller to ensure that the
// forward positioning routines respect the lower bound and the reverse
// positioning routines respect the upper bound (i.e. calling SeekGE instead of
// First if there is a lower bound, and SeekLT instead of Last if there is an
// upper bound). This imposition is done in order to elevate that enforcement to
// the caller (generally pebble.Iterator or pebble.mergingIter) rather than
// having it duplicated in every InternalIterator implementation.
//
// Additionally, the caller needs to ensure that SeekGE/SeekPrefixGE are not
// called with a key > the upper bound, and SeekLT is not called with a key <
// the lower bound. InternalIterator implementations are required to respect
// the iterator bounds, never returning records outside of the bounds with one
// exception: an iterator may generate synthetic RANGEDEL marker records. See
// levelIter.syntheticBoundary for the sole existing example of this behavior.
// Specifically, levelIter can return synthetic keys whose user key is equal to
// the lower/upper bound.
//
// The bounds provided to an internal iterator must remain valid until a
// subsequent call to SetBounds has returned. This requirement exists so that
// iterator implementations may compare old and new bounds to apply low-level
// optimizations. The pebble.Iterator satisfies this requirement by maintaining
// two bound buffers and switching between them.
//
// An iterator must be closed after use, but it is not necessary to read an
// iterator until exhaustion.
//
// An iterator is not goroutine-safe, but it is safe to use multiple iterators
// concurrently, either in separate goroutines or switching between the
// iterators in a single goroutine.
//
// It is also safe to use an iterator concurrently with modifying its
// underlying DB, if that DB permits modification. However, the resultant
// key/value pairs are not guaranteed to be a consistent snapshot of that DB
// at a particular point in time.
//
// InternalIterators accumulate errors encountered during operation, exposing
// them through the Error method. All of the absolute positioning methods
// reset any accumulated error before positioning. Relative positioning
// methods return without advancing if the iterator has accumulated an error.
//
// nilv == shorthand for LazyValue{}, which represents a nil value.
type InternalIterator interface {
// SeekGE moves the iterator to the first key/value pair whose key is greater
// than or equal to the given key. Returns the key and value if the iterator
// is pointing at a valid entry, and (nil, nilv) otherwise. Note that SeekGE
// only checks the upper bound. It is up to the caller to ensure that key
// is greater than or equal to the lower bound.
SeekGE(key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
// SeekPrefixGE moves the iterator to the first key/value pair whose key is
// greater than or equal to the given key. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nilv) otherwise. Note that
// SeekPrefixGE only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound.
//
// The prefix argument is used by some InternalIterator implementations (e.g.
// sstable.Reader) to avoid expensive operations. A user-defined Split
// function must be supplied to the Comparer for the DB. The supplied prefix
// will be the prefix of the given key returned by that Split function. If
// the iterator is able to determine that no key with the prefix exists, it
// can return (nil,nilv). Unlike SeekGE, this is not an indication that
// iteration is exhausted.
//
// Note that the iterator may return keys not matching the prefix. It is up
// to the caller to check if the prefix matches.
//
// Calling SeekPrefixGE places the receiver into prefix iteration mode. Once
// in this mode, reverse iteration may not be supported and will return an
// error. Note that pebble/Iterator.SeekPrefixGE has this same restriction on
// not supporting reverse iteration in prefix iteration mode until a
// different positioning routine (SeekGE, SeekLT, First or Last) switches the
// iterator out of prefix iteration.
SeekPrefixGE(prefix, key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
// SeekLT moves the iterator to the last key/value pair whose key is less
// than the given key. Returns the key and value if the iterator is pointing
// at a valid entry, and (nil, nilv) otherwise. Note that SeekLT only checks
// the lower bound. It is up to the caller to ensure that key is less than
// the upper bound.
SeekLT(key []byte, flags SeekLTFlags) (*InternalKey, LazyValue)
// First moves the iterator the the first key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that First only checks the upper bound. It is up to the
// caller to ensure that First() is not called when there is a lower bound,
// and instead call SeekGE(lower).
First() (*InternalKey, LazyValue)
// Last moves the iterator the the last key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that Last only checks the lower bound. It is up to the
// caller to ensure that Last() is not called when there is an upper bound,
// and instead call SeekLT(upper).
Last() (*InternalKey, LazyValue)
// Next moves the iterator to the next key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that Next only checks the upper bound. It is up to the
// caller to ensure that key is greater than or equal to the lower bound.
//
// It is valid to call Next when the iterator is positioned before the first
// key/value pair due to either a prior call to SeekLT or Prev which returned
// (nil, nilv). It is not allowed to call Next when the previous call to SeekGE,
// SeekPrefixGE or Next returned (nil, nilv).
Next() (*InternalKey, LazyValue)
// NextPrefix moves the iterator to the next key/value pair with a different
// prefix than the key at the current iterator position. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise. Note that NextPrefix only checks the upper bound. It is up to
// the caller to ensure that key is greater than or equal to the lower
// bound.
//
// NextPrefix is passed the immediate successor to the current prefix key. A
// valid implementation of NextPrefix is to call SeekGE with succKey.
//
// It is not allowed to call NextPrefix when the previous call was a reverse
// positioning operation or a call to a forward positioning method that
// returned (nil, nilv). It is also not allowed to call NextPrefix when the
// iterator is in prefix iteration mode.
NextPrefix(succKey []byte) (*InternalKey, LazyValue)
// Prev moves the iterator to the previous key/value pair. Returns the key
// and value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that Prev only checks the lower bound. It is up to the
// caller to ensure that key is less than the upper bound.
//
// It is valid to call Prev when the iterator is positioned after the last
// key/value pair due to either a prior call to SeekGE or Next which returned
// (nil, nilv). It is not allowed to call Prev when the previous call to SeekLT
// or Prev returned (nil, nilv).
Prev() (*InternalKey, LazyValue)
// Error returns any accumulated error. It may not include errors returned
// to the client when calling LazyValue.Value().
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
// all the key/value pairs in a table is not considered to be an error.
// It is valid to call Close multiple times. Other methods should not be
// called after the iterator has been closed.
Close() error
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
//
// The bounds provided must remain valid until a subsequent call to
// SetBounds has returned. This requirement exists so that iterator
// implementations may compare old and new bounds to apply low-level
// optimizations.
SetBounds(lower, upper []byte)
fmt.Stringer
}
// SeekGEFlags holds flags that may configure the behavior of a forward seek.
// Not all flags are relevant to all iterators.
type SeekGEFlags uint8
const (
seekGEFlagTrySeekUsingNext uint8 = iota
seekGEFlagRelativeSeek
seekGEFlagBatchJustRefreshed
)
// SeekGEFlagsNone is the default value of SeekGEFlags, with all flags disabled.
const SeekGEFlagsNone = SeekGEFlags(0)
// TrySeekUsingNext indicates whether a performance optimization was enabled
// by a caller, indicating the caller has not done any action to move this
// iterator beyond the first key that would be found if this iterator were to
// honestly do the intended seek. For example, say the caller did a
// SeekGE(k1...), followed by SeekGE(k2...) where k1 <= k2, without any
// intermediate positioning calls. The caller can safely specify true for this
// parameter in the second call. As another example, say the caller did do one
// call to Next between the two Seek calls, and k1 < k2. Again, the caller can
// safely specify a true value for this parameter. Note that a false value is
// always safe. The callee is free to ignore the true value if its
// implementation does not permit this optimization.
//
// We make the caller do this determination since a string comparison of k1, k2
// is not necessarily cheap, and there may be many iterators in the iterator
// stack. Doing it once at the root of the iterator stack is cheaper.
//
// This optimization could also be applied to SeekLT (where it would be
// trySeekUsingPrev). We currently only do it for SeekPrefixGE and SeekGE
// because this is where this optimization helps the performance of CockroachDB.
// The SeekLT cases in CockroachDB are typically accompanied with bounds that
// change between seek calls, and is optimized inside certain iterator
// implementations, like singleLevelIterator, without any extra parameter
// passing (though the same amortization of string comparisons could be done to
// improve that optimization, by making the root of the iterator stack do it).
func (s SeekGEFlags) TrySeekUsingNext() bool { return (s & (1 << seekGEFlagTrySeekUsingNext)) != 0 }
// RelativeSeek is set when in the course of a forward positioning operation, a
// higher-level iterator seeks a lower-level iterator to a larger key than the
// one at the current iterator position.
//
// Concretely, this occurs when the merging iterator observes a range deletion
// covering the key at a level's current position, and the merging iterator
// seeks the level to the range deletion's end key. During lazy-combined
// iteration, this flag signals to the level iterator that the seek is NOT an
// absolute-positioning operation from the perspective of the pebble.Iterator,
// and the level iterator must look for range keys in tables between the current
// iterator position and the new seeked position.
func (s SeekGEFlags) RelativeSeek() bool { return (s & (1 << seekGEFlagRelativeSeek)) != 0 }
// BatchJustRefreshed is set by Seek[Prefix]GE when an iterator's view of an
// indexed batch was just refreshed. It serves as a signal to the batch iterator
// to ignore the TrySeekUsingNext optimization, because the external knowledge
// imparted by the TrySeekUsingNext flag does not apply to the batch iterator's
// position. See (pebble.Iterator).batchJustRefreshed.
func (s SeekGEFlags) BatchJustRefreshed() bool { return (s & (1 << seekGEFlagBatchJustRefreshed)) != 0 }
// EnableTrySeekUsingNext returns the provided flags with the
// try-seek-using-next optimization enabled. See TrySeekUsingNext for an
// explanation of this optimization.
func (s SeekGEFlags) EnableTrySeekUsingNext() SeekGEFlags {
return s | (1 << seekGEFlagTrySeekUsingNext)
}
// DisableTrySeekUsingNext returns the provided flags with the
// try-seek-using-next optimization disabled.
func (s SeekGEFlags) DisableTrySeekUsingNext() SeekGEFlags {
return s &^ (1 << seekGEFlagTrySeekUsingNext)
}
// EnableRelativeSeek returns the provided flags with the relative-seek flag
// enabled. See RelativeSeek for an explanation of this flag's use.
func (s SeekGEFlags) EnableRelativeSeek() SeekGEFlags {
return s | (1 << seekGEFlagRelativeSeek)
}
// DisableRelativeSeek returns the provided flags with the relative-seek flag
// disabled.
func (s SeekGEFlags) DisableRelativeSeek() SeekGEFlags {
return s &^ (1 << seekGEFlagRelativeSeek)
}
// EnableBatchJustRefreshed returns the provided flags with the
// batch-just-refreshed bit set. See BatchJustRefreshed for an explanation of
// this flag.
func (s SeekGEFlags) EnableBatchJustRefreshed() SeekGEFlags {
return s | (1 << seekGEFlagBatchJustRefreshed)
}
// DisableBatchJustRefreshed returns the provided flags with the
// batch-just-refreshed bit unset.
func (s SeekGEFlags) DisableBatchJustRefreshed() SeekGEFlags {
return s &^ (1 << seekGEFlagBatchJustRefreshed)
}
// SeekLTFlags holds flags that may configure the behavior of a reverse seek.
// Not all flags are relevant to all iterators.
type SeekLTFlags uint8
const (
seekLTFlagRelativeSeek uint8 = iota
)
// SeekLTFlagsNone is the default value of SeekLTFlags, with all flags disabled.
const SeekLTFlagsNone = SeekLTFlags(0)
// RelativeSeek is set when in the course of a reverse positioning operation, a
// higher-level iterator seeks a lower-level iterator to a smaller key than the
// one at the current iterator position.
//
// Concretely, this occurs when the merging iterator observes a range deletion
// covering the key at a level's current position, and the merging iterator
// seeks the level to the range deletion's start key. During lazy-combined
// iteration, this flag signals to the level iterator that the seek is NOT an
// absolute-positioning operation from the perspective of the pebble.Iterator,
// and the level iterator must look for range keys in tables between the current
// iterator position and the new seeked position.
func (s SeekLTFlags) RelativeSeek() bool { return s&(1<<seekLTFlagRelativeSeek) != 0 }
// EnableRelativeSeek returns the provided flags with the relative-seek flag
// enabled. See RelativeSeek for an explanation of this flag's use.
func (s SeekLTFlags) EnableRelativeSeek() SeekLTFlags {
return s | (1 << seekLTFlagRelativeSeek)
}
// DisableRelativeSeek returns the provided flags with the relative-seek flag
// disabled.
func (s SeekLTFlags) DisableRelativeSeek() SeekLTFlags {
return s &^ (1 << seekLTFlagRelativeSeek)
}
// InternalIteratorStats contains miscellaneous stats produced by
// InternalIterators that are part of the InternalIterator tree. Not every
// field is relevant for an InternalIterator implementation. The field values
// are aggregated as one goes up the InternalIterator tree.
type InternalIteratorStats struct {
// Bytes in the loaded blocks. If the block was compressed, this is the
// compressed bytes. Currently, only the index blocks, data blocks
// containing points, and filter blocks are included.
BlockBytes uint64
// Subset of BlockBytes that were in the block cache.
BlockBytesInCache uint64
// BlockReadDuration accumulates the duration spent fetching blocks
// due to block cache misses.
// TODO(sumeer): this currently excludes the time spent in Reader creation,
// and in reading the rangedel and rangekey blocks. Fix that.
BlockReadDuration time.Duration
// The following can repeatedly count the same points if they are iterated
// over multiple times. Additionally, they may count a point twice when
// switching directions. The latter could be improved if needed.
// Bytes in keys that were iterated over. Currently, only point keys are
// included.
KeyBytes uint64
// Bytes in values that were iterated over. Currently, only point values are
// included. For separated values, this is the size of the handle.
ValueBytes uint64
// The count of points iterated over.
PointCount uint64
// Points that were iterated over that were covered by range tombstones. It
// can be useful for discovering instances of
// https://github.com/cockroachdb/pebble/issues/1070.
PointsCoveredByRangeTombstones uint64
// Stats related to points in value blocks encountered during iteration.
// These are useful to understand outliers, since typical user facing
// iteration should tend to only look at the latest point, and hence have
// the following stats close to 0.
SeparatedPointValue struct {
// Count is a count of points that were in value blocks. This is not a
// subset of PointCount: PointCount is produced by mergingIter and if
// positioned once, and successful in returning a point, will have a
// PointCount of 1, regardless of how many sstables (and memtables etc.)
// in the heap got positioned. The count here includes every sstable
// iterator that got positioned in the heap.
Count uint64
// ValueBytes represent the total byte length of the values (in value
// blocks) of the points corresponding to Count.
ValueBytes uint64
// ValueBytesFetched is the total byte length of the values (in value
// blocks) that were retrieved.
ValueBytesFetched uint64
}
}
// Merge merges the stats in from into the given stats.
func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
s.BlockBytes += from.BlockBytes
s.BlockBytesInCache += from.BlockBytesInCache
s.BlockReadDuration += from.BlockReadDuration
s.KeyBytes += from.KeyBytes
s.ValueBytes += from.ValueBytes
s.PointCount += from.PointCount
s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones
s.SeparatedPointValue.Count += from.SeparatedPointValue.Count
s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
}

287
vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go generated vendored Normal file
View File

@@ -0,0 +1,287 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/pebble/internal/invariants"
// A value can have user-defined attributes that are a function of the value
// byte slice. For now, we only support "short attributes", which can be
// encoded in 3 bits. We will likely extend this to "long attributes" later
// for values that are even more expensive to access than those in value
// blocks in the same sstable.
//
// When a sstable writer chooses not to store a value together with the key,
// it can call the ShortAttributeExtractor to extract the attribute and store
// it together with the key. This allows for cheap retrieval of
// AttributeAndLen on the read-path, without doing a more expensive retrieval
// of the value. In general, the extraction code may want to also look at the
// key to decide how to treat the value, hence the key* parameters.
//
// Write path performance: The ShortAttributeExtractor func cannot be inlined,
// so we will pay the cost of this function call. However, we will only pay
// this when (a) the value is not being stored together with the key, and (b)
// the key-value pair is being initially written to the DB, or a compaction is
// transitioning the key-value pair from being stored together to being stored
// separately.
// ShortAttribute encodes a user-specified attribute of the value.
type ShortAttribute uint8
// MaxShortAttribute is the maximum value of the short attribute (3 bits).
const MaxShortAttribute = 7
// ShortAttributeExtractor is an extractor that given the value, will return
// the ShortAttribute.
type ShortAttributeExtractor func(
key []byte, keyPrefixLen int, value []byte) (ShortAttribute, error)
// AttributeAndLen represents the pair of value length and the short
// attribute.
type AttributeAndLen struct {
ValueLen int32
ShortAttribute ShortAttribute
}
// LazyValue represents a value that may not already have been extracted.
// Currently, it can represent either an in-place value (stored with the key)
// or a value stored in the value section. However, the interface is general
// enough to support values that are stored in separate files.
//
// LazyValue is used in the InternalIterator interface, such that all
// positioning calls return (*InternalKey, LazyValue). It is also exposed via
// the public Iterator for callers that need to remember a recent but not
// necessarily latest LazyValue, in case they need the actual value in the
// future. An example is a caller that is iterating in reverse and looking for
// the latest MVCC version for a key -- it cannot identify the latest MVCC
// version without stepping to the previous key-value pair e.g.
// storage.pebbleMVCCScanner in CockroachDB.
//
// Performance note: It is important for this struct to not exceed a sizeof 32
// bytes, for optimizing the common case of the in-place value. Prior to
// introducing LazyValue, we were passing around a []byte which is 24 bytes.
// Passing a 40 byte or larger struct causes performance to drop by 75% on
// some benchmarks that do tight iteration loops.
//
// Memory management:
// This is subtle, but important for performance.
//
// A LazyValue returned by an InternalIterator or Iterator is unstable in that
// repositioning the iterator will invalidate the memory inside it. A caller
// wishing to maintain that LazyValue needs to call LazyValue.Clone(). Note
// that this does not fetch the value if it is not in-place. Clone() should
// ideally not be called if LazyValue.Value() has been called, since the
// cloned LazyValue will forget the extracted/fetched value, and calling
// Value() on this clone will cause the value to be extracted again. That is,
// Clone() does not make any promise about the memory stability of the
// underlying value.
//
// A user of an iterator that calls LazyValue.Value() wants as much as
// possible for the returned value []byte to point to iterator owned memory.
//
// 1. [P1] The underlying iterator that owns that memory also needs a promise
// from that user that at any time there is at most one value []byte slice
// that the caller is expecting it to maintain. Otherwise, the underlying
// iterator has to maintain multiple such []byte slices which results in
// more complicated and inefficient code.
//
// 2. [P2] The underlying iterator, in order to make the promise that it is
// maintaining the one value []byte slice, also needs a way to know when
// it is relieved of that promise. One way it is relieved of that promise
// is by being told that it is being repositioned. Typically, the owner of
// the value []byte slice is a sstable iterator, and it will know that it
// is relieved of the promise when it is repositioned. However, consider
// the case where the caller has used LazyValue.Clone() and repositioned
// the iterator (which is actually a tree of iterators). In this case the
// underlying sstable iterator may not even be open. LazyValue.Value()
// will still work (at a higher cost), but since the sstable iterator is
// not open, it does not have a mechanism to know when the retrieved value
// is no longer in use. We refer to this situation as "not satisfying P2".
// To handle this situation, the LazyValue.Value() method accepts a caller
// owned buffer, that the callee will use if needed. The callee explicitly
// tells the caller whether the []byte slice for the value is now owned by
// the caller. This will be true if the callee attempted to use buf and
// either successfully used it or allocated a new []byte slice.
//
// To ground the above in reality, we consider three examples of callers of
// LazyValue.Value():
//
// - Iterator: it calls LazyValue.Value for its own use when merging values.
// When merging during reverse iteration, it may have cloned the LazyValue.
// In this case it calls LazyValue.Value() on the cloned value, merges it,
// and then calls LazyValue.Value() on the current iterator position and
// merges it. So it is honoring P1.
//
// - Iterator on behalf of Iterator clients: The Iterator.Value() method
// needs to call LazyValue.Value(). The client of Iterator is satisfying P1
// because of the inherent Iterator interface constraint, i.e., it is calling
// Iterator.Value() on the current Iterator position. It is possible that
// the Iterator has cloned this LazyValue (for the reverse iteration case),
// which the client is unaware of, so the underlying sstable iterator may
// not be able to satisfy P2. This is ok because Iterator will call
// LazyValue.Value with its (reusable) owned buffer.
//
// - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
// since during reverse iteration in order to find the highest version that
// satisfies a read it needs to clone the LazyValue, step back the iterator
// and then decide whether it needs the value from the previously cloned
// LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
// similar to the previous case in that it will call LazyValue.Value with
// its (reusable) owned buffer.
//
// Corollary: callers that directly use InternalIterator can know that they
// have done nothing to interfere with promise P2 can pass in a nil buf and be
// sure that it will not trigger an allocation.
//
// Repeated calling of LazyValue.Value:
// This is ok as long as the caller continues to satisfy P1. The previously
// fetched value will be remembered inside LazyValue to avoid fetching again.
// So if the caller's buffer is used the first time the value was fetched, it
// is still in use.
//
// LazyValue fields are visible outside the package for use in
// InternalIterator implementations and in Iterator, but not meant for direct
// use by users of Pebble.
type LazyValue struct {
// ValueOrHandle represents a value, or a handle to be passed to ValueFetcher.
// - Fetcher == nil: ValueOrHandle is a value.
// - Fetcher != nil: ValueOrHandle is a handle and Fetcher.Attribute is
// initialized.
// The ValueOrHandle exposed by InternalIterator or Iterator may not be stable
// if the iterator is stepped. To make it stable, make a copy using Clone.
ValueOrHandle []byte
// Fetcher provides support for fetching an actually lazy value.
Fetcher *LazyFetcher
}
// LazyFetcher supports fetching a lazy value.
//
// Fetcher and Attribute are to be initialized at creation time. The fields
// are arranged to reduce the sizeof this struct.
type LazyFetcher struct {
// Fetcher, given a handle, returns the value.
Fetcher ValueFetcher
err error
value []byte
// Attribute includes the short attribute and value length.
Attribute AttributeAndLen
fetched bool
callerOwned bool
}
// ValueFetcher is an interface for fetching a value.
type ValueFetcher interface {
// Fetch returns the value, given the handle. It is acceptable to call the
// ValueFetcher.Fetch as long as the DB is open. However, one should assume
// there is a fast-path when the iterator tree has not moved off the sstable
// iterator that initially provided this LazyValue. Hence, to utilize this
// fast-path the caller should try to decide whether it needs the value or
// not as soon as possible, with minimal possible stepping of the iterator.
//
// buf will be used if the fetcher cannot satisfy P2 (see earlier comment).
// If the fetcher attempted to use buf *and* len(buf) was insufficient, it
// will allocate a new slice for the value. In either case it will set
// callerOwned to true.
Fetch(
handle []byte, valLen int32, buf []byte) (val []byte, callerOwned bool, err error)
}
// Value returns the underlying value.
func (lv *LazyValue) Value(buf []byte) (val []byte, callerOwned bool, err error) {
if lv.Fetcher == nil {
return lv.ValueOrHandle, false, nil
}
// Do the rest of the work in a separate method to attempt mid-stack
// inlining of Value(). Unfortunately, this still does not inline since the
// cost of 85 exceeds the budget of 80.
//
// TODO(sumeer): Packing the return values into a struct{[]byte error bool}
// causes it to be below the budget. Consider this if we need to recover
// more performance. I suspect that inlining this only matters in
// micro-benchmarks, and in actual use cases in CockroachDB it will not
// matter because there is substantial work done with a fetched value.
return lv.fetchValue(buf)
}
// INVARIANT: lv.Fetcher != nil
func (lv *LazyValue) fetchValue(buf []byte) (val []byte, callerOwned bool, err error) {
f := lv.Fetcher
if !f.fetched {
f.fetched = true
f.value, f.callerOwned, f.err = f.Fetcher.Fetch(
lv.ValueOrHandle, lv.Fetcher.Attribute.ValueLen, buf)
}
return f.value, f.callerOwned, f.err
}
// InPlaceValue returns the value under the assumption that it is in-place.
// This is for Pebble-internal code.
func (lv *LazyValue) InPlaceValue() []byte {
if invariants.Enabled && lv.Fetcher != nil {
panic("value must be in-place")
}
return lv.ValueOrHandle
}
// Len returns the length of the value.
func (lv *LazyValue) Len() int {
if lv.Fetcher == nil {
return len(lv.ValueOrHandle)
}
return int(lv.Fetcher.Attribute.ValueLen)
}
// TryGetShortAttribute returns the ShortAttribute and a bool indicating
// whether the ShortAttribute was populated.
func (lv *LazyValue) TryGetShortAttribute() (ShortAttribute, bool) {
if lv.Fetcher == nil {
return 0, false
}
return lv.Fetcher.Attribute.ShortAttribute, true
}
// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
// The fetcher parameter must be non-nil and may be over-written and used
// inside the returned LazyValue -- this is needed to avoid an allocation.
// Most callers have at most K cloned LazyValues, where K is hard-coded, so
// they can have a pool of exactly K LazyFetcher structs they can reuse in
// these calls. The alternative of allocating LazyFetchers from a sync.Pool is
// not viable since we have no code trigger for returning to the pool
// (LazyValues are simply GC'd).
//
// NB: It is highly preferable that LazyValue.Value() has not been called,
// since the Clone will forget any previously extracted value, and a future
// call to Value will cause it to be fetched again. We do this since we don't
// want to reason about whether or not to clone an already extracted value
// inside the Fetcher (we don't). Property P1 applies here too: if lv1.Value()
// has been called, and then lv2 is created as a clone of lv1, then calling
// lv2.Value() can invalidate any backing memory maintained inside the fetcher
// for lv1 (even though these are the same values). We initially prohibited
// calling LazyValue.Clone() if LazyValue.Value() has been called, but there
// is at least one complex caller (pebbleMVCCScanner inside CockroachDB) where
// it is not easy to prove this invariant.
func (lv *LazyValue) Clone(buf []byte, fetcher *LazyFetcher) (LazyValue, []byte) {
var lvCopy LazyValue
if lv.Fetcher != nil {
*fetcher = LazyFetcher{
Fetcher: lv.Fetcher.Fetcher,
Attribute: lv.Fetcher.Attribute,
// Not copying anything that has been extracted.
}
lvCopy.Fetcher = fetcher
}
vLen := len(lv.ValueOrHandle)
if vLen == 0 {
return lvCopy, buf
}
bufLen := len(buf)
buf = append(buf, lv.ValueOrHandle...)
lvCopy.ValueOrHandle = buf[bufLen : bufLen+vLen]
return lvCopy, buf
}
// MakeInPlaceValue constructs an in-place value.
func MakeInPlaceValue(val []byte) LazyValue {
return LazyValue{ValueOrHandle: val}
}

144
vendor/github.com/cockroachdb/pebble/internal/base/logger.go generated vendored Normal file
View File

@@ -0,0 +1,144 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"bytes"
"context"
"fmt"
"log"
"os"
"runtime"
"sync"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Logger defines an interface for writing log messages.
type Logger interface {
Infof(format string, args ...interface{})
Fatalf(format string, args ...interface{})
}
type defaultLogger struct{}
// DefaultLogger logs to the Go stdlib logs.
var DefaultLogger defaultLogger
var _ Logger = DefaultLogger
// Infof implements the Logger.Infof interface.
func (defaultLogger) Infof(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
}
// Fatalf implements the Logger.Fatalf interface.
func (defaultLogger) Fatalf(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
os.Exit(1)
}
// InMemLogger implements Logger using an in-memory buffer (used for testing).
// The buffer can be read via String() and cleared via Reset().
type InMemLogger struct {
mu struct {
sync.Mutex
buf bytes.Buffer
}
}
var _ Logger = (*InMemLogger)(nil)
// Reset clears the internal buffer.
func (b *InMemLogger) Reset() {
b.mu.Lock()
defer b.mu.Unlock()
b.mu.buf.Reset()
}
// String returns the current internal buffer.
func (b *InMemLogger) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.mu.buf.String()
}
// Infof is part of the Logger interface.
func (b *InMemLogger) Infof(format string, args ...interface{}) {
s := fmt.Sprintf(format, args...)
b.mu.Lock()
defer b.mu.Unlock()
b.mu.buf.Write([]byte(s))
if n := len(s); n == 0 || s[n-1] != '\n' {
b.mu.buf.Write([]byte("\n"))
}
}
// Fatalf is part of the Logger interface.
func (b *InMemLogger) Fatalf(format string, args ...interface{}) {
b.Infof(format, args...)
runtime.Goexit()
}
// LoggerAndTracer defines an interface for logging and tracing.
type LoggerAndTracer interface {
Logger
// Eventf formats and emits a tracing log, if tracing is enabled in the
// current context.
Eventf(ctx context.Context, format string, args ...interface{})
// IsTracingEnabled returns true if tracing is enabled. It can be used as an
// optimization to avoid calling Eventf (which will be a noop when tracing
// is not enabled) to avoid the overhead of boxing the args.
IsTracingEnabled(ctx context.Context) bool
}
// LoggerWithNoopTracer wraps a logger and does no tracing.
type LoggerWithNoopTracer struct {
Logger
}
var _ LoggerAndTracer = &LoggerWithNoopTracer{}
// Eventf implements LoggerAndTracer.
func (*LoggerWithNoopTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
if invariants.Enabled && ctx == nil {
panic("Eventf context is nil")
}
}
// IsTracingEnabled implements LoggerAndTracer.
func (*LoggerWithNoopTracer) IsTracingEnabled(ctx context.Context) bool {
if invariants.Enabled && ctx == nil {
panic("IsTracingEnabled ctx is nil")
}
return false
}
// NoopLoggerAndTracer does no logging and tracing. Remember that struct{} is
// special cased in Go and does not incur an allocation when it backs the
// interface LoggerAndTracer.
type NoopLoggerAndTracer struct{}
var _ LoggerAndTracer = NoopLoggerAndTracer{}
// Infof implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Infof(format string, args ...interface{}) {}
// Fatalf implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Fatalf(format string, args ...interface{}) {}
// Eventf implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
if invariants.Enabled && ctx == nil {
panic("Eventf context is nil")
}
}
// IsTracingEnabled implements LoggerAndTracer.
func (l NoopLoggerAndTracer) IsTracingEnabled(ctx context.Context) bool {
if invariants.Enabled && ctx == nil {
panic("IsTracingEnabled ctx is nil")
}
return false
}

133
vendor/github.com/cockroachdb/pebble/internal/base/merger.go generated vendored Normal file
View File

@@ -0,0 +1,133 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "io"
// Merge creates a ValueMerger for the specified key initialized with the value
// of one merge operand.
type Merge func(key, value []byte) (ValueMerger, error)
// ValueMerger receives merge operands one by one. The operand received is either
// newer or older than all operands received so far as indicated by the function
// names, `MergeNewer()` and `MergeOlder()`. Once all operands have been received,
// the client will invoke `Finish()` to obtain the final result. The order of
// a merge is not changed after the first call to `MergeNewer()` or
// `MergeOlder()`, i.e. the same method is used to submit all operands.
//
// The implementation may choose to merge values into the result immediately upon
// receiving each operand, or buffer operands until Finish() is called. For example,
// buffering may be useful to avoid (de)serializing partial merge results.
//
// The merge operation must be associative. That is, for the values A, B, C:
//
// Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
//
// Examples of merge operators are integer addition, list append, and string
// concatenation.
type ValueMerger interface {
// MergeNewer adds an operand that is newer than all existing operands.
// The caller retains ownership of value.
//
// If an error is returned the merge is aborted and no other methods must
// be called.
MergeNewer(value []byte) error
// MergeOlder adds an operand that is older than all existing operands.
// The caller retains ownership of value.
//
// If an error is returned the merge is aborted and no other methods must
// be called.
MergeOlder(value []byte) error
// Finish does any final processing of the added operands and returns a
// result. The caller can assume the returned byte slice will not be mutated.
//
// Finish must be the last function called on the ValueMerger. The caller
// must not call any other ValueMerger functions after calling Finish.
//
// If `includesBase` is true, the oldest merge operand was part of the
// merge. This will always be the true during normal iteration, but may be
// false during compaction when only a subset of operands may be
// available. Note that `includesBase` is set to true conservatively: a false
// value means that we could not definitely determine that the base merge
// operand was included.
//
// If a Closer is returned, the returned slice will remain valid until it is
// closed. The caller must arrange for the closer to be eventually closed.
Finish(includesBase bool) ([]byte, io.Closer, error)
}
// DeletableValueMerger is an extension to ValueMerger which allows indicating that the
// result of a merge operation is non-existent. Such non-existent entries will eventually
// be deleted during compaction. Note that during compaction, non-existence of the result
// of a merge means that the merge operands will not result in any record being output.
// This is not the same as transforming the merge operands into a deletion tombstone, as
// older merge operands will still be visible during iteration. Deletion of the merge operands
// in this way is akin to the way a SingleDelete+Set combine into non-existence while leaving
// older records for the same key unaffected.
type DeletableValueMerger interface {
ValueMerger
// DeletableFinish enables a value merger to indicate that the result of a merge operation
// is non-existent. See Finish for a description of includesBase.
DeletableFinish(includesBase bool) (value []byte, delete bool, closer io.Closer, err error)
}
// Merger defines an associative merge operation. The merge operation merges
// two or more values for a single key. A merge operation is requested by
// writing a value using {Batch,DB}.Merge(). The value at that key is merged
// with any existing value. It is valid to Set a value at a key and then Merge
// a new value. Similar to non-merged values, a merged value can be deleted by
// either Delete or DeleteRange.
//
// The merge operation is invoked when a merge value is encountered during a
// read, either during a compaction or during iteration.
type Merger struct {
Merge Merge
// Name is the name of the merger.
//
// Pebble stores the merger name on disk, and opening a database with a
// different merger from the one it was created with will result in an error.
Name string
}
// AppendValueMerger concatenates merge operands in order from oldest to newest.
type AppendValueMerger struct {
buf []byte
}
// MergeNewer appends value to the result.
func (a *AppendValueMerger) MergeNewer(value []byte) error {
a.buf = append(a.buf, value...)
return nil
}
// MergeOlder prepends value to the result, which involves allocating a new buffer.
func (a *AppendValueMerger) MergeOlder(value []byte) error {
buf := make([]byte, len(a.buf)+len(value))
copy(buf, value)
copy(buf[len(value):], a.buf)
a.buf = buf
return nil
}
// Finish returns the buffer that was constructed on-demand in `Merge{OlderNewer}()` calls.
func (a *AppendValueMerger) Finish(includesBase bool) ([]byte, io.Closer, error) {
return a.buf, nil, nil
}
// DefaultMerger is the default implementation of the Merger interface. It
// concatenates the two values to merge.
var DefaultMerger = &Merger{
Merge: func(key, value []byte) (ValueMerger, error) {
res := &AppendValueMerger{}
res.buf = append(res.buf, value...)
return res, nil
},
Name: "pebble.concatenate",
}

98
vendor/github.com/cockroachdb/pebble/internal/base/metrics.go generated vendored Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "time"
// ThroughputMetric is used to measure the byte throughput of some component
// that performs work in a single-threaded manner. The throughput can be
// approximated by Bytes/(WorkDuration+IdleTime). The idle time is represented
// separately, so that the user of this metric could approximate the peak
// throughput as Bytes/WorkTime. The metric is designed to be cumulative (see
// Merge).
type ThroughputMetric struct {
// Bytes is the processes bytes by the component.
Bytes int64
// WorkDuration is the duration that the component spent doing work.
WorkDuration time.Duration
// IdleDuration is the duration that the component was idling, waiting for
// work.
IdleDuration time.Duration
}
// Merge accumulates the information from another throughput metric.
func (tm *ThroughputMetric) Merge(x ThroughputMetric) {
tm.Bytes += x.Bytes
tm.WorkDuration += x.WorkDuration
tm.IdleDuration += x.IdleDuration
}
// Subtract subtracts the information from another ThroughputMetric
func (tm *ThroughputMetric) Subtract(x ThroughputMetric) {
tm.Bytes -= x.Bytes
tm.WorkDuration -= x.WorkDuration
tm.IdleDuration -= x.IdleDuration
}
// PeakRate returns the approximate peak rate if there was no idling.
func (tm *ThroughputMetric) PeakRate() int64 {
if tm.Bytes == 0 {
return 0
}
return int64((float64(tm.Bytes) / float64(tm.WorkDuration)) * float64(time.Second))
}
// Rate returns the observed rate.
func (tm *ThroughputMetric) Rate() int64 {
if tm.Bytes == 0 {
return 0
}
return int64((float64(tm.Bytes) / float64(tm.WorkDuration+tm.IdleDuration)) *
float64(time.Second))
}
// Utilization returns a percent [0, 1.0] indicating the percent of time
// work was performed.
func (tm *ThroughputMetric) Utilization() float64 {
if tm.WorkDuration == 0 {
return 0
}
return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
}
// GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
// accumulating samples of that gauge.
type GaugeSampleMetric struct {
// The sum of all the samples.
sampleSum int64
// The number of samples.
count int64
}
// AddSample adds the given sample.
func (gsm *GaugeSampleMetric) AddSample(sample int64) {
gsm.sampleSum += sample
gsm.count++
}
// Merge accumulates the information from another gauge metric.
func (gsm *GaugeSampleMetric) Merge(x GaugeSampleMetric) {
gsm.sampleSum += x.sampleSum
gsm.count += x.count
}
// Subtract subtracts the information from another gauge metric.
func (gsm *GaugeSampleMetric) Subtract(x GaugeSampleMetric) {
gsm.sampleSum -= x.sampleSum
gsm.count -= x.count
}
// Mean returns the mean value.
func (gsm *GaugeSampleMetric) Mean() float64 {
if gsm.count == 0 {
return 0
}
return float64(gsm.sampleSum) / float64(gsm.count)
}

76
vendor/github.com/cockroachdb/pebble/internal/base/options.go generated vendored Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
// SSTable block defaults.
const (
DefaultBlockRestartInterval = 16
DefaultBlockSize = 4096
DefaultBlockSizeThreshold = 90
)
// FilterType is the level at which to apply a filter: block or table.
type FilterType int
// The available filter types.
const (
TableFilter FilterType = iota
)
func (t FilterType) String() string {
switch t {
case TableFilter:
return "table"
}
return "unknown"
}
// FilterWriter provides an interface for creating filter blocks. See
// FilterPolicy for more details about filters.
type FilterWriter interface {
// AddKey adds a key to the current filter block.
AddKey(key []byte)
// Finish appends to dst an encoded filter tha holds the current set of
// keys. The writer state is reset after the call to Finish allowing the
// writer to be reused for the creation of additional filters.
Finish(dst []byte) []byte
}
// FilterPolicy is an algorithm for probabilistically encoding a set of keys.
// The canonical implementation is a Bloom filter.
//
// Every FilterPolicy has a name. This names the algorithm itself, not any one
// particular instance. Aspects specific to a particular instance, such as the
// set of keys or any other parameters, will be encoded in the []byte filter
// returned by NewWriter.
//
// The name may be written to files on disk, along with the filter data. To use
// these filters, the FilterPolicy name at the time of writing must equal the
// name at the time of reading. If they do not match, the filters will be
// ignored, which will not affect correctness but may affect performance.
type FilterPolicy interface {
// Name names the filter policy.
Name() string
// MayContain returns whether the encoded filter may contain given key.
// False positives are possible, where it returns true for keys not in the
// original set.
MayContain(ftype FilterType, filter, key []byte) bool
// NewWriter creates a new FilterWriter.
NewWriter(ftype FilterType) FilterWriter
}
// BlockPropertyFilter is used in an Iterator to filter sstables and blocks
// within the sstable. It should not maintain any per-sstable state, and must
// be thread-safe.
type BlockPropertyFilter interface {
// Name returns the name of the block property collector.
Name() string
// Intersects returns true if the set represented by prop intersects with
// the set in the filter.
Intersects(prop []byte) (bool, error)
}

56
vendor/github.com/cockroachdb/pebble/internal/batchskl/README.md generated vendored Normal file
View File

@@ -0,0 +1,56 @@
# batchskl
Fast, non-concurrent skiplist implementation in Go that supports
forward and backward iteration.
## Limitations
* The interface is tailored for use in indexing pebble batches. Keys
and values are stored outside of the skiplist making the skiplist
awkward for general purpose use.
* Deletion is not supported. Instead, higher-level code is expected to
add deletion tombstones and needs to process those tombstones
appropriately.
## Pedigree
This code is based on Andy Kimball's arenaskl code.
The arenaskl code is based on the skiplist found in Badger, a Go-based
KV store:
https://github.com/dgraph-io/badger/tree/master/skl
The skiplist in Badger is itself based on a C++ skiplist built for
Facebook's RocksDB:
https://github.com/facebook/rocksdb/tree/master/memtable
## Benchmarks
The benchmarks consist of a mix of reads and writes executed in parallel. The
fraction of reads is indicated in the run name: "frac_X" indicates a run where
X percent of the operations are reads.
```
name time/op
ReadWrite/frac_0 1.03µs ± 2%
ReadWrite/frac_10 1.32µs ± 1%
ReadWrite/frac_20 1.26µs ± 1%
ReadWrite/frac_30 1.18µs ± 1%
ReadWrite/frac_40 1.09µs ± 1%
ReadWrite/frac_50 987ns ± 2%
ReadWrite/frac_60 1.07µs ± 1%
ReadWrite/frac_70 909ns ± 1%
ReadWrite/frac_80 693ns ± 2%
ReadWrite/frac_90 599ns ± 2%
ReadWrite/frac_100 45.3ns ± 3%
```
Forward and backward iteration are also fast:
```
name time/op
IterNext 4.49ns ± 3%
IterPrev 4.48ns ± 3%
```

223
vendor/github.com/cockroachdb/pebble/internal/batchskl/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,223 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package batchskl
import "github.com/cockroachdb/pebble/internal/base"
type splice struct {
prev uint32
next uint32
}
// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
// to construct an iterator. The current state of the iterator can be cloned
// by simply value copying the struct.
type Iterator struct {
list *Skiplist
nd uint32
key base.InternalKey
lower []byte
upper []byte
}
// Close resets the iterator.
func (it *Iterator) Close() error {
it.list = nil
it.nd = 0
return nil
}
// SeekGE moves the iterator to the first entry whose key is greater than or
// equal to the given key. Returns true if the iterator is pointing at a valid
// entry and false otherwise. Note that SeekGE only checks the upper bound. It
// is up to the caller to ensure that key is greater than or equal to the lower
// bound.
func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) *base.InternalKey {
if flags.TrySeekUsingNext() {
if it.nd == it.list.tail {
// Iterator is done.
return nil
}
less := it.list.cmp(it.key.UserKey, key) < 0
// Arbitrary constant. By measuring the seek cost as a function of the
// number of elements in the skip list, and fitting to a model, we
// could adjust the number of nexts based on the current size of the
// skip list.
const numNexts = 5
for i := 0; less && i < numNexts; i++ {
k := it.Next()
if k == nil {
// Iterator is done.
return nil
}
less = it.list.cmp(k.UserKey, key) < 0
}
if !less {
return &it.key
}
}
_, it.nd = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// SeekLT moves the iterator to the last entry whose key is less the given
// key. Returns true if the iterator is pointing at a valid entry and false
// otherwise. Note that SeekLT only checks the lower bound. It is up to the
// caller to ensure that key is less than the upper bound.
func (it *Iterator) SeekLT(key []byte) *base.InternalKey {
it.nd, _ = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// First seeks position at the first entry in list. Final state of iterator is
// Valid() iff list is not empty. Note that First only checks the upper
// bound. It is up to the caller to ensure that key is greater than or equal to
// the lower bound (e.g. via a call to SeekGE(lower)).
func (it *Iterator) First() *base.InternalKey {
it.nd = it.list.getNext(it.list.head, 0)
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// Last seeks position at the last entry in list. Final state of iterator is
// Valid() iff list is not empty. Note that Last only checks the lower
// bound. It is up to the caller to ensure that key is less than the upper
// bound (e.g. via a call to SeekLT(upper)).
func (it *Iterator) Last() *base.InternalKey {
it.nd = it.list.getPrev(it.list.tail, 0)
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// Next advances to the next position. If there are no following nodes, then
// Valid() will be false after this call.
func (it *Iterator) Next() *base.InternalKey {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// Prev moves to the previous position. If there are no previous nodes, then
// Valid() will be false after this call.
func (it *Iterator) Prev() *base.InternalKey {
it.nd = it.list.getPrev(it.nd, 0)
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// Key returns the key at the current position.
func (it *Iterator) Key() *base.InternalKey {
return &it.key
}
// KeyInfo returns the offset of the start of the record, the start of the key,
// and the end of the key.
func (it *Iterator) KeyInfo() (offset, keyStart, keyEnd uint32) {
n := it.list.node(it.nd)
return n.offset, n.keyStart, n.keyEnd
}
// Head true iff the iterator is positioned at the sentinel head node.
func (it *Iterator) Head() bool {
return it.nd == it.list.head
}
// Tail true iff the iterator is positioned at the sentinel tail node.
func (it *Iterator) Tail() bool {
return it.nd == it.list.tail
}
// Valid returns nil iff the iterator is positioned at a valid node.
func (it *Iterator) Valid() bool {
return it.list != nil && it.nd != it.list.head && it.nd != it.list.tail
}
func (it *Iterator) String() string {
return "batch"
}
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekLT, First, or Last.
func (it *Iterator) SetBounds(lower, upper []byte) {
it.lower = lower
it.upper = upper
}
func (it *Iterator) seekForBaseSplice(key []byte, abbreviatedKey uint64) (prev, next uint32) {
prev = it.list.head
for level := it.list.height - 1; ; level-- {
prev, next = it.list.findSpliceForLevel(key, abbreviatedKey, level, prev)
if level == 0 {
break
}
}
return
}

442
vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go generated vendored Normal file
View File

@@ -0,0 +1,442 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Adapted from RocksDB inline skiplist.
Key differences:
- No optimization for sequential inserts (no "prev").
- No custom comparator.
- Support overwrites. This requires care when we see the same key when inserting.
For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
there is no need for values. We don't intend to support versioning. In-place updates of values
would be more efficient.
- We discard all non-concurrent code.
- We do not support Splices. This simplifies the code a lot.
- No AllocateNode or other pointer arithmetic.
- We combine the findLessThan, findGreaterOrEqual, etc into one function.
*/
/*
Further adapted from Badger: https://github.com/dgraph-io/badger.
Key differences:
- Support for previous pointers - doubly linked lists. Note that it's up to higher
level code to deal with the intermediate state that occurs during insertion,
where node A is linked to node B, but node B is not yet linked back to node A.
- Iterator includes mutator functions.
*/
/*
Further adapted from arenaskl: https://github.com/andy-kimball/arenaskl
Key differences:
- Removed support for deletion.
- Removed support for concurrency.
- External storage of keys.
- Node storage grows to an arbitrary size.
*/
package batchskl // import "github.com/cockroachdb/pebble/internal/batchskl"
import (
"bytes"
"encoding/binary"
"fmt"
"math"
"time"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/constants"
"golang.org/x/exp/rand"
)
const (
maxHeight = 20
maxNodeSize = uint64(unsafe.Sizeof(node{}))
linksSize = uint64(unsafe.Sizeof(links{}))
maxNodesSize = constants.MaxUint32OrInt
)
var (
// ErrExists indicates that a duplicate record was inserted. This should never
// happen for normal usage of batchskl as every key should have a unique
// sequence number.
ErrExists = errors.New("record with this key already exists")
// ErrTooManyRecords is a sentinel error returned when the size of the raw
// nodes slice exceeds the maximum allowed size (currently 1 << 32 - 1). This
// corresponds to ~117 M skiplist entries.
ErrTooManyRecords = errors.New("too many records")
)
type links struct {
next uint32
prev uint32
}
type node struct {
// The offset of the start of the record in the storage.
offset uint32
// The offset of the start and end of the key in storage.
keyStart uint32
keyEnd uint32
// A fixed 8-byte abbreviation of the key, used to avoid retrieval of the key
// during seek operations. The key retrieval can be expensive purely due to
// cache misses while the abbreviatedKey stored here will be in the same
// cache line as the key and the links making accessing and comparing against
// it almost free.
abbreviatedKey uint64
// Most nodes do not need to use the full height of the link tower, since the
// probability of each successive level decreases exponentially. Because
// these elements are never accessed, they do not need to be allocated.
// Therefore, when a node is allocated, its memory footprint is deliberately
// truncated to not include unneeded link elements.
links [maxHeight]links
}
// Skiplist is a fast, non-cocnurrent skiplist implementation that supports
// forward and backward iteration. See arenaskl.Skiplist for a concurrent
// skiplist. Keys and values are stored externally from the skiplist via the
// Storage interface. Deletion is not supported. Instead, higher-level code is
// expected to perform deletion via tombstones and needs to process those
// tombstones appropriately during retrieval operations.
type Skiplist struct {
storage *[]byte
cmp base.Compare
abbreviatedKey base.AbbreviatedKey
nodes []byte
head uint32
tail uint32
height uint32 // Current height: 1 <= height <= maxHeight
rand rand.PCGSource
}
var (
probabilities [maxHeight]uint32
)
func init() {
const pValue = 1 / math.E
// Precompute the skiplist probabilities so that only a single random number
// needs to be generated and so that the optimal pvalue can be used (inverse
// of Euler's number).
p := float64(1.0)
for i := 0; i < maxHeight; i++ {
probabilities[i] = uint32(float64(math.MaxUint32) * p)
p *= pValue
}
}
// NewSkiplist constructs and initializes a new, empty skiplist.
func NewSkiplist(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) *Skiplist {
s := &Skiplist{}
s.Init(storage, cmp, abbreviatedKey)
return s
}
// Reset the fields in the skiplist for reuse.
func (s *Skiplist) Reset() {
*s = Skiplist{
nodes: s.nodes[:0],
height: 1,
}
const batchMaxRetainedSize = 1 << 20 // 1 MB
if cap(s.nodes) > batchMaxRetainedSize {
s.nodes = nil
}
}
// Init the skiplist to empty and re-initialize.
func (s *Skiplist) Init(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) {
*s = Skiplist{
storage: storage,
cmp: cmp,
abbreviatedKey: abbreviatedKey,
nodes: s.nodes[:0],
height: 1,
}
s.rand.Seed(uint64(time.Now().UnixNano()))
const initBufSize = 256
if cap(s.nodes) < initBufSize {
s.nodes = make([]byte, 0, initBufSize)
}
// Allocate head and tail nodes. While allocating a new node can fail, in the
// context of initializing the skiplist we consider it unrecoverable.
var err error
s.head, err = s.newNode(maxHeight, 0, 0, 0, 0)
if err != nil {
panic(err)
}
s.tail, err = s.newNode(maxHeight, 0, 0, 0, 0)
if err != nil {
panic(err)
}
// Link all head/tail levels together.
headNode := s.node(s.head)
tailNode := s.node(s.tail)
for i := uint32(0); i < maxHeight; i++ {
headNode.links[i].next = s.tail
tailNode.links[i].prev = s.head
}
}
// Add adds a new key to the skiplist if it does not yet exist. If the record
// already exists, then Add returns ErrRecordExists.
func (s *Skiplist) Add(keyOffset uint32) error {
data := (*s.storage)[keyOffset+1:]
v, n := binary.Uvarint(data)
if n <= 0 {
return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
}
data = data[n:]
if v > uint64(len(data)) {
return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
}
keyStart := 1 + keyOffset + uint32(n)
keyEnd := keyStart + uint32(v)
key := data[:v]
abbreviatedKey := s.abbreviatedKey(key)
// spl holds the list of next and previous links for each level in the
// skiplist indicating where the new node will be inserted.
var spl [maxHeight]splice
// Fast-path for in-order insertion of keys: compare the new key against the
// last key.
prev := s.getPrev(s.tail, 0)
if prevNode := s.node(prev); prev == s.head ||
abbreviatedKey > prevNode.abbreviatedKey ||
(abbreviatedKey == prevNode.abbreviatedKey &&
s.cmp(key, (*s.storage)[prevNode.keyStart:prevNode.keyEnd]) > 0) {
for level := uint32(0); level < s.height; level++ {
spl[level].prev = s.getPrev(s.tail, level)
spl[level].next = s.tail
}
} else {
s.findSplice(key, abbreviatedKey, &spl)
}
height := s.randomHeight()
// Increase s.height as necessary.
for ; s.height < height; s.height++ {
spl[s.height].next = s.tail
spl[s.height].prev = s.head
}
// We always insert from the base level and up. After you add a node in base
// level, we cannot create a node in the level above because it would have
// discovered the node in the base level.
nd, err := s.newNode(height, keyOffset, keyStart, keyEnd, abbreviatedKey)
if err != nil {
return err
}
newNode := s.node(nd)
for level := uint32(0); level < height; level++ {
next := spl[level].next
prev := spl[level].prev
newNode.links[level].next = next
newNode.links[level].prev = prev
s.node(next).links[level].prev = nd
s.node(prev).links[level].next = nd
}
return nil
}
// NewIter returns a new Iterator object. The lower and upper bound parameters
// control the range of keys the iterator will return. Specifying for nil for
// lower or upper bound disables the check for that boundary. Note that lower
// bound is not checked on {SeekGE,First} and upper bound is not check on
// {SeekLT,Last}. The user is expected to perform that check. Note that it is
// safe for an iterator to be copied by value.
func (s *Skiplist) NewIter(lower, upper []byte) Iterator {
return Iterator{list: s, lower: lower, upper: upper}
}
func (s *Skiplist) newNode(
height,
offset, keyStart, keyEnd uint32, abbreviatedKey uint64,
) (uint32, error) {
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
unusedSize := uint64(maxHeight-int(height)) * linksSize
nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
if err != nil {
return 0, err
}
nd := s.node(nodeOffset)
nd.offset = offset
nd.keyStart = keyStart
nd.keyEnd = keyEnd
nd.abbreviatedKey = abbreviatedKey
return nodeOffset, nil
}
func (s *Skiplist) alloc(size uint32) (uint32, error) {
offset := uint64(len(s.nodes))
// We only have a need for memory up to offset + size, but we never want
// to allocate a node whose tail points into unallocated memory.
minAllocSize := offset + maxNodeSize
if uint64(cap(s.nodes)) < minAllocSize {
allocSize := uint64(cap(s.nodes)) * 2
if allocSize < minAllocSize {
allocSize = minAllocSize
}
// Cap the allocation at the max allowed size to avoid wasted capacity.
if allocSize > maxNodesSize {
// The new record may still not fit within the allocation, in which case
// we return early with an error. This avoids the panic below when we
// resize the slice. It also avoids the allocation and copy.
if uint64(offset)+uint64(size) > maxNodesSize {
return 0, errors.Wrapf(ErrTooManyRecords,
"alloc of new record (size=%d) would overflow uint32 (current size=%d)",
uint64(offset)+uint64(size), offset,
)
}
allocSize = maxNodesSize
}
tmp := make([]byte, len(s.nodes), allocSize)
copy(tmp, s.nodes)
s.nodes = tmp
}
newSize := uint32(offset) + size
s.nodes = s.nodes[:newSize]
return uint32(offset), nil
}
func (s *Skiplist) node(offset uint32) *node {
return (*node)(unsafe.Pointer(&s.nodes[offset]))
}
func (s *Skiplist) randomHeight() uint32 {
rnd := uint32(s.rand.Uint64())
h := uint32(1)
for h < maxHeight && rnd <= probabilities[h] {
h++
}
return h
}
func (s *Skiplist) findSplice(key []byte, abbreviatedKey uint64, spl *[maxHeight]splice) {
prev := s.head
for level := s.height - 1; ; level-- {
// The code in this loop is the same as findSpliceForLevel(). For some
// reason, calling findSpliceForLevel() here is much much slower than the
// inlined code below. The excess time is also caught up in the final
// return statement which makes little sense. Revisit when in go1.14 or
// later if inlining improves.
next := s.getNext(prev, level)
for next != s.tail {
// Assume prev.key < key.
nextNode := s.node(next)
nextAbbreviatedKey := nextNode.abbreviatedKey
if abbreviatedKey < nextAbbreviatedKey {
// We are done for this level, since prev.key < key < next.key.
break
}
if abbreviatedKey == nextAbbreviatedKey {
if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
// We are done for this level, since prev.key < key <= next.key.
break
}
}
// Keep moving right on this level.
prev = next
next = nextNode.links[level].next
}
spl[level].prev = prev
spl[level].next = next
if level == 0 {
break
}
}
}
func (s *Skiplist) findSpliceForLevel(
key []byte, abbreviatedKey uint64, level, start uint32,
) (prev, next uint32) {
prev = start
next = s.getNext(prev, level)
for next != s.tail {
// Assume prev.key < key.
nextNode := s.node(next)
nextAbbreviatedKey := nextNode.abbreviatedKey
if abbreviatedKey < nextAbbreviatedKey {
// We are done for this level, since prev.key < key < next.key.
break
}
if abbreviatedKey == nextAbbreviatedKey {
if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
}
// Keep moving right on this level.
prev = next
next = nextNode.links[level].next
}
return
}
func (s *Skiplist) getKey(nd uint32) base.InternalKey {
n := s.node(nd)
kind := base.InternalKeyKind((*s.storage)[n.offset])
key := (*s.storage)[n.keyStart:n.keyEnd]
return base.MakeInternalKey(key, uint64(n.offset)|base.InternalKeySeqNumBatch, kind)
}
func (s *Skiplist) getNext(nd, h uint32) uint32 {
return s.node(nd).links[h].next
}
func (s *Skiplist) getPrev(nd, h uint32) uint32 {
return s.node(nd).links[h].prev
}
func (s *Skiplist) debug() string {
var buf bytes.Buffer
for level := uint32(0); level < s.height; level++ {
var count int
for nd := s.head; nd != s.tail; nd = s.getNext(nd, level) {
count++
}
fmt.Fprintf(&buf, "%d: %d\n", level, count)
}
return buf.String()
}
// Silence unused warning.
var _ = (*Skiplist).debug

69
vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go generated vendored Normal file
View File

@@ -0,0 +1,69 @@
// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package bytealloc
import "github.com/cockroachdb/pebble/internal/rawalloc"
// An A provides chunk allocation of []byte, amortizing the overhead of each
// allocation. Because the underlying storage for the slices is shared, they
// should share a similar lifetime in order to avoid pinning large amounts of
// memory unnecessarily. The allocator itself is a []byte where cap() indicates
// the total amount of memory and len() is the amount already allocated. The
// size of the buffer to allocate from is grown exponentially when it runs out
// of room up to a maximum size (chunkAllocMaxSize).
type A []byte
const chunkAllocMinSize = 512
const chunkAllocMaxSize = 512 << 10 // 512 KB
func (a A) reserve(n int) A {
allocSize := cap(a) * 2
if allocSize < chunkAllocMinSize {
allocSize = chunkAllocMinSize
} else if allocSize > chunkAllocMaxSize {
allocSize = chunkAllocMaxSize
}
if allocSize < n {
allocSize = n
}
return rawalloc.New(0, allocSize)
}
// Alloc allocates a new chunk of memory with the specified length.
func (a A) Alloc(n int) (A, []byte) {
if cap(a)-len(a) < n {
a = a.reserve(n)
}
p := len(a)
r := a[p : p+n : p+n]
a = a[:p+n]
return a, r
}
// Copy allocates a new chunk of memory, initializing it from src.
func (a A) Copy(src []byte) (A, []byte) {
var alloc []byte
a, alloc = a.Alloc(len(src))
copy(alloc, src)
return a, alloc
}
// Reset returns the current chunk, resetting allocated memory back to none.
// Future allocations will use memory previously allocated by previous calls to
// Alloc or Copy, so the caller must know know that none of the previously
// allocated byte slices are still in use.
func (a A) Reset() A {
return a[:0]
}

21
vendor/github.com/cockroachdb/pebble/internal/cache/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
The MIT License
Copyright (c) 2018 Damian Gryski
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

10
vendor/github.com/cockroachdb/pebble/internal/cache/cgo_disabled.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !cgo
// +build !cgo
package cache
const cgoEnabled = false

10
vendor/github.com/cockroachdb/pebble/internal/cache/cgo_enabled.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build cgo
// +build cgo
package cache
const cgoEnabled = true

909
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go generated vendored Normal file
View File

@@ -0,0 +1,909 @@
// Copyright 2018. All rights reserved. Use of this source code is governed by
// an MIT-style license that can be found in the LICENSE file.
// Package cache implements the CLOCK-Pro caching algorithm.
//
// CLOCK-Pro is a patent-free alternative to the Adaptive Replacement Cache,
// https://en.wikipedia.org/wiki/Adaptive_replacement_cache.
// It is an approximation of LIRS ( https://en.wikipedia.org/wiki/LIRS_caching_algorithm ),
// much like the CLOCK page replacement algorithm is an approximation of LRU.
//
// This implementation is based on the python code from https://bitbucket.org/SamiLehtinen/pyclockpro .
//
// Slides describing the algorithm: http://fr.slideshare.net/huliang64/clockpro
//
// The original paper: http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html
//
// It is MIT licensed, like the original.
package cache // import "github.com/cockroachdb/pebble/internal/cache"
import (
"fmt"
"os"
"runtime"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
type fileKey struct {
// id is the namespace for fileNums.
id uint64
fileNum base.DiskFileNum
}
type key struct {
fileKey
offset uint64
}
// file returns the "file key" for the receiver. This is the key used for the
// shard.files map.
func (k key) file() key {
k.offset = 0
return k
}
func (k key) String() string {
return fmt.Sprintf("%d/%d/%d", k.id, k.fileNum, k.offset)
}
// Handle provides a strong reference to a value in the cache. The reference
// does not pin the value in the cache, but it does prevent the underlying byte
// slice from being reused.
type Handle struct {
value *Value
}
// Get returns the value stored in handle.
func (h Handle) Get() []byte {
if h.value != nil {
// NB: We don't increment shard.hits in this code path because we only want
// to record a hit when the handle is retrieved from the cache.
return h.value.buf
}
return nil
}
// Release releases the reference to the cache entry.
func (h Handle) Release() {
h.value.release()
}
type shard struct {
hits atomic.Int64
misses atomic.Int64
mu sync.RWMutex
reservedSize int64
maxSize int64
coldTarget int64
blocks robinHoodMap // fileNum+offset -> block
files robinHoodMap // fileNum -> list of blocks
// The blocks and files maps store values in manually managed memory that is
// invisible to the Go GC. This is fine for Value and entry objects that are
// stored in manually managed memory, but when the "invariants" build tag is
// set, all Value and entry objects are Go allocated and the entries map will
// contain a reference to every entry.
entries map[*entry]struct{}
handHot *entry
handCold *entry
handTest *entry
sizeHot int64
sizeCold int64
sizeTest int64
// The count fields are used exclusively for asserting expectations.
// We've seen infinite looping (cockroachdb/cockroach#70154) that
// could be explained by a corrupted sizeCold. Through asserting on
// these fields, we hope to gain more insight from any future
// reproductions.
countHot int64
countCold int64
countTest int64
}
func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
c.mu.RLock()
var value *Value
if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
value = e.acquireValue()
if value != nil {
e.referenced.Store(true)
}
}
c.mu.RUnlock()
if value == nil {
c.misses.Add(1)
return Handle{}
}
c.hits.Add(1)
return Handle{value: value}
}
func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
if n := value.refs(); n != 1 {
panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
}
c.mu.Lock()
defer c.mu.Unlock()
k := key{fileKey{id, fileNum}, offset}
e := c.blocks.Get(k)
switch {
case e == nil:
// no cache entry? add it
e = newEntry(c, k, int64(len(value.buf)))
e.setValue(value)
if c.metaAdd(k, e) {
value.ref.trace("add-cold")
c.sizeCold += e.size
c.countCold++
} else {
value.ref.trace("skip-cold")
e.free()
e = nil
}
case e.peekValue() != nil:
// cache entry was a hot or cold page
e.setValue(value)
e.referenced.Store(true)
delta := int64(len(value.buf)) - e.size
e.size = int64(len(value.buf))
if e.ptype == etHot {
value.ref.trace("add-hot")
c.sizeHot += delta
} else {
value.ref.trace("add-cold")
c.sizeCold += delta
}
c.evict()
default:
// cache entry was a test page
c.sizeTest -= e.size
c.countTest--
c.metaDel(e).release()
c.metaCheck(e)
e.size = int64(len(value.buf))
c.coldTarget += e.size
if c.coldTarget > c.targetSize() {
c.coldTarget = c.targetSize()
}
e.referenced.Store(false)
e.setValue(value)
e.ptype = etHot
if c.metaAdd(k, e) {
value.ref.trace("add-hot")
c.sizeHot += e.size
c.countHot++
} else {
value.ref.trace("skip-hot")
e.free()
e = nil
}
}
c.checkConsistency()
// Values are initialized with a reference count of 1. That reference count
// is being transferred to the returned Handle.
return Handle{value: value}
}
func (c *shard) checkConsistency() {
// See the comment above the count{Hot,Cold,Test} fields.
switch {
case c.sizeHot < 0 || c.sizeCold < 0 || c.sizeTest < 0 || c.countHot < 0 || c.countCold < 0 || c.countTest < 0:
panic(fmt.Sprintf("pebble: unexpected negative: %d (%d bytes) hot, %d (%d bytes) cold, %d (%d bytes) test",
c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest))
case c.sizeHot > 0 && c.countHot == 0:
panic(fmt.Sprintf("pebble: mismatch %d hot size, %d hot count", c.sizeHot, c.countHot))
case c.sizeCold > 0 && c.countCold == 0:
panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
case c.sizeTest > 0 && c.countTest == 0:
panic(fmt.Sprintf("pebble: mismatch %d test size, %d test count", c.sizeTest, c.countTest))
}
}
// Delete deletes the cached value for the specified file and offset.
func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
// The common case is there is nothing to delete, so do a quick check with
// shared lock.
k := key{fileKey{id, fileNum}, offset}
c.mu.RLock()
exists := c.blocks.Get(k) != nil
c.mu.RUnlock()
if !exists {
return
}
var deletedValue *Value
func() {
c.mu.Lock()
defer c.mu.Unlock()
e := c.blocks.Get(k)
if e == nil {
return
}
deletedValue = c.metaEvict(e)
c.checkConsistency()
}()
// Now that the mutex has been dropped, release the reference which will
// potentially free the memory associated with the previous cached value.
deletedValue.release()
}
// EvictFile evicts all of the cache values for the specified file.
func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
fkey := key{fileKey{id, fileNum}, 0}
for c.evictFileRun(fkey) {
// Sched switch to give another goroutine an opportunity to acquire the
// shard mutex.
runtime.Gosched()
}
}
func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
// If most of the file's blocks are held in the block cache, evicting all
// the blocks may take a while. We don't want to block the entire cache
// shard, forcing concurrent readers to wait until we're finished. We drop
// the mutex every [blocksPerMutexAcquisition] blocks to give other
// goroutines an opportunity to make progress.
const blocksPerMutexAcquisition = 5
c.mu.Lock()
// Releasing a value may result in free-ing it back to the memory allocator.
// This can have a nontrivial cost that we'd prefer to not pay while holding
// the shard mutex, so we collect the evicted values in a local slice and
// only release them in a defer after dropping the cache mutex.
var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
obsoleteValues := obsoleteValuesAlloc[:0]
defer func() {
c.mu.Unlock()
for _, v := range obsoleteValues {
v.release()
}
}()
blocks := c.files.Get(fkey)
if blocks == nil {
// No blocks for this file.
return false
}
// b is the current head of the doubly linked list, and n is the entry after b.
for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
n = b.fileLink.next
obsoleteValues = append(obsoleteValues, c.metaEvict(b))
if b == n {
// b == n represents the case where b was the last entry remaining
// in the doubly linked list, which is why it pointed at itself. So
// no more entries left.
c.checkConsistency()
return false
}
}
// Exhausted blocksPerMutexAcquisition.
return true
}
func (c *shard) Free() {
c.mu.Lock()
defer c.mu.Unlock()
// NB: we use metaDel rather than metaEvict in order to avoid the expensive
// metaCheck call when the "invariants" build tag is specified.
for c.handHot != nil {
e := c.handHot
c.metaDel(c.handHot).release()
e.free()
}
c.blocks.free()
c.files.free()
}
func (c *shard) Reserve(n int) {
c.mu.Lock()
defer c.mu.Unlock()
c.reservedSize += int64(n)
// Changing c.reservedSize will either increase or decrease
// the targetSize. But we want coldTarget to be in the range
// [0, targetSize]. So, if c.targetSize decreases, make sure
// that the coldTarget fits within the limits.
targetSize := c.targetSize()
if c.coldTarget > targetSize {
c.coldTarget = targetSize
}
c.evict()
c.checkConsistency()
}
// Size returns the current space used by the cache.
func (c *shard) Size() int64 {
c.mu.RLock()
size := c.sizeHot + c.sizeCold
c.mu.RUnlock()
return size
}
func (c *shard) targetSize() int64 {
target := c.maxSize - c.reservedSize
// Always return a positive integer for targetSize. This is so that we don't
// end up in an infinite loop in evict(), in cases where reservedSize is
// greater than or equal to maxSize.
if target < 1 {
return 1
}
return target
}
// Add the entry to the cache, returning true if the entry was added and false
// if it would not fit in the cache.
func (c *shard) metaAdd(key key, e *entry) bool {
c.evict()
if e.size > c.targetSize() {
// The entry is larger than the target cache size.
return false
}
c.blocks.Put(key, e)
if entriesGoAllocated {
// Go allocated entries need to be referenced from Go memory. The entries
// map provides that reference.
c.entries[e] = struct{}{}
}
if c.handHot == nil {
// first element
c.handHot = e
c.handCold = e
c.handTest = e
} else {
c.handHot.link(e)
}
if c.handCold == c.handHot {
c.handCold = c.handCold.prev()
}
fkey := key.file()
if fileBlocks := c.files.Get(fkey); fileBlocks == nil {
c.files.Put(fkey, e)
} else {
fileBlocks.linkFile(e)
}
return true
}
// Remove the entry from the cache. This removes the entry from the blocks map,
// the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
// entry. Returns the deleted value that must be released, if any.
func (c *shard) metaDel(e *entry) (deletedValue *Value) {
if value := e.peekValue(); value != nil {
value.ref.trace("metaDel")
}
// Remove the pointer to the value.
deletedValue = e.val
e.val = nil
c.blocks.Delete(e.key)
if entriesGoAllocated {
// Go allocated entries need to be referenced from Go memory. The entries
// map provides that reference.
delete(c.entries, e)
}
if e == c.handHot {
c.handHot = c.handHot.prev()
}
if e == c.handCold {
c.handCold = c.handCold.prev()
}
if e == c.handTest {
c.handTest = c.handTest.prev()
}
if e.unlink() == e {
// This was the last entry in the cache.
c.handHot = nil
c.handCold = nil
c.handTest = nil
}
fkey := e.key.file()
if next := e.unlinkFile(); e == next {
c.files.Delete(fkey)
} else {
c.files.Put(fkey, next)
}
return deletedValue
}
// Check that the specified entry is not referenced by the cache.
func (c *shard) metaCheck(e *entry) {
if invariants.Enabled {
if _, ok := c.entries[e]; ok {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in entries map\n%s",
e, e.key, debug.Stack())
os.Exit(1)
}
if c.blocks.findByValue(e) != nil {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks map\n%s\n%s",
e, e.key, &c.blocks, debug.Stack())
os.Exit(1)
}
if c.files.findByValue(e) != nil {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in files map\n%s\n%s",
e, e.key, &c.files, debug.Stack())
os.Exit(1)
}
// NB: c.hand{Hot,Cold,Test} are pointers into a single linked list. We
// only have to traverse one of them to check all of them.
var countHot, countCold, countTest int64
var sizeHot, sizeCold, sizeTest int64
for t := c.handHot.next(); t != nil; t = t.next() {
// Recompute count{Hot,Cold,Test} and size{Hot,Cold,Test}.
switch t.ptype {
case etHot:
countHot++
sizeHot += t.size
case etCold:
countCold++
sizeCold += t.size
case etTest:
countTest++
sizeTest += t.size
}
if e == t {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks list\n%s",
e, e.key, debug.Stack())
os.Exit(1)
}
if t == c.handHot {
break
}
}
if countHot != c.countHot || countCold != c.countCold || countTest != c.countTest ||
sizeHot != c.sizeHot || sizeCold != c.sizeCold || sizeTest != c.sizeTest {
fmt.Fprintf(os.Stderr, `divergence of Hot,Cold,Test statistics
cache's statistics: hot %d, %d, cold %d, %d, test %d, %d
recalculated statistics: hot %d, %d, cold %d, %d, test %d, %d\n%s`,
c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest,
countHot, sizeHot, countCold, sizeCold, countTest, sizeTest,
debug.Stack())
os.Exit(1)
}
}
}
func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
switch e.ptype {
case etHot:
c.sizeHot -= e.size
c.countHot--
case etCold:
c.sizeCold -= e.size
c.countCold--
case etTest:
c.sizeTest -= e.size
c.countTest--
}
evictedValue = c.metaDel(e)
c.metaCheck(e)
e.free()
return evictedValue
}
func (c *shard) evict() {
for c.targetSize() <= c.sizeHot+c.sizeCold && c.handCold != nil {
c.runHandCold(c.countCold, c.sizeCold)
}
}
func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
// countColdDebug and sizeColdDebug should equal c.countCold and
// c.sizeCold. They're parameters only to aid in debugging of
// cockroachdb/cockroach#70154. Since they're parameters, their
// arguments will appear within stack traces should we encounter
// a reproduction.
if c.countCold != countColdDebug || c.sizeCold != sizeColdDebug {
panic(fmt.Sprintf("runHandCold: cold count and size are %d, %d, arguments are %d and %d",
c.countCold, c.sizeCold, countColdDebug, sizeColdDebug))
}
e := c.handCold
if e.ptype == etCold {
if e.referenced.Load() {
e.referenced.Store(false)
e.ptype = etHot
c.sizeCold -= e.size
c.countCold--
c.sizeHot += e.size
c.countHot++
} else {
e.setValue(nil)
e.ptype = etTest
c.sizeCold -= e.size
c.countCold--
c.sizeTest += e.size
c.countTest++
for c.targetSize() < c.sizeTest && c.handTest != nil {
c.runHandTest()
}
}
}
c.handCold = c.handCold.next()
for c.targetSize()-c.coldTarget <= c.sizeHot && c.handHot != nil {
c.runHandHot()
}
}
func (c *shard) runHandHot() {
if c.handHot == c.handTest && c.handTest != nil {
c.runHandTest()
if c.handHot == nil {
return
}
}
e := c.handHot
if e.ptype == etHot {
if e.referenced.Load() {
e.referenced.Store(false)
} else {
e.ptype = etCold
c.sizeHot -= e.size
c.countHot--
c.sizeCold += e.size
c.countCold++
}
}
c.handHot = c.handHot.next()
}
func (c *shard) runHandTest() {
if c.sizeCold > 0 && c.handTest == c.handCold && c.handCold != nil {
// sizeCold is > 0, so assert that countCold == 0. See the
// comment above count{Hot,Cold,Test}.
if c.countCold == 0 {
panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
}
c.runHandCold(c.countCold, c.sizeCold)
if c.handTest == nil {
return
}
}
e := c.handTest
if e.ptype == etTest {
c.sizeTest -= e.size
c.countTest--
c.coldTarget -= e.size
if c.coldTarget < 0 {
c.coldTarget = 0
}
c.metaDel(e).release()
c.metaCheck(e)
e.free()
}
c.handTest = c.handTest.next()
}
// Metrics holds metrics for the cache.
type Metrics struct {
// The number of bytes inuse by the cache.
Size int64
// The count of objects (blocks or tables) in the cache.
Count int64
// The number of cache hits.
Hits int64
// The number of cache misses.
Misses int64
}
// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
// used for page replacement
// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
// order to provide better concurrency, 4 x NumCPUs shards are created, with
// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
// is run independently on each shard.
//
// Blocks are keyed by an (id, fileNum, offset) triple. The ID is a namespace
// for file numbers and allows a single Cache to be shared between multiple
// Pebble instances. The fileNum and offset refer to an sstable file number and
// the offset of the block within the file. Because sstables are immutable and
// file numbers are never reused, (fileNum,offset) are unique for the lifetime
// of a Pebble instance.
//
// In addition to maintaining a map from (fileNum,offset) to data, each shard
// maintains a map of the cached blocks for a particular fileNum. This allows
// efficient eviction of all of the blocks for a file which is used when an
// sstable is deleted from disk.
//
// # Memory Management
//
// In order to reduce pressure on the Go GC, manual memory management is
// performed for the data stored in the cache. Manual memory management is
// performed by calling into C.{malloc,free} to allocate memory. Cache.Values
// are reference counted and the memory backing a manual value is freed when
// the reference count drops to 0.
//
// Manual memory management brings the possibility of memory leaks. It is
// imperative that every Handle returned by Cache.{Get,Set} is eventually
// released. The "invariants" build tag enables a leak detection facility that
// places a GC finalizer on cache.Value. When the cache.Value finalizer is run,
// if the underlying buffer is still present a leak has occurred. The "tracing"
// build tag enables tracing of cache.Value reference count manipulation and
// eases finding where a leak has occurred. These two facilities are usually
// used in combination by specifying `-tags invariants,tracing`. Note that
// "tracing" produces a significant slowdown, while "invariants" does not.
type Cache struct {
refs atomic.Int64
maxSize int64
idAlloc atomic.Uint64
shards []shard
// Traces recorded by Cache.trace. Used for debugging.
tr struct {
sync.Mutex
msgs []string
}
}
// New creates a new cache of the specified size. Memory for the cache is
// allocated on demand, not during initialization. The cache is created with a
// reference count of 1. Each DB it is associated with adds a reference, so the
// creator of the cache should usually release their reference after the DB is
// created.
//
// c := cache.New(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func New(size int64) *Cache {
// How many cache shards should we create?
//
// Note that the probability two processors will try to access the same
// shard at the same time increases superlinearly with the number of
// processors (Eg, consider the brithday problem where each CPU is a person,
// and each shard is a possible birthday).
//
// We could consider growing the number of shards superlinearly, but
// increasing the shard count may reduce the effectiveness of the caching
// algorithm if frequently-accessed blocks are insufficiently distributed
// across shards. If a shard's size is smaller than a single frequently
// scanned sstable, then the shard will be unable to hold the entire
// frequently-scanned table in memory despite other shards still holding
// infrequently accessed blocks.
//
// Experimentally, we've observed contention contributing to tail latencies
// at 2 shards per processor. For now we use 4 shards per processor,
// recognizing this may not be final word.
m := 4 * runtime.GOMAXPROCS(0)
// In tests we can use large CPU machines with small cache sizes and have
// many caches in existence at a time. If sharding into m shards would
// produce too small shards, constrain the number of shards to 4.
const minimumShardSize = 4 << 20 // 4 MiB
if m > 4 && int(size)/m < minimumShardSize {
m = 4
}
return newShards(size, m)
}
func newShards(size int64, shards int) *Cache {
c := &Cache{
maxSize: size,
shards: make([]shard, shards),
}
c.refs.Store(1)
c.idAlloc.Store(1)
c.trace("alloc", c.refs.Load())
for i := range c.shards {
c.shards[i] = shard{
maxSize: size / int64(len(c.shards)),
coldTarget: size / int64(len(c.shards)),
}
if entriesGoAllocated {
c.shards[i].entries = make(map[*entry]struct{})
}
c.shards[i].blocks.init(16)
c.shards[i].files.init(16)
}
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(c, func(obj interface{}) {
c := obj.(*Cache)
if v := c.refs.Load(); v != 0 {
c.tr.Lock()
fmt.Fprintf(os.Stderr,
"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
if len(c.tr.msgs) > 0 {
fmt.Fprintf(os.Stderr, "%s\n", strings.Join(c.tr.msgs, "\n"))
}
c.tr.Unlock()
os.Exit(1)
}
})
return c
}
func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
// Inlined version of fnv.New64 + Write.
const offset64 = 14695981039346656037
const prime64 = 1099511628211
h := uint64(offset64)
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(id & 0xff)
id >>= 8
}
fileNumVal := uint64(fileNum.FileNum())
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(fileNumVal) & 0xff
fileNumVal >>= 8
}
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(offset & 0xff)
offset >>= 8
}
return &c.shards[h%uint64(len(c.shards))]
}
// Ref adds a reference to the cache. The cache only remains valid as long a
// reference is maintained to it.
func (c *Cache) Ref() {
v := c.refs.Add(1)
if v <= 1 {
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
c.trace("ref", v)
}
// Unref releases a reference on the cache.
func (c *Cache) Unref() {
v := c.refs.Add(-1)
c.trace("unref", v)
switch {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
for i := range c.shards {
c.shards[i].Free()
}
}
}
// Get retrieves the cache value for the specified file and offset, returning
// nil if no value is present.
func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
}
// Set sets the cache value for the specified file and offset, overwriting an
// existing value if present. A Handle is returned which provides faster
// retrieval of the cached value than Get (lock-free and avoidance of the map
// lookup). The value must have been allocated by Cache.Alloc.
func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
}
// Delete deletes the cached value for the specified file and offset.
func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
}
// EvictFile evicts all of the cache values for the specified file.
func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
for i := range c.shards {
c.shards[i].EvictFile(id, fileNum)
}
}
// MaxSize returns the max size of the cache.
func (c *Cache) MaxSize() int64 {
return c.maxSize
}
// Size returns the current space used by the cache.
func (c *Cache) Size() int64 {
var size int64
for i := range c.shards {
size += c.shards[i].Size()
}
return size
}
// Alloc allocates a byte slice of the specified size, possibly reusing
// previously allocated but unused memory. The memory backing the value is
// manually managed. The caller MUST either add the value to the cache (via
// Cache.Set), or release the value (via Cache.Free). Failure to do so will
// result in a memory leak.
func Alloc(n int) *Value {
return newValue(n)
}
// Free frees the specified value. The buffer associated with the value will
// possibly be reused, making it invalid to use the buffer after calling
// Free. Do not call Free on a value that has been added to the cache.
func Free(v *Value) {
if n := v.refs(); n > 1 {
panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
}
v.release()
}
// Reserve N bytes in the cache. This effectively shrinks the size of the cache
// by N bytes, without actually consuming any memory. The returned closure
// should be invoked to release the reservation.
func (c *Cache) Reserve(n int) func() {
// Round-up the per-shard reservation. Most reservations should be large, so
// this probably doesn't matter in practice.
shardN := (n + len(c.shards) - 1) / len(c.shards)
for i := range c.shards {
c.shards[i].Reserve(shardN)
}
return func() {
if shardN == -1 {
panic("pebble: cache reservation already released")
}
for i := range c.shards {
c.shards[i].Reserve(-shardN)
}
shardN = -1
}
}
// Metrics returns the metrics for the cache.
func (c *Cache) Metrics() Metrics {
var m Metrics
for i := range c.shards {
s := &c.shards[i]
s.mu.RLock()
m.Count += int64(s.blocks.Count())
m.Size += s.sizeHot + s.sizeCold
s.mu.RUnlock()
m.Hits += s.hits.Load()
m.Misses += s.misses.Load()
}
return m
}
// NewID returns a new ID to be used as a namespace for cached file
// blocks.
func (c *Cache) NewID() uint64 {
return c.idAlloc.Add(1)
}

10
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_normal.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !tracing
// +build !tracing
package cache
func (c *Cache) trace(_ string, _ int64) {}

20
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build tracing
// +build tracing
package cache
import (
"fmt"
"runtime/debug"
)
func (c *Cache) trace(msg string, refs int64) {
s := fmt.Sprintf("%s: refs=%d\n%s", msg, refs, debug.Stack())
c.tr.Lock()
c.tr.msgs = append(c.tr.msgs, s)
c.tr.Unlock()
}

155
vendor/github.com/cockroachdb/pebble/internal/cache/entry.go generated vendored Normal file
View File

@@ -0,0 +1,155 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
import "sync/atomic"
type entryType int8
const (
etTest entryType = iota
etCold
etHot
)
func (p entryType) String() string {
switch p {
case etTest:
return "test"
case etCold:
return "cold"
case etHot:
return "hot"
}
return "unknown"
}
// entry holds the metadata for a cache entry. The memory for an entry is
// allocated from manually managed memory.
//
// Using manual memory management for entries is technically a volation of the
// Cgo pointer rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// Specifically, Go pointers should not be stored in C allocated memory. The
// reason for this rule is that the Go GC will not look at C allocated memory
// to find pointers to Go objects. If the only reference to a Go object is
// stored in C allocated memory, the object will be reclaimed. The shard field
// of the entry struct points to a Go allocated object, thus the
// violation. What makes this "safe" is that the Cache guarantees that there
// are other pointers to the shard which will keep it alive.
type entry struct {
key key
// The value associated with the entry. The entry holds a reference on the
// value which is maintained by entry.setValue().
val *Value
blockLink struct {
next *entry
prev *entry
}
fileLink struct {
next *entry
prev *entry
}
size int64
ptype entryType
// referenced is atomically set to indicate that this entry has been accessed
// since the last time one of the clock hands swept it.
referenced atomic.Bool
shard *shard
// Reference count for the entry. The entry is freed when the reference count
// drops to zero.
ref refcnt
}
func newEntry(s *shard, key key, size int64) *entry {
e := entryAllocNew()
*e = entry{
key: key,
size: size,
ptype: etCold,
shard: s,
}
e.blockLink.next = e
e.blockLink.prev = e
e.fileLink.next = e
e.fileLink.prev = e
e.ref.init(1)
return e
}
func (e *entry) free() {
e.setValue(nil)
*e = entry{}
entryAllocFree(e)
}
func (e *entry) next() *entry {
if e == nil {
return nil
}
return e.blockLink.next
}
func (e *entry) prev() *entry {
if e == nil {
return nil
}
return e.blockLink.prev
}
func (e *entry) link(s *entry) {
s.blockLink.prev = e.blockLink.prev
s.blockLink.prev.blockLink.next = s
s.blockLink.next = e
s.blockLink.next.blockLink.prev = s
}
func (e *entry) unlink() *entry {
next := e.blockLink.next
e.blockLink.prev.blockLink.next = e.blockLink.next
e.blockLink.next.blockLink.prev = e.blockLink.prev
e.blockLink.prev = e
e.blockLink.next = e
return next
}
func (e *entry) linkFile(s *entry) {
s.fileLink.prev = e.fileLink.prev
s.fileLink.prev.fileLink.next = s
s.fileLink.next = e
s.fileLink.next.fileLink.prev = s
}
func (e *entry) unlinkFile() *entry {
next := e.fileLink.next
e.fileLink.prev.fileLink.next = e.fileLink.next
e.fileLink.next.fileLink.prev = e.fileLink.prev
e.fileLink.prev = e
e.fileLink.next = e
return next
}
func (e *entry) setValue(v *Value) {
if v != nil {
v.acquire()
}
old := e.val
e.val = v
old.release()
}
func (e *entry) peekValue() *Value {
return e.val
}
func (e *entry) acquireValue() *Value {
v := e.val
if v != nil {
v.acquire()
}
return v
}

38
vendor/github.com/cockroachdb/pebble/internal/cache/entry_invariants.go generated vendored Normal file
View File

@@ -0,0 +1,38 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package cache
import (
"fmt"
"os"
"github.com/cockroachdb/pebble/internal/invariants"
)
// When the "invariants" or "tracing" build tags are enabled, we need to
// allocate entries using the Go allocator so entry.val properly maintains a
// reference to the Value.
const entriesGoAllocated = true
func entryAllocNew() *entry {
e := &entry{}
// Note: this is a no-op if invariants and tracing are disabled or race is
// enabled.
invariants.SetFinalizer(e, func(obj interface{}) {
e := obj.(*entry)
if v := e.ref.refs(); v != 0 {
fmt.Fprintf(os.Stderr, "%p: cache entry has non-zero reference count: %d\n%s",
e, v, e.ref.traces())
os.Exit(1)
}
})
return e
}
func entryAllocFree(e *entry) {
}

103
vendor/github.com/cockroachdb/pebble/internal/cache/entry_normal.go generated vendored Normal file
View File

@@ -0,0 +1,103 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package cache
import (
"runtime"
"sync"
"unsafe"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
const (
entrySize = int(unsafe.Sizeof(entry{}))
entryAllocCacheLimit = 128
// Avoid using runtime.SetFinalizer in race builds as finalizers tickle a bug
// in the Go race detector in go1.15 and earlier versions. This requires that
// entries are Go allocated rather than manually allocated.
//
// If cgo is disabled we need to allocate the entries using the Go allocator
// and is violates the Go GC rules to put Go pointers (such as the entry
// pointer fields) into untyped memory (i.e. a []byte).
entriesGoAllocated = invariants.RaceEnabled || !cgoEnabled
)
var entryAllocPool = sync.Pool{
New: func() interface{} {
return newEntryAllocCache()
},
}
func entryAllocNew() *entry {
a := entryAllocPool.Get().(*entryAllocCache)
e := a.alloc()
entryAllocPool.Put(a)
return e
}
func entryAllocFree(e *entry) {
a := entryAllocPool.Get().(*entryAllocCache)
a.free(e)
entryAllocPool.Put(a)
}
type entryAllocCache struct {
entries []*entry
}
func newEntryAllocCache() *entryAllocCache {
c := &entryAllocCache{}
if !entriesGoAllocated {
// Note the use of a "real" finalizer here (as opposed to a build tag-gated
// no-op finalizer). Without the finalizer, objects released from the pool
// and subsequently GC'd by the Go runtime would fail to have their manually
// allocated memory freed, which results in a memory leak.
// lint:ignore SetFinalizer
runtime.SetFinalizer(c, freeEntryAllocCache)
}
return c
}
func freeEntryAllocCache(obj interface{}) {
c := obj.(*entryAllocCache)
for i, e := range c.entries {
c.dealloc(e)
c.entries[i] = nil
}
}
func (c *entryAllocCache) alloc() *entry {
n := len(c.entries)
if n == 0 {
if entriesGoAllocated {
return &entry{}
}
b := manual.New(entrySize)
return (*entry)(unsafe.Pointer(&b[0]))
}
e := c.entries[n-1]
c.entries = c.entries[:n-1]
return e
}
func (c *entryAllocCache) dealloc(e *entry) {
if !entriesGoAllocated {
buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(e))[:entrySize:entrySize]
manual.Free(buf)
}
}
func (c *entryAllocCache) free(e *entry) {
if len(c.entries) == entryAllocCacheLimit {
c.dealloc(e)
return
}
c.entries = append(c.entries, e)
}

59
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_normal.go generated vendored Normal file
View File

@@ -0,0 +1,59 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !tracing
// +build !tracing
package cache
import (
"fmt"
"sync/atomic"
"github.com/cockroachdb/redact"
)
// refcnt provides an atomic reference count. This version is used when the
// "tracing" build tag is not enabled. See refcnt_tracing.go for the "tracing"
// enabled version.
type refcnt struct {
val atomic.Int32
}
// initialize the reference count to the specified value.
func (v *refcnt) init(val int32) {
v.val.Store(val)
}
func (v *refcnt) refs() int32 {
return v.val.Load()
}
func (v *refcnt) acquire() {
switch v := v.val.Add(1); {
case v <= 1:
panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
}
}
func (v *refcnt) release() bool {
switch v := v.val.Add(-1); {
case v < 0:
panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
case v == 0:
return true
default:
return false
}
}
func (v *refcnt) trace(msg string) {
}
func (v *refcnt) traces() string {
return ""
}
// Silence unused warning.
var _ = (*refcnt)(nil).traces

66
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,66 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build tracing
// +build tracing
package cache
import (
"fmt"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
)
// refcnt provides an atomic reference count, along with a tracing facility for
// debugging logic errors in manipulating the reference count. This version is
// used when the "tracing" build tag is enabled.
type refcnt struct {
val atomic.Int32
sync.Mutex
msgs []string
}
func (v *refcnt) init(val int32) {
v.val.Store(val)
v.trace("init")
}
func (v *refcnt) refs() int32 {
return v.val.Load()
}
func (v *refcnt) acquire() {
switch n := v.val.Add(1); {
case n <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
}
v.trace("acquire")
}
func (v *refcnt) release() bool {
n := v.val.Add(-1)
switch {
case n < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
}
v.trace("release")
return n == 0
}
func (v *refcnt) trace(msg string) {
s := fmt.Sprintf("%s: refs=%d\n%s", msg, v.refs(), debug.Stack())
v.Lock()
v.msgs = append(v.msgs, s)
v.Unlock()
}
func (v *refcnt) traces() string {
v.Lock()
s := strings.Join(v.msgs, "\n")
v.Unlock()
return s
}

320
vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go generated vendored Normal file
View File

@@ -0,0 +1,320 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
import (
"fmt"
"math/bits"
"os"
"runtime/debug"
"strings"
"time"
"unsafe"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
var hashSeed = uint64(time.Now().UnixNano())
// Fibonacci hash: https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/
func robinHoodHash(k key, shift uint32) uint32 {
const m = 11400714819323198485
h := hashSeed
h ^= k.id * m
h ^= uint64(k.fileNum.FileNum()) * m
h ^= k.offset * m
return uint32(h >> shift)
}
type robinHoodEntry struct {
key key
// Note that value may point to a Go allocated object (if the "invariants"
// build tag was specified), even though the memory for the entry itself is
// manually managed. This is technically a volation of the Cgo pointer rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// Specifically, Go pointers should not be stored in C allocated memory. The
// reason for this rule is that the Go GC will not look at C allocated memory
// to find pointers to Go objects. If the only reference to a Go object is
// stored in C allocated memory, the object will be reclaimed. What makes
// this "safe" is that the Cache guarantees that there are other pointers to
// the entry and shard which will keep them alive. In particular, every Go
// allocated entry in the cache is referenced by the shard.entries map. And
// every shard is referenced by the Cache.shards map.
value *entry
// The distance the entry is from its desired position.
dist uint32
}
type robinHoodEntries struct {
ptr unsafe.Pointer
len uint32
}
func newRobinHoodEntries(n uint32) robinHoodEntries {
size := uintptr(n) * unsafe.Sizeof(robinHoodEntry{})
return robinHoodEntries{
ptr: unsafe.Pointer(&(manual.New(int(size)))[0]),
len: n,
}
}
func (e robinHoodEntries) at(i uint32) *robinHoodEntry {
return (*robinHoodEntry)(unsafe.Pointer(uintptr(e.ptr) +
uintptr(i)*unsafe.Sizeof(robinHoodEntry{})))
}
func (e robinHoodEntries) free() {
size := uintptr(e.len) * unsafe.Sizeof(robinHoodEntry{})
buf := (*[manual.MaxArrayLen]byte)(e.ptr)[:size:size]
manual.Free(buf)
}
// robinHoodMap is an implementation of Robin Hood hashing. Robin Hood hashing
// is an open-address hash table using linear probing. The twist is that the
// linear probe distance is reduced by moving existing entries when inserting
// and deleting. This is accomplished by keeping track of how far an entry is
// from its "desired" slot (hash of key modulo number of slots). During
// insertion, if the new entry being inserted is farther from its desired slot
// than the target entry, we swap the target and new entry. This effectively
// steals from the "rich" target entry and gives to the "poor" new entry (thus
// the origin of the name).
//
// An extension over the base Robin Hood hashing idea comes from
// https://probablydance.com/2017/02/26/i-wrote-the-fastest-hashtable/. A cap
// is placed on the max distance an entry can be from its desired slot. When
// this threshold is reached during insertion, the size of the table is doubled
// and insertion is restarted. Additionally, the entries slice is given "max
// dist" extra entries on the end. The very last entry in the entries slice is
// never used and acts as a sentinel which terminates loops. The previous
// maxDist-1 entries act as the extra entries. For example, if the size of the
// table is 2, maxDist is computed as 4 and the actual size of the entry slice
// is 6.
//
// +---+---+---+---+---+---+
// | 0 | 1 | 2 | 3 | 4 | 5 |
// +---+---+---+---+---+---+
// ^
// size
//
// In this scenario, the target entry for a key will always be in the range
// [0,1]. Valid entries may reside in the range [0,4] due to the linear probing
// of up to maxDist entries. The entry at index 5 will never contain a value,
// and instead acts as a sentinel (its distance is always 0). The max distance
// threshold is set to log2(num-entries). This ensures that retrieval is O(log
// N), though note that N is the number of total entries, not the count of
// valid entries.
//
// Deletion is implemented via the backward shift delete mechanism instead of
// tombstones. This preserves the performance of the table in the presence of
// deletions. See
// http://codecapsule.com/2013/11/17/robin-hood-hashing-backward-shift-deletion
// for details.
type robinHoodMap struct {
entries robinHoodEntries
size uint32
shift uint32
count uint32
maxDist uint32
}
func maxDistForSize(size uint32) uint32 {
desired := uint32(bits.Len32(size))
if desired < 4 {
desired = 4
}
return desired
}
func newRobinHoodMap(initialCapacity int) *robinHoodMap {
m := &robinHoodMap{}
m.init(initialCapacity)
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(m, func(obj interface{}) {
m := obj.(*robinHoodMap)
if m.entries.ptr != nil {
fmt.Fprintf(os.Stderr, "%p: robin-hood map not freed\n", m)
os.Exit(1)
}
})
return m
}
func (m *robinHoodMap) init(initialCapacity int) {
if initialCapacity < 1 {
initialCapacity = 1
}
targetSize := 1 << (uint(bits.Len(uint(2*initialCapacity-1))) - 1)
m.rehash(uint32(targetSize))
}
func (m *robinHoodMap) free() {
if m.entries.ptr != nil {
m.entries.free()
m.entries.ptr = nil
}
}
func (m *robinHoodMap) rehash(size uint32) {
oldEntries := m.entries
m.size = size
m.shift = uint32(64 - bits.Len32(m.size-1))
m.maxDist = maxDistForSize(size)
m.entries = newRobinHoodEntries(size + m.maxDist)
m.count = 0
for i := uint32(0); i < oldEntries.len; i++ {
e := oldEntries.at(i)
if e.value != nil {
m.Put(e.key, e.value)
}
}
if oldEntries.ptr != nil {
oldEntries.free()
}
}
// Find an entry containing the specified value. This is intended to be used
// from debug and test code.
func (m *robinHoodMap) findByValue(v *entry) *robinHoodEntry {
for i := uint32(0); i < m.entries.len; i++ {
e := m.entries.at(i)
if e.value == v {
return e
}
}
return nil
}
func (m *robinHoodMap) Count() int {
return int(m.count)
}
func (m *robinHoodMap) Put(k key, v *entry) {
maybeExists := true
n := robinHoodEntry{key: k, value: v, dist: 0}
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if maybeExists && k == e.key {
// Entry already exists: overwrite.
e.value = n.value
m.checkEntry(i)
return
}
if e.value == nil {
// Found an empty entry: insert here.
*e = n
m.count++
m.checkEntry(i)
return
}
if e.dist < n.dist {
// Swap the new entry with the current entry because the current is
// rich. We then continue to loop, looking for a new location for the
// current entry. Note that this is also the not-found condition for
// retrieval, which means that "k" is not present in the map. See Get().
n, *e = *e, n
m.checkEntry(i)
maybeExists = false
}
// The new entry gradually moves away from its ideal position.
n.dist++
// If we've reached the max distance threshold, grow the table and restart
// the insertion.
if n.dist == m.maxDist {
m.rehash(2 * m.size)
i = robinHoodHash(n.key, m.shift) - 1
n.dist = 0
maybeExists = false
}
}
}
func (m *robinHoodMap) Get(k key) *entry {
var dist uint32
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if k == e.key {
// Found.
return e.value
}
if e.dist < dist {
// Not found.
return nil
}
dist++
}
}
func (m *robinHoodMap) Delete(k key) {
var dist uint32
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if k == e.key {
m.checkEntry(i)
// We found the entry to delete. Shift the following entries backwards
// until the next empty value or entry with a zero distance. Note that
// empty values are guaranteed to have "dist == 0".
m.count--
for j := i + 1; ; j++ {
t := m.entries.at(j)
if t.dist == 0 {
*e = robinHoodEntry{}
return
}
e.key = t.key
e.value = t.value
e.dist = t.dist - 1
e = t
m.checkEntry(j)
}
}
if dist > e.dist {
// Not found.
return
}
dist++
}
}
func (m *robinHoodMap) checkEntry(i uint32) {
if invariants.Enabled {
e := m.entries.at(i)
if e.value != nil {
pos := robinHoodHash(e.key, m.shift)
if (uint32(i) - pos) != e.dist {
fmt.Fprintf(os.Stderr, "%d: invalid dist=%d, expected %d: %s\n%s",
i, e.dist, uint32(i)-pos, e.key, debug.Stack())
os.Exit(1)
}
if e.dist > m.maxDist {
fmt.Fprintf(os.Stderr, "%d: invalid dist=%d > maxDist=%d: %s\n%s",
i, e.dist, m.maxDist, e.key, debug.Stack())
os.Exit(1)
}
}
}
}
func (m *robinHoodMap) String() string {
var buf strings.Builder
fmt.Fprintf(&buf, "count: %d\n", m.count)
for i := uint32(0); i < m.entries.len; i++ {
e := m.entries.at(i)
if e.value != nil {
fmt.Fprintf(&buf, "%d: [%s,%p,%d]\n", i, e.key, e.value, e.dist)
}
}
return buf.String()
}

46
vendor/github.com/cockroachdb/pebble/internal/cache/value.go generated vendored Normal file
View File

@@ -0,0 +1,46 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
// Value holds a reference counted immutable value.
type Value struct {
buf []byte
// Reference count for the value. The value is freed when the reference count
// drops to zero.
ref refcnt
}
// Buf returns the buffer associated with the value. The contents of the buffer
// should not be changed once the value has been added to the cache. Instead, a
// new Value should be created and added to the cache to replace the existing
// value.
func (v *Value) Buf() []byte {
if v == nil {
return nil
}
return v.buf
}
// Truncate the buffer to the specified length. The buffer length should not be
// changed once the value has been added to the cache as there may be
// concurrent readers of the Value. Instead, a new Value should be created and
// added to the cache to replace the existing value.
func (v *Value) Truncate(n int) {
v.buf = v.buf[:n]
}
func (v *Value) refs() int32 {
return v.ref.refs()
}
func (v *Value) acquire() {
v.ref.acquire()
}
func (v *Value) release() {
if v != nil && v.ref.release() {
v.free()
}
}

55
vendor/github.com/cockroachdb/pebble/internal/cache/value_invariants.go generated vendored Normal file
View File

@@ -0,0 +1,55 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package cache
import (
"fmt"
"os"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
// newValue creates a Value with a manually managed buffer of size n.
//
// This definition of newValue is used when either the "invariants" or
// "tracing" build tags are specified. It hooks up a finalizer to the returned
// Value that checks for memory leaks when the GC determines the Value is no
// longer reachable.
func newValue(n int) *Value {
if n == 0 {
return nil
}
b := manual.New(n)
v := &Value{buf: b}
v.ref.init(1)
// Note: this is a no-op if invariants and tracing are disabled or race is
// enabled.
invariants.SetFinalizer(v, func(obj interface{}) {
v := obj.(*Value)
if v.buf != nil {
fmt.Fprintf(os.Stderr, "%p: cache value was not freed: refs=%d\n%s",
v, v.refs(), v.ref.traces())
os.Exit(1)
}
})
return v
}
func (v *Value) free() {
// When "invariants" are enabled set the value contents to 0xff in order to
// cache use-after-free bugs.
for i := range v.buf {
v.buf[i] = 0xff
}
manual.Free(v.buf)
// Setting Value.buf to nil is needed for correctness of the leak checking
// that is performed when the "invariants" or "tracing" build tags are
// enabled.
v.buf = nil
}

57
vendor/github.com/cockroachdb/pebble/internal/cache/value_normal.go generated vendored Normal file
View File

@@ -0,0 +1,57 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package cache
import (
"unsafe"
"github.com/cockroachdb/pebble/internal/manual"
)
const valueSize = int(unsafe.Sizeof(Value{}))
func newValue(n int) *Value {
if n == 0 {
return nil
}
if !cgoEnabled {
// If Cgo is disabled then all memory is allocated from the Go heap and we
// can't play the trick below to combine the Value and buffer allocation.
v := &Value{buf: make([]byte, n)}
v.ref.init(1)
return v
}
// When we're not performing leak detection, the lifetime of the returned
// Value is exactly the lifetime of the backing buffer and we can manually
// allocate both.
//
// TODO(peter): It may be better to separate the allocation of the value and
// the buffer in order to reduce internal fragmentation in malloc. If the
// buffer is right at a power of 2, adding valueSize might push the
// allocation over into the next larger size.
b := manual.New(valueSize + n)
v := (*Value)(unsafe.Pointer(&b[0]))
v.buf = b[valueSize:]
v.ref.init(1)
return v
}
func (v *Value) free() {
if !cgoEnabled {
return
}
// When we're not performing leak detection, the Value and buffer were
// allocated contiguously.
n := valueSize + cap(v.buf)
buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(v))[:n:n]
v.buf = nil
manual.Free(buf)
}

17
vendor/github.com/cockroachdb/pebble/internal/constants/constants.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package constants
const (
// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
oneIf64Bit = ^uint(0) >> 63
// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
// - MaxUint32 on 64-bit platforms;
// - MaxInt on 32-bit platforms.
// It is used when slices are limited to Uint32 on 64-bit platforms (the
// length limit for slices is naturally MaxInt on 32-bit platforms).
MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
)

42
vendor/github.com/cockroachdb/pebble/internal/crc/crc.go generated vendored Normal file
View File

@@ -0,0 +1,42 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package crc implements the checksum algorithm used throughout pebble.
//
// The algorithm is CRC-32 with Castagnoli's polynomial, followed by a bit
// rotation and an additional delta. The additional processing is to lessen the
// probability of arbitrary key/value data coincidentally containing bytes that
// look like a checksum.
//
// To calculate the uint32 checksum of some data:
//
// var u uint32 = crc.New(data).Value()
//
// In pebble, the uint32 value is then stored in little-endian format.
package crc // import "github.com/cockroachdb/pebble/internal/crc"
import "hash/crc32"
var table = crc32.MakeTable(crc32.Castagnoli)
// CRC is a small convenience wrapper for computing the CRC32 checksum used by
// pebble. This is the same algorithm as used by RocksDB.
type CRC uint32
// New returns the result of adding the bytes to the zero-value CRC.
func New(b []byte) CRC {
return CRC(0).Update(b)
}
// Update returns the result of adding the bytes to the CRC.
func (c CRC) Update(b []byte) CRC {
return CRC(crc32.Update(uint32(c), table, b))
}
// Value returns the cooked CRC value. The additional processing is to lessen
// the probability of arbitrary key/value data coincidentally containing bytes
// that look like a checksum.
func (c CRC) Value() uint32 {
return uint32(c>>15|c<<17) + 0xa282ead8
}

17
vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package fastrand
import _ "unsafe" // required by go:linkname
// Uint32 returns a lock free uint32 value.
//
//go:linkname Uint32 runtime.fastrand
func Uint32() uint32
// Uint32n returns a lock free uint32 value in the interval [0, n).
//
//go:linkname Uint32n runtime.fastrandn
func Uint32n(n uint32) uint32

68
vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go generated vendored Normal file
View File

@@ -0,0 +1,68 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package humanize
import (
"fmt"
"math"
"github.com/cockroachdb/redact"
)
func logn(n, b float64) float64 {
return math.Log(n) / math.Log(b)
}
func humanate(s uint64, base float64, suffixes []string) string {
if s < 10 {
return fmt.Sprintf("%d%s", s, suffixes[0])
}
e := math.Floor(logn(float64(s), base))
suffix := suffixes[int(e)]
val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10
f := "%.0f%s"
if val < 10 {
f = "%.1f%s"
}
return fmt.Sprintf(f, val, suffix)
}
type config struct {
base float64
suffix []string
}
// Bytes produces human readable representations of byte values in IEC units.
var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
// Count produces human readable representations of unitless values in SI units.
var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
// Int64 produces a human readable representation of the value.
func (c *config) Int64(s int64) FormattedString {
if s < 0 {
return FormattedString("-" + humanate(uint64(-s), c.base, c.suffix))
}
return FormattedString(humanate(uint64(s), c.base, c.suffix))
}
// Uint64 produces a human readable representation of the value.
func (c *config) Uint64(s uint64) FormattedString {
return FormattedString(humanate(s, c.base, c.suffix))
}
// FormattedString represents a human readable representation of a value. It
// implements the redact.SafeValue interface to signal that it represents a
// a string that does not need to be redacted.
type FormattedString string
var _ redact.SafeValue = FormattedString("")
// SafeValue implements redact.SafeValue.
func (fs FormattedString) SafeValue() {}
// String implements fmt.Stringer.
func (fs FormattedString) String() string { return string(fs) }

27
vendor/github.com/cockroachdb/pebble/internal/intern/intern.go generated vendored Normal file
View File

@@ -0,0 +1,27 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package intern
import "sync"
var pool = sync.Pool{
New: func() interface{} {
return make(map[string]string)
},
}
// Bytes returns b converted to a string, interned.
func Bytes(b []byte) string {
m := pool.Get().(map[string]string)
c, ok := m[string(b)]
if ok {
pool.Put(m)
return c
}
s := string(b)
m[s] = s
pool.Put(m)
return s
}

14
vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_off.go generated vendored Normal file
View File

@@ -0,0 +1,14 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package invariants
// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
// race builds or if neither the invariants or tracing build tags are
// specified.
func SetFinalizer(obj, finalizer interface{}) {
}

17
vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_on.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package invariants
import "runtime"
// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
// race builds or if neither the invariants or tracing build tags are
// specified.
func SetFinalizer(obj, finalizer interface{}) {
runtime.SetFinalizer(obj, finalizer)
}

11
vendor/github.com/cockroachdb/pebble/internal/invariants/off.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !invariants && !race
// +build !invariants,!race
package invariants
// Enabled is true if we were built with the "invariants" or "race" build tags.
const Enabled = false

11
vendor/github.com/cockroachdb/pebble/internal/invariants/on.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build invariants || race
// +build invariants race
package invariants
// Enabled is true if we were built with the "invariants" or "race" build tags.
const Enabled = true

11
vendor/github.com/cockroachdb/pebble/internal/invariants/race_off.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !race
// +build !race
package invariants
// RaceEnabled is true if we were built with the "race" build tag.
const RaceEnabled = false

11
vendor/github.com/cockroachdb/pebble/internal/invariants/race_on.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build race
// +build race
package invariants
// RaceEnabled is true if we were built with the "race" build tag.
const RaceEnabled = true

268
vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go generated vendored Normal file
View File

@@ -0,0 +1,268 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// TODO(jackson): Consider removing this type and adding bounds enforcement
// directly to the MergingIter. This type is probably too lightweight to warrant
// its own type, but for now we implement it separately for expediency.
// boundedIterPos records the position of the BoundedIter relative to the
// underlying iterator's position. It's used to avoid Next/Prev-ing the iterator
// if there can't possibly be another span within bounds, because the current
// span overlaps the bound.
//
// Imagine bounds [a,c) and an iterator that seeks to a span [b,d). The span
// [b,d) overlaps some portion of the iterator bounds, so the iterator must
// return it. If the iterator is subsequently Nexted, Next can tell that the
// iterator is exhausted without advancing the underlying iterator because the
// current span's end bound of d is ≥ the upper bound of c. In this case, the
// bounded iterator returns nil and records i.pos as posAtUpperLimit to remember
// that the underlying iterator position does not match the current BoundedIter
// position.
type boundedIterPos int8
const (
posAtLowerLimit boundedIterPos = -1
posAtIterSpan boundedIterPos = 0
posAtUpperLimit boundedIterPos = +1
)
// BoundedIter implements FragmentIterator and enforces bounds.
//
// Like the point InternalIterator interface, the bounded iterator's forward
// positioning routines (SeekGE, First, and Next) only check the upper bound.
// The reverse positioning routines (SeekLT, Last, and Prev) only check the
// lower bound. It is up to the caller to ensure that the forward positioning
// routines respect the lower bound and the reverse positioning routines respect
// the upper bound (i.e. calling SeekGE instead of First if there is a lower
// bound, and SeekLT instead of Last if there is an upper bound).
//
// When the hasPrefix parameter indicates that the iterator is in prefix
// iteration mode, BoundedIter elides any spans that do not overlap with the
// prefix's keyspace. In prefix iteration mode, reverse iteration is disallowed,
// except for an initial SeekLT with a seek key greater than or equal to the
// prefix. In prefix iteration mode, the first seek must position the iterator
// at or immediately before the first fragment covering a key greater than or
// equal to the prefix.
type BoundedIter struct {
iter FragmentIterator
iterSpan *Span
cmp base.Compare
split base.Split
lower []byte
upper []byte
hasPrefix *bool
prefix *[]byte
pos boundedIterPos
}
// Init initializes the bounded iterator.
//
// In addition to the iterator bounds, Init takes pointers to a boolean
// indicating whether the iterator is in prefix iteration mode and the prefix
// key if it is. This is used to exclude spans that are outside the iteration
// prefix.
//
// hasPrefix and prefix are allowed to be nil, however if hasPrefix != nil,
// prefix must also not be nil.
func (i *BoundedIter) Init(
cmp base.Compare,
split base.Split,
iter FragmentIterator,
lower, upper []byte,
hasPrefix *bool,
prefix *[]byte,
) {
*i = BoundedIter{
iter: iter,
cmp: cmp,
split: split,
lower: lower,
upper: upper,
hasPrefix: hasPrefix,
prefix: prefix,
}
}
var _ FragmentIterator = (*BoundedIter)(nil)
// Seek calls.
//
// Seek calls check iterator bounds in the direction of the seek. Additionally,
// if the iterator is in prefix iteration mode, seek calls check both start and
// end bounds against the prefix's bounds. We check both bounds for defense in
// depth. This optimization has been a source of various bugs due to various
// other prefix iteration optimizations that can result in seek keys that don't
// respect the prefix bounds.
// SeekGE implements FragmentIterator.
func (i *BoundedIter) SeekGE(key []byte) *Span {
s := i.iter.SeekGE(key)
s = i.checkPrefixSpanStart(s)
s = i.checkPrefixSpanEnd(s)
return i.checkForwardBound(s)
}
// SeekLT implements FragmentIterator.
func (i *BoundedIter) SeekLT(key []byte) *Span {
s := i.iter.SeekLT(key)
s = i.checkPrefixSpanStart(s)
s = i.checkPrefixSpanEnd(s)
return i.checkBackwardBound(s)
}
// First implements FragmentIterator.
func (i *BoundedIter) First() *Span {
s := i.iter.First()
s = i.checkPrefixSpanStart(s)
return i.checkForwardBound(s)
}
// Last implements FragmentIterator.
func (i *BoundedIter) Last() *Span {
s := i.iter.Last()
s = i.checkPrefixSpanEnd(s)
return i.checkBackwardBound(s)
}
// Next implements FragmentIterator.
func (i *BoundedIter) Next() *Span {
switch i.pos {
case posAtLowerLimit:
// The BoundedIter had previously returned nil, because it knew from
// i.iterSpan's bounds that there was no previous span. To Next, we only
// need to return the current iter span and reset i.pos to reflect that
// we're no longer positioned at the limit.
i.pos = posAtIterSpan
return i.iterSpan
case posAtIterSpan:
// If the span at the underlying iterator position extends to or beyond the
// upper bound, we can avoid advancing because the next span is necessarily
// out of bounds.
if i.iterSpan != nil && i.upper != nil && i.cmp(i.iterSpan.End, i.upper) >= 0 {
i.pos = posAtUpperLimit
return nil
}
// Similarly, if the span extends to the next prefix and we're in prefix
// iteration mode, we can avoid advancing.
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
ei := i.split(i.iterSpan.End)
if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 {
i.pos = posAtUpperLimit
return nil
}
}
return i.checkForwardBound(i.checkPrefixSpanStart(i.iter.Next()))
case posAtUpperLimit:
// Already exhausted.
return nil
default:
panic("unreachable")
}
}
// Prev implements FragmentIterator.
func (i *BoundedIter) Prev() *Span {
switch i.pos {
case posAtLowerLimit:
// Already exhausted.
return nil
case posAtIterSpan:
// If the span at the underlying iterator position extends to or beyond
// the lower bound, we can avoid advancing because the previous span is
// necessarily out of bounds.
if i.iterSpan != nil && i.lower != nil && i.cmp(i.iterSpan.Start, i.lower) <= 0 {
i.pos = posAtLowerLimit
return nil
}
// Similarly, if the span extends to or beyond the current prefix and
// we're in prefix iteration mode, we can avoid advancing.
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(i.iterSpan.Start)
if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 {
i.pos = posAtLowerLimit
return nil
}
}
return i.checkBackwardBound(i.checkPrefixSpanEnd(i.iter.Prev()))
case posAtUpperLimit:
// The BoundedIter had previously returned nil, because it knew from
// i.iterSpan's bounds that there was no next span. To Prev, we only
// need to return the current iter span and reset i.pos to reflect that
// we're no longer positioned at the limit.
i.pos = posAtIterSpan
return i.iterSpan
default:
panic("unreachable")
}
}
// Error implements FragmentIterator.
func (i *BoundedIter) Error() error {
return i.iter.Error()
}
// Close implements FragmentIterator.
func (i *BoundedIter) Close() error {
return i.iter.Close()
}
// SetBounds modifies the FragmentIterator's bounds.
func (i *BoundedIter) SetBounds(lower, upper []byte) {
i.lower, i.upper = lower, upper
}
func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(span.Start)
if i.cmp(span.Start[:si], *i.prefix) > 0 {
// This span starts at a prefix that sorts after our current prefix.
span = nil
}
}
return span
}
// checkForwardBound enforces the upper bound, returning nil if the provided
// span is wholly outside the upper bound. It also updates i.pos and i.iterSpan
// to reflect the new iterator position.
func (i *BoundedIter) checkForwardBound(span *Span) *Span {
// Compare to the upper bound.
if span != nil && i.upper != nil && i.cmp(span.Start, i.upper) >= 0 {
span = nil
}
i.iterSpan = span
if i.pos != posAtIterSpan {
i.pos = posAtIterSpan
}
return span
}
func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && i.hasPrefix != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
// This span ends before the current prefix.
span = nil
}
return span
}
// checkBackward enforces the lower bound, returning nil if the provided span is
// wholly outside the lower bound. It also updates i.pos and i.iterSpan to
// reflect the new iterator position.
func (i *BoundedIter) checkBackwardBound(span *Span) *Span {
// Compare to the lower bound.
if span != nil && i.lower != nil && i.cmp(span.End, i.lower) <= 0 {
span = nil
}
i.iterSpan = span
if i.pos != posAtIterSpan {
i.pos = posAtIterSpan
}
return span
}

505
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored Normal file
View File

@@ -0,0 +1,505 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"bytes"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/invariants"
)
// bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
// that DefragmentingIter will reuse. Buffers larger than this will be
// discarded and reallocated as necessary.
const bufferReuseMaxCapacity = 10 << 10 // 10 KB
// keysReuseMaxCapacity is the maximum capacity of a []keyspan.Key buffer that
// DefragmentingIter will reuse. Buffers larger than this will be discarded and
// reallocated as necessary.
const keysReuseMaxCapacity = 100
// DefragmentMethod configures the defragmentation performed by the
// DefragmentingIter.
type DefragmentMethod interface {
// ShouldDefragment takes two abutting spans and returns whether the two
// spans should be combined into a single, defragmented Span.
ShouldDefragment(equal base.Equal, left, right *Span) bool
}
// The DefragmentMethodFunc type is an adapter to allow the use of ordinary
// functions as DefragmentMethods. If f is a function with the appropriate
// signature, DefragmentMethodFunc(f) is a DefragmentMethod that calls f.
type DefragmentMethodFunc func(equal base.Equal, left, right *Span) bool
// ShouldDefragment calls f(equal, left, right).
func (f DefragmentMethodFunc) ShouldDefragment(equal base.Equal, left, right *Span) bool {
return f(equal, left, right)
}
// DefragmentInternal configures a DefragmentingIter to defragment spans
// only if they have identical keys. It requires spans' keys to be sorted in
// trailer descending order.
//
// This defragmenting method is intended for use in compactions that may see
// internal range keys fragments that may now be joined, because the state that
// required their fragmentation has been dropped.
var DefragmentInternal DefragmentMethod = DefragmentMethodFunc(func(equal base.Equal, a, b *Span) bool {
if a.KeysOrder != ByTrailerDesc || b.KeysOrder != ByTrailerDesc {
panic("pebble: span keys unexpectedly not in trailer descending order")
}
if len(a.Keys) != len(b.Keys) {
return false
}
for i := range a.Keys {
if a.Keys[i].Trailer != b.Keys[i].Trailer {
return false
}
if !equal(a.Keys[i].Suffix, b.Keys[i].Suffix) {
return false
}
if !bytes.Equal(a.Keys[i].Value, b.Keys[i].Value) {
return false
}
}
return true
})
// DefragmentReducer merges the current and next Key slices, returning a new Key
// slice.
//
// Implementations should modify and return `cur` to save on allocations, or
// consider allocating a new slice, as the `cur` slice may be retained by the
// DefragmentingIter and mutated. The `next` slice must not be mutated.
//
// The incoming slices are sorted by (SeqNum, Kind) descending. The output slice
// must also have this sort order.
type DefragmentReducer func(cur, next []Key) []Key
// StaticDefragmentReducer is a no-op DefragmentReducer that simply returns the
// current key slice, effectively retaining the first set of keys encountered
// for a defragmented span.
//
// This reducer can be used, for example, when the set of Keys for each Span
// being reduced is not expected to change, and therefore the keys from the
// first span encountered can be used without considering keys in subsequent
// spans.
var StaticDefragmentReducer DefragmentReducer = func(cur, _ []Key) []Key {
return cur
}
// iterPos is an enum indicating the position of the defragmenting iter's
// wrapped iter. The defragmenting iter must look ahead or behind when
// defragmenting forward or backwards respectively, and this enum records that
// current position.
type iterPos int8
const (
iterPosPrev iterPos = -1
iterPosCurr iterPos = 0
iterPosNext iterPos = +1
)
// DefragmentingIter wraps a key span iterator, defragmenting physical
// fragmentation during iteration.
//
// During flushes and compactions, keys applied over a span may be split at
// sstable boundaries. This fragmentation can produce internal key bounds that
// do not match any of the bounds ever supplied to a user operation. This
// physical fragmentation is necessary to avoid excessively wide sstables.
//
// The defragmenting iterator undoes this physical fragmentation, joining spans
// with abutting bounds and equal state. The defragmenting iterator takes a
// DefragmentMethod to determine what is "equal state" for a span. The
// DefragmentMethod is a function type, allowing arbitrary comparisons between
// Span keys.
//
// Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may
// land on a physical fragment in the middle of several fragments that must be
// defragmented. A seek that lands in a fragment straddling the seek key must
// first degfragment in the opposite direction of iteration to find the
// beginning of the defragmented span, and then defragments in the iteration
// direction, ensuring it's found a whole defragmented span.
type DefragmentingIter struct {
// DefragmentingBuffers holds buffers used for copying iterator state.
*DefragmentingBuffers
comparer *base.Comparer
equal base.Equal
iter FragmentIterator
iterSpan *Span
iterPos iterPos
// curr holds the span at the current iterator position.
curr Span
// method is a comparison function for two spans. method is called when two
// spans are abutting to determine whether they may be defragmented.
// method does not itself check for adjacency for the two spans.
method DefragmentMethod
// reduce is the reducer function used to collect Keys across all spans that
// constitute a defragmented span.
reduce DefragmentReducer
}
// DefragmentingBuffers holds buffers used for copying iterator state.
type DefragmentingBuffers struct {
// currBuf is a buffer for use when copying user keys for curr. currBuf is
// cleared between positioning methods.
currBuf bytealloc.A
// keysBuf is a buffer for use when copying Keys for DefragmentingIter.curr.
keysBuf []Key
// keyBuf is a buffer specifically for the defragmented start key when
// defragmenting backwards or the defragmented end key when defragmenting
// forwards. These bounds are overwritten repeatedly during defragmentation,
// and the defragmentation routines overwrite keyBuf repeatedly to store
// these extended bounds.
keyBuf []byte
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *DefragmentingBuffers) PrepareForReuse() {
if cap(bufs.currBuf) > bufferReuseMaxCapacity {
bufs.currBuf = nil
}
if cap(bufs.keyBuf) > bufferReuseMaxCapacity {
bufs.keyBuf = nil
}
if cap(bufs.keysBuf) > keysReuseMaxCapacity {
bufs.keysBuf = nil
}
}
// Assert that *DefragmentingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*DefragmentingIter)(nil)
// Init initializes the defragmenting iter using the provided defragment
// method.
func (i *DefragmentingIter) Init(
comparer *base.Comparer,
iter FragmentIterator,
equal DefragmentMethod,
reducer DefragmentReducer,
bufs *DefragmentingBuffers,
) {
*i = DefragmentingIter{
DefragmentingBuffers: bufs,
comparer: comparer,
equal: comparer.Equal,
iter: iter,
method: equal,
reduce: reducer,
}
}
// Error returns any accumulated error.
func (i *DefragmentingIter) Error() error {
return i.iter.Error()
}
// Close closes the underlying iterators.
func (i *DefragmentingIter) Close() error {
return i.iter.Close()
}
// SeekGE moves the iterator to the first span covering a key greater than or
// equal to the given key. This is equivalent to seeking to the first span with
// an end key greater than the given key.
func (i *DefragmentingIter) SeekGE(key []byte) *Span {
i.iterSpan = i.iter.SeekGE(key)
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
} else if i.iterSpan.Empty() {
i.iterPos = iterPosCurr
return i.iterSpan
}
// If the span starts strictly after key, we know there mustn't be an
// earlier span that ends at i.iterSpan.Start, otherwise i.iter would've
// returned that span instead.
if i.comparer.Compare(i.iterSpan.Start, key) > 0 {
return i.defragmentForward()
}
// The span we landed on has a Start bound ≤ key. There may be additional
// fragments before this span. Defragment backward to find the start of the
// defragmented span.
i.defragmentBackward()
if i.iterPos == iterPosPrev {
// Next once back onto the span.
i.iterSpan = i.iter.Next()
}
// Defragment the full span from its start.
return i.defragmentForward()
}
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
func (i *DefragmentingIter) SeekLT(key []byte) *Span {
i.iterSpan = i.iter.SeekLT(key)
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
} else if i.iterSpan.Empty() {
i.iterPos = iterPosCurr
return i.iterSpan
}
// If the span ends strictly before key, we know there mustn't be a later
// span that starts at i.iterSpan.End, otherwise i.iter would've returned
// that span instead.
if i.comparer.Compare(i.iterSpan.End, key) < 0 {
return i.defragmentBackward()
}
// The span we landed on has a End bound ≥ key. There may be additional
// fragments after this span. Defragment forward to find the end of the
// defragmented span.
i.defragmentForward()
if i.iterPos == iterPosNext {
// Prev once back onto the span.
i.iterSpan = i.iter.Prev()
}
// Defragment the full span from its end.
return i.defragmentBackward()
}
// First seeks the iterator to the first span and returns it.
func (i *DefragmentingIter) First() *Span {
i.iterSpan = i.iter.First()
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentForward()
}
// Last seeks the iterator to the last span and returns it.
func (i *DefragmentingIter) Last() *Span {
i.iterSpan = i.iter.Last()
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentBackward()
}
// Next advances to the next span and returns it.
func (i *DefragmentingIter) Next() *Span {
switch i.iterPos {
case iterPosPrev:
// Switching directions; The iterator is currently positioned over the
// last span of the previous set of fragments. In the below diagram,
// the iterator is positioned over the last span that contributes to
// the defragmented x position. We want to be positioned over the first
// span that contributes to the z position.
//
// x x x y y y z z z
// ^ ^
// old new
//
// Next once to move onto y, defragment forward to land on the first z
// position.
i.iterSpan = i.iter.Next()
if invariants.Enabled && i.iterSpan == nil {
panic("pebble: invariant violation: no next span while switching directions")
}
// We're now positioned on the first span that was defragmented into the
// current iterator position. Skip over the rest of the current iterator
// position's constitutent fragments. In the above example, this would
// land on the first 'z'.
i.defragmentForward()
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
// Now that we're positioned over the first of the next set of
// fragments, defragment forward.
return i.defragmentForward()
case iterPosCurr:
// iterPosCurr is only used when the iter is exhausted or when the iterator
// is at an empty span.
if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
}
i.iterSpan = i.iter.Next()
if i.iterSpan == nil {
return nil
}
return i.defragmentForward()
case iterPosNext:
// Already at the next span.
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentForward()
default:
panic("unreachable")
}
}
// Prev steps back to the previous span and returns it.
func (i *DefragmentingIter) Prev() *Span {
switch i.iterPos {
case iterPosPrev:
// Already at the previous span.
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentBackward()
case iterPosCurr:
// iterPosCurr is only used when the iter is exhausted or when the iterator
// is at an empty span.
if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
}
i.iterSpan = i.iter.Prev()
if i.iterSpan == nil {
return nil
}
return i.defragmentBackward()
case iterPosNext:
// Switching directions; The iterator is currently positioned over the
// first fragment of the next set of fragments. In the below diagram,
// the iterator is positioned over the first span that contributes to
// the defragmented z position. We want to be positioned over the last
// span that contributes to the x position.
//
// x x x y y y z z z
// ^ ^
// new old
//
// Prev once to move onto y, defragment backward to land on the last x
// position.
i.iterSpan = i.iter.Prev()
if invariants.Enabled && i.iterSpan == nil {
panic("pebble: invariant violation: no previous span while switching directions")
}
// We're now positioned on the last span that was defragmented into the
// current iterator position. Skip over the rest of the current iterator
// position's constitutent fragments. In the above example, this would
// land on the last 'x'.
i.defragmentBackward()
// Now that we're positioned over the last of the prev set of
// fragments, defragment backward.
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentBackward()
default:
panic("unreachable")
}
}
// checkEqual checks the two spans for logical equivalence. It uses the passed-in
// DefragmentMethod and ensures both spans are NOT empty; not defragmenting empty
// spans is an optimization that lets us load fewer sstable blocks.
func (i *DefragmentingIter) checkEqual(left, right *Span) bool {
return (!left.Empty() && !right.Empty()) && i.method.ShouldDefragment(i.equal, i.iterSpan, &i.curr)
}
// defragmentForward defragments spans in the forward direction, starting from
// i.iter's current position. The span at the current position must be non-nil,
// but may be Empty().
func (i *DefragmentingIter) defragmentForward() *Span {
if i.iterSpan.Empty() {
// An empty span will never be equal to another span; see checkEqual for
// why. To avoid loading non-empty range keys further ahead by calling Next,
// return early.
i.iterPos = iterPosCurr
return i.iterSpan
}
i.saveCurrent()
i.iterPos = iterPosNext
i.iterSpan = i.iter.Next()
for i.iterSpan != nil {
if !i.equal(i.curr.End, i.iterSpan.Start) {
// Not a continuation.
break
}
if !i.checkEqual(i.iterSpan, &i.curr) {
// Not a continuation.
break
}
i.keyBuf = append(i.keyBuf[:0], i.iterSpan.End...)
i.curr.End = i.keyBuf
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Next()
}
i.curr.Keys = i.keysBuf
return &i.curr
}
// defragmentBackward defragments spans in the backward direction, starting from
// i.iter's current position. The span at the current position must be non-nil,
// but may be Empty().
func (i *DefragmentingIter) defragmentBackward() *Span {
if i.iterSpan.Empty() {
// An empty span will never be equal to another span; see checkEqual for
// why. To avoid loading non-empty range keys further ahead by calling Next,
// return early.
i.iterPos = iterPosCurr
return i.iterSpan
}
i.saveCurrent()
i.iterPos = iterPosPrev
i.iterSpan = i.iter.Prev()
for i.iterSpan != nil {
if !i.equal(i.curr.Start, i.iterSpan.End) {
// Not a continuation.
break
}
if !i.checkEqual(i.iterSpan, &i.curr) {
// Not a continuation.
break
}
i.keyBuf = append(i.keyBuf[:0], i.iterSpan.Start...)
i.curr.Start = i.keyBuf
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Prev()
}
i.curr.Keys = i.keysBuf
return &i.curr
}
func (i *DefragmentingIter) saveCurrent() {
i.currBuf.Reset()
i.keysBuf = i.keysBuf[:0]
i.keyBuf = i.keyBuf[:0]
if i.iterSpan == nil {
return
}
i.curr = Span{
Start: i.saveBytes(i.iterSpan.Start),
End: i.saveBytes(i.iterSpan.End),
KeysOrder: i.iterSpan.KeysOrder,
}
for j := range i.iterSpan.Keys {
i.keysBuf = append(i.keysBuf, Key{
Trailer: i.iterSpan.Keys[j].Trailer,
Suffix: i.saveBytes(i.iterSpan.Keys[j].Suffix),
Value: i.saveBytes(i.iterSpan.Keys[j].Value),
})
}
i.curr.Keys = i.keysBuf
}
func (i *DefragmentingIter) saveBytes(b []byte) []byte {
if b == nil {
return nil
}
i.currBuf, b = i.currBuf.Copy(b)
return b
}

13
vendor/github.com/cockroachdb/pebble/internal/keyspan/doc.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Package keyspan provides facilities for sorting, fragmenting and
// iterating over spans of user keys.
//
// A Span represents a range of user key space with an inclusive start
// key and exclusive end key. A span may hold any number of Keys which are
// applied over the entirety of the span's keyspace.
//
// Spans are used within Pebble as an in-memory representation of range
// deletion tombstones, and range key sets, unsets and deletes. Spans
// are fragmented at overlapping key boundaries by the Fragmenter type.
// This package's various iteration facilities require these
// non-overlapping fragmented spans.
package keyspan

98
vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go generated vendored Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
// FilterFunc defines a transform from the input Span into the output Span. The
// function returns true if the Span should be returned by the iterator, and
// false if the Span should be skipped. The FilterFunc is permitted to mutate
// the output Span, for example, to elice certain keys, or update the Span's
// bounds if so desired. The output Span's Keys slice may be reused to reduce
// allocations.
type FilterFunc func(in *Span, out *Span) (keep bool)
// filteringIter is a FragmentIterator that uses a FilterFunc to select which
// Spans from the input iterator are returned in the output.
//
// A note on Span lifetimes: as the FilterFunc reuses a Span with a mutable
// slice of Keys to reduce allocations, Spans returned by this iterator are only
// valid until the next relative or absolute positioning method is called.
type filteringIter struct {
iter FragmentIterator
filterFn FilterFunc
// span is a mutable Span passed to the filterFn. The filterFn is free to
// mutate this Span. The slice of Keys in the Span is reused with every call
// to the filterFn.
span Span
}
var _ FragmentIterator = (*filteringIter)(nil)
// Filter returns a new filteringIter that will filter the Spans from the
// provided child iterator using the provided FilterFunc.
func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter}
}
// SeekGE implements FragmentIterator.
func (i *filteringIter) SeekGE(key []byte) *Span {
return i.filter(i.iter.SeekGE(key), +1)
}
// SeekLT implements FragmentIterator.
func (i *filteringIter) SeekLT(key []byte) *Span {
return i.filter(i.iter.SeekLT(key), -1)
}
// First implements FragmentIterator.
func (i *filteringIter) First() *Span {
return i.filter(i.iter.First(), +1)
}
// Last implements FragmentIterator.
func (i *filteringIter) Last() *Span {
return i.filter(i.iter.Last(), -1)
}
// Next implements FragmentIterator.
func (i *filteringIter) Next() *Span {
return i.filter(i.iter.Next(), +1)
}
// Prev implements FragmentIterator.
func (i *filteringIter) Prev() *Span {
return i.filter(i.iter.Prev(), -1)
}
// Error implements FragmentIterator.
func (i *filteringIter) Error() error {
return i.iter.Error()
}
// Close implements FragmentIterator.
func (i *filteringIter) Close() error {
return i.iter.Close()
}
// filter uses the filterFn (if configured) to filter and possibly mutate the
// given Span. If the current Span is to be skipped, the iterator continues
// iterating in the given direction until it lands on a Span that should be
// returned, or the iterator becomes invalid.
func (i *filteringIter) filter(span *Span, dir int8) *Span {
if i.filterFn == nil {
return span
}
for i.Error() == nil && span != nil {
if keep := i.filterFn(span, &i.span); keep {
return &i.span
}
if dir == +1 {
span = i.iter.Next()
} else {
span = i.iter.Prev()
}
}
return span
}

483
vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go generated vendored Normal file
View File

@@ -0,0 +1,483 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"sort"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
type spansByStartKey struct {
cmp base.Compare
buf []Span
}
func (v *spansByStartKey) Len() int { return len(v.buf) }
func (v *spansByStartKey) Less(i, j int) bool {
return v.cmp(v.buf[i].Start, v.buf[j].Start) < 0
}
func (v *spansByStartKey) Swap(i, j int) {
v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
}
type spansByEndKey struct {
cmp base.Compare
buf []Span
}
func (v *spansByEndKey) Len() int { return len(v.buf) }
func (v *spansByEndKey) Less(i, j int) bool {
return v.cmp(v.buf[i].End, v.buf[j].End) < 0
}
func (v *spansByEndKey) Swap(i, j int) {
v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
}
// keysBySeqNumKind sorts spans by the start key's sequence number in
// descending order. If two spans have equal sequence number, they're compared
// by key kind in descending order. This ordering matches the ordering of
// base.InternalCompare among keys with matching user keys.
type keysBySeqNumKind []Key
func (v *keysBySeqNumKind) Len() int { return len(*v) }
func (v *keysBySeqNumKind) Less(i, j int) bool { return (*v)[i].Trailer > (*v)[j].Trailer }
func (v *keysBySeqNumKind) Swap(i, j int) { (*v)[i], (*v)[j] = (*v)[j], (*v)[i] }
// Sort the spans by start key. This is the ordering required by the
// Fragmenter. Usually spans are naturally sorted by their start key,
// but that isn't true for range deletion tombstones in the legacy
// range-del-v1 block format.
func Sort(cmp base.Compare, spans []Span) {
sorter := spansByStartKey{
cmp: cmp,
buf: spans,
}
sort.Sort(&sorter)
}
// Fragmenter fragments a set of spans such that overlapping spans are
// split at their overlap points. The fragmented spans are output to the
// supplied Output function.
type Fragmenter struct {
Cmp base.Compare
Format base.FormatKey
// Emit is called to emit a fragmented span and its keys. Every key defined
// within the emitted Span applies to the entirety of the Span's key span.
// Keys are ordered in decreasing order of their sequence numbers, and if
// equal, decreasing order of key kind.
Emit func(Span)
// pending contains the list of pending fragments that have not been
// flushed to the block writer. Note that the spans have not been
// fragmented on the end keys yet. That happens as the spans are
// flushed. All pending spans have the same Start.
pending []Span
// doneBuf is used to buffer completed span fragments when flushing to a
// specific key (e.g. TruncateAndFlushTo). It is cached in the Fragmenter to
// allow reuse.
doneBuf []Span
// sortBuf is used to sort fragments by end key when flushing.
sortBuf spansByEndKey
// flushBuf is used to sort keys by (seqnum,kind) before emitting.
flushBuf keysBySeqNumKind
// flushedKey is the key that fragments have been flushed up to. Any
// additional spans added to the fragmenter must have a start key >=
// flushedKey. A nil value indicates flushedKey has not been set.
flushedKey []byte
finished bool
}
func (f *Fragmenter) checkInvariants(buf []Span) {
for i := 1; i < len(buf); i++ {
if f.Cmp(buf[i].Start, buf[i].End) >= 0 {
panic(fmt.Sprintf("pebble: empty pending span invariant violated: %s", buf[i]))
}
if f.Cmp(buf[i-1].Start, buf[i].Start) != 0 {
panic(fmt.Sprintf("pebble: pending span invariant violated: %s %s",
f.Format(buf[i-1].Start), f.Format(buf[i].Start)))
}
}
}
// Add adds a span to the fragmenter. Spans may overlap and the
// fragmenter will internally split them. The spans must be presented in
// increasing start key order. That is, Add must be called with a series
// of spans like:
//
// a---e
// c---g
// c-----i
// j---n
// j-l
//
// We need to fragment the spans at overlap points. In the above
// example, we'd create:
//
// a-c-e
// c-e-g
// c-e-g-i
// j-l-n
// j-l
//
// The fragments need to be output sorted by start key, and for equal start
// keys, sorted by descending sequence number. This last part requires a mild
// bit of care as the fragments are not created in descending sequence number
// order.
//
// Once a start key has been seen, we know that we'll never see a smaller
// start key and can thus flush all of the fragments that lie before that
// start key.
//
// Walking through the example above, we start with:
//
// a---e
//
// Next we add [c,g) resulting in:
//
// a-c-e
// c---g
//
// The fragment [a,c) is flushed leaving the pending spans as:
//
// c-e
// c---g
//
// The next span is [c,i):
//
// c-e
// c---g
// c-----i
//
// No fragments are flushed. The next span is [j,n):
//
// c-e
// c---g
// c-----i
// j---n
//
// The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments
// by their end key, then split the fragments on the end keys:
//
// c-e
// c-e-g
// c-e---i
//
// The [c,e) fragments all get flushed leaving:
//
// e-g
// e---i
//
// This process continues until there are no more fragments to flush.
//
// WARNING: the slices backing Start, End, Keys, Key.Suffix and Key.Value are
// all retained after this method returns and should not be modified. This is
// safe for spans that are added from a memtable or batch. It is partially
// unsafe for a span read from an sstable. Specifically, the Keys slice of a
// Span returned during sstable iteration is only valid until the next iterator
// operation. The stability of the user keys depend on whether the block is
// prefix compressed, and in practice Pebble never prefix compresses range
// deletion and range key blocks, so these keys are stable. Because of this key
// stability, typically callers only need to perform a shallow clone of the Span
// before Add-ing it to the fragmenter.
//
// Add requires the provided span's keys are sorted in Trailer descending order.
func (f *Fragmenter) Add(s Span) {
if f.finished {
panic("pebble: span fragmenter already finished")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span keys unexpectedly not in trailer descending order")
}
if f.flushedKey != nil {
switch c := f.Cmp(s.Start, f.flushedKey); {
case c < 0:
panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
f.Format(s.Start), f.Format(f.flushedKey)))
}
}
if f.Cmp(s.Start, s.End) >= 0 {
// An empty span, we can ignore it.
return
}
if invariants.RaceEnabled {
f.checkInvariants(f.pending)
defer func() { f.checkInvariants(f.pending) }()
}
if len(f.pending) > 0 {
// Since all of the pending spans have the same start key, we only need
// to compare against the first one.
switch c := f.Cmp(f.pending[0].Start, s.Start); {
case c > 0:
panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
f.Format(f.pending[0].Start), f.Format(s.Start)))
case c == 0:
// The new span has the same start key as the existing pending
// spans. Add it to the pending buffer.
f.pending = append(f.pending, s)
return
}
// At this point we know that the new start key is greater than the pending
// spans start keys.
f.truncateAndFlush(s.Start)
}
f.pending = append(f.pending, s)
}
// Cover is returned by Framenter.Covers and describes a span's relationship to
// a key at a particular snapshot.
type Cover int8
const (
// NoCover indicates the tested key does not fall within the span's bounds,
// or the span contains no keys with sequence numbers higher than the key's.
NoCover Cover = iota
// CoversInvisibly indicates the tested key does fall within the span's
// bounds and the span contains at least one key with a higher sequence
// number, but none visible at the provided snapshot.
CoversInvisibly
// CoversVisibly indicates the tested key does fall within the span's
// bounds, and the span constains at least one key with a sequence number
// higher than the key's sequence number that is visible at the provided
// snapshot.
CoversVisibly
)
// Covers returns an enum indicating whether the specified key is covered by one
// of the pending keys. The provided key must be consistent with the ordering of
// the spans. That is, it is invalid to specify a key here that is out of order
// with the span start keys passed to Add.
func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) Cover {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if len(f.pending) == 0 {
return NoCover
}
if f.Cmp(f.pending[0].Start, key.UserKey) > 0 {
panic(fmt.Sprintf("pebble: keys must be in order: %s > %s",
f.Format(f.pending[0].Start), key.Pretty(f.Format)))
}
cover := NoCover
seqNum := key.SeqNum()
for _, s := range f.pending {
if f.Cmp(key.UserKey, s.End) < 0 {
// NB: A range deletion tombstone does not delete a point operation
// at the same sequence number, and broadly a span is not considered
// to cover a point operation at the same sequence number.
for i := range s.Keys {
if kseq := s.Keys[i].SeqNum(); kseq > seqNum {
// This key from the span has a higher sequence number than
// `key`. It covers `key`, although the span's key might not
// be visible if its snapshot is too high.
//
// Batch keys are always be visible.
if kseq < snapshot || kseq&base.InternalKeySeqNumBatch != 0 {
return CoversVisibly
}
// s.Keys[i] is not visible.
cover = CoversInvisibly
}
}
}
}
return cover
}
// Empty returns true if all fragments added so far have finished flushing.
func (f *Fragmenter) Empty() bool {
return f.finished || len(f.pending) == 0
}
// TruncateAndFlushTo flushes all of the fragments with a start key <= key,
// truncating spans to the specified end key. Used during compaction to force
// emitting of spans which straddle an sstable boundary. Consider
// the scenario:
//
// a---------k#10
// f#8
// f#7
//
// Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will
// flush this span:
//
// a-------g#10
// f#8
// f#7
//
// And leave this one in f.pending:
//
// g----k#10
//
// WARNING: The fragmenter could hold on to the specified end key. Ensure it's
// a safe byte slice that could outlast the current sstable output, and one
// that will never be modified.
func (f *Fragmenter) TruncateAndFlushTo(key []byte) {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if f.flushedKey != nil {
switch c := f.Cmp(key, f.flushedKey); {
case c < 0:
panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
f.Format(key), f.Format(f.flushedKey)))
}
}
if invariants.RaceEnabled {
f.checkInvariants(f.pending)
defer func() { f.checkInvariants(f.pending) }()
}
if len(f.pending) > 0 {
// Since all of the pending spans have the same start key, we only need
// to compare against the first one.
switch c := f.Cmp(f.pending[0].Start, key); {
case c > 0:
panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
f.Format(f.pending[0].Start), f.Format(key)))
case c == 0:
return
}
}
f.truncateAndFlush(key)
}
// Start returns the start key of the first span in the pending buffer, or nil
// if there are no pending spans. The start key of all pending spans is the same
// as that of the first one.
func (f *Fragmenter) Start() []byte {
if len(f.pending) > 0 {
return f.pending[0].Start
}
return nil
}
// Flushes all pending spans up to key (exclusive).
//
// WARNING: The specified key is stored without making a copy, so all callers
// must ensure it is safe.
func (f *Fragmenter) truncateAndFlush(key []byte) {
f.flushedKey = append(f.flushedKey[:0], key...)
done := f.doneBuf[:0]
pending := f.pending
f.pending = f.pending[:0]
// pending and f.pending share the same underlying storage. As we iterate
// over pending we append to f.pending, but only one entry is appended in
// each iteration, after we have read the entry being overwritten.
for _, s := range pending {
if f.Cmp(key, s.End) < 0 {
// s: a--+--e
// new: c------
if f.Cmp(s.Start, key) < 0 {
done = append(done, Span{
Start: s.Start,
End: key,
Keys: s.Keys,
})
}
f.pending = append(f.pending, Span{
Start: key,
End: s.End,
Keys: s.Keys,
})
} else {
// s: a-----e
// new: e----
done = append(done, s)
}
}
f.doneBuf = done[:0]
f.flush(done, nil)
}
// flush a group of range spans to the block. The spans are required to all have
// the same start key. We flush all span fragments until startKey > lastKey. If
// lastKey is nil, all span fragments are flushed. The specification of a
// non-nil lastKey occurs for range deletion tombstones during compaction where
// we want to flush (but not truncate) all range tombstones that start at or
// before the first key in the next sstable. Consider:
//
// a---e#10
// a------h#9
//
// If a compaction splits the sstables at key c we want the first sstable to
// contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally
// produce a tombstone [e,h)#9, but we don't need to output that tombstone to
// the first sstable.
func (f *Fragmenter) flush(buf []Span, lastKey []byte) {
if invariants.RaceEnabled {
f.checkInvariants(buf)
}
// Sort the spans by end key. This will allow us to walk over the spans and
// easily determine the next split point (the smallest end-key).
f.sortBuf.cmp = f.Cmp
f.sortBuf.buf = buf
sort.Sort(&f.sortBuf)
// Loop over the spans, splitting by end key.
for len(buf) > 0 {
// A prefix of spans will end at split. remove represents the count of
// that prefix.
remove := 1
split := buf[0].End
f.flushBuf = append(f.flushBuf[:0], buf[0].Keys...)
for i := 1; i < len(buf); i++ {
if f.Cmp(split, buf[i].End) == 0 {
remove++
}
f.flushBuf = append(f.flushBuf, buf[i].Keys...)
}
sort.Sort(&f.flushBuf)
f.Emit(Span{
Start: buf[0].Start,
End: split,
// Copy the sorted keys to a new slice.
//
// This allocation is an unfortunate side effect of the Fragmenter and
// the expectation that the spans it produces are available in-memory
// indefinitely.
//
// Eventually, we should be able to replace the fragmenter with the
// keyspan.MergingIter which will perform just-in-time
// fragmentation, and only guaranteeing the memory lifetime for the
// current span. The MergingIter fragments while only needing to
// access one Span per level. It only accesses the Span at the
// current position for each level. During compactions, we can write
// these spans to sstables without retaining previous Spans.
Keys: append([]Key(nil), f.flushBuf...),
})
if lastKey != nil && f.Cmp(split, lastKey) > 0 {
break
}
// Adjust the start key for every remaining span.
buf = buf[remove:]
for i := range buf {
buf[i].Start = split
}
}
}
// Finish flushes any remaining fragments to the output. It is an error to call
// this if any other spans will be added.
func (f *Fragmenter) Finish() {
if f.finished {
panic("pebble: span fragmenter already finished")
}
f.flush(f.pending, nil)
f.finished = true
}

53
vendor/github.com/cockroachdb/pebble/internal/keyspan/get.go generated vendored Normal file
View File

@@ -0,0 +1,53 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Get returns the newest span that contains the target key. If no span
// contains the target key, an empty span is returned. The snapshot
// parameter controls the visibility of spans (only spans older than the
// snapshot sequence number are visible). The iterator must contain
// fragmented spans: no span may overlap another.
func Get(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were
// to use SeekGE then the search key `c` would land on the span
// [e,i) and we'd have to backtrack. The one complexity here is what
// happens for the search key `e`. In that case SeekLT will land us
// on the span [a,e) and we'll have to move forward.
iterSpan := iter.SeekLT(key)
if iterSpan == nil {
iterSpan = iter.Next()
if iterSpan == nil {
// The iterator is empty.
return nil
}
if cmp(key, iterSpan.Start) < 0 {
// The search key lies before the first span.
return nil
}
}
// Invariant: key > iterSpan.Start
if cmp(key, iterSpan.End) >= 0 {
// The current span lies before the search key. Advance the iterator
// once to potentially land on a key with a start key exactly equal to
// key. (See the comment at the beginning of this function.)
iterSpan = iter.Next()
if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
// We've run out of spans or we've moved on to a span which
// starts after our search key.
return nil
}
}
return iterSpan
}

1046
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

118
vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go generated vendored Normal file
View File

@@ -0,0 +1,118 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// InternalIteratorShim is a temporary iterator type used as a shim between
// keyspan.MergingIter and base.InternalIterator. It's used temporarily for
// range deletions during compactions, allowing range deletions to be
// interleaved by a compaction input iterator.
//
// TODO(jackson): This type should be removed, and the usages converted to using
// an InterleavingIterator type that interleaves keyspan.Spans from a
// keyspan.FragmentIterator with point keys.
type InternalIteratorShim struct {
miter MergingIter
mbufs MergingBuffers
span *Span
iterKey base.InternalKey
}
// Assert that InternalIteratorShim implements InternalIterator.
var _ base.InternalIterator = &InternalIteratorShim{}
// Init initializes the internal iterator shim to merge the provided fragment
// iterators.
func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) {
i.miter.Init(cmp, noopTransform, &i.mbufs, iters...)
}
// Span returns the span containing the full set of keys over the key span at
// the current iterator position.
func (i *InternalIteratorShim) Span() *Span {
return i.span
}
// SeekGE implements (base.InternalIterator).SeekGE.
func (i *InternalIteratorShim) SeekGE(
key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// SeekPrefixGE implements (base.InternalIterator).SeekPrefixGE.
func (i *InternalIteratorShim) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// SeekLT implements (base.InternalIterator).SeekLT.
func (i *InternalIteratorShim) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// First implements (base.InternalIterator).First.
func (i *InternalIteratorShim) First() (*base.InternalKey, base.LazyValue) {
i.span = i.miter.First()
for i.span != nil && i.span.Empty() {
i.span = i.miter.Next()
}
if i.span == nil {
return nil, base.LazyValue{}
}
i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
return &i.iterKey, base.MakeInPlaceValue(i.span.End)
}
// Last implements (base.InternalIterator).Last.
func (i *InternalIteratorShim) Last() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Next implements (base.InternalIterator).Next.
func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) {
i.span = i.miter.Next()
for i.span != nil && i.span.Empty() {
i.span = i.miter.Next()
}
if i.span == nil {
return nil, base.LazyValue{}
}
i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
return &i.iterKey, base.MakeInPlaceValue(i.span.End)
}
// NextPrefix implements (base.InternalIterator).NextPrefix.
func (i *InternalIteratorShim) NextPrefix([]byte) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Prev implements (base.InternalIterator).Prev.
func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Error implements (base.InternalIterator).Error.
func (i *InternalIteratorShim) Error() error {
return i.miter.Error()
}
// Close implements (base.InternalIterator).Close.
func (i *InternalIteratorShim) Close() error {
return i.miter.Close()
}
// SetBounds implements (base.InternalIterator).SetBounds.
func (i *InternalIteratorShim) SetBounds(lower, upper []byte) {
}
// String implements fmt.Stringer.
func (i *InternalIteratorShim) String() string {
return i.miter.String()
}

218
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored Normal file
View File

@@ -0,0 +1,218 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/manifest"
)
// FragmentIterator defines an iterator interface over spans. The spans
// surfaced by a FragmentIterator must be non-overlapping. This is achieved by
// fragmenting spans at overlap points (see Fragmenter).
//
// A Span returned by a FragmentIterator is only valid until the next
// positioning method. Some implementations (eg, keyspan.Iter) may provide
// longer lifetimes but implementations need only guarantee stability until the
// next positioning method.
type FragmentIterator interface {
// SeekGE moves the iterator to the first span covering a key greater than
// or equal to the given key. This is equivalent to seeking to the first
// span with an end key greater than the given key.
SeekGE(key []byte) *Span
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
SeekLT(key []byte) *Span
// First moves the iterator to the first span.
First() *Span
// Last moves the iterator to the last span.
Last() *Span
// Next moves the iterator to the next span.
//
// It is valid to call Next when the iterator is positioned before the first
// key/value pair due to either a prior call to SeekLT or Prev which
// returned an invalid span. It is not allowed to call Next when the
// previous call to SeekGE, SeekPrefixGE or Next returned an invalid span.
Next() *Span
// Prev moves the iterator to the previous span.
//
// It is valid to call Prev when the iterator is positioned after the last
// key/value pair due to either a prior call to SeekGE or Next which
// returned an invalid span. It is not allowed to call Prev when the
// previous call to SeekLT or Prev returned an invalid span.
Prev() *Span
// Error returns any accumulated error.
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
// the iterator is not considered to be an error. It is valid to call Close
// multiple times. Other methods should not be called after the iterator has
// been closed.
Close() error
}
// TableNewSpanIter creates a new iterator for range key spans for the given
// file.
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
// per-sstable span iterators.
type SpanIterOptions struct {
// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
// when iterating over range keys.
RangeKeyFilters []base.BlockPropertyFilter
}
// Iter is an iterator over a set of fragmented spans.
type Iter struct {
cmp base.Compare
spans []Span
index int
}
// Iter implements the FragmentIterator interface.
var _ FragmentIterator = (*Iter)(nil)
// NewIter returns a new iterator over a set of fragmented spans.
func NewIter(cmp base.Compare, spans []Span) *Iter {
i := &Iter{}
i.Init(cmp, spans)
return i
}
// Count returns the number of spans contained by Iter.
func (i *Iter) Count() int {
return len(i.spans)
}
// Init initializes an Iter with the provided spans.
func (i *Iter) Init(cmp base.Compare, spans []Span) {
*i = Iter{
cmp: cmp,
spans: spans,
index: -1,
}
}
// SeekGE implements FragmentIterator.SeekGE.
func (i *Iter) SeekGE(key []byte) *Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(j) = false iff the span i.spans[j] is strictly before `key`
// (equivalently, i.spans[j].End ≤ key.)
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
upper := len(i.spans)
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].End) >= 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
if i.index >= len(i.spans) {
return nil
}
return &i.spans[i.index]
}
// SeekLT implements FragmentIterator.SeekLT.
func (i *Iter) SeekLT(key []byte) *Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
upper := len(i.spans)
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].Start) > 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
// Since keys are strictly increasing, if i.index > 0 then i.index-1 will be
// the largest whose key is < the key sought.
i.index--
if i.index < 0 {
return nil
}
return &i.spans[i.index]
}
// First implements FragmentIterator.First.
func (i *Iter) First() *Span {
if len(i.spans) == 0 {
return nil
}
i.index = 0
return &i.spans[i.index]
}
// Last implements FragmentIterator.Last.
func (i *Iter) Last() *Span {
if len(i.spans) == 0 {
return nil
}
i.index = len(i.spans) - 1
return &i.spans[i.index]
}
// Next implements FragmentIterator.Next.
func (i *Iter) Next() *Span {
if i.index >= len(i.spans) {
return nil
}
i.index++
if i.index >= len(i.spans) {
return nil
}
return &i.spans[i.index]
}
// Prev implements FragmentIterator.Prev.
func (i *Iter) Prev() *Span {
if i.index < 0 {
return nil
}
i.index--
if i.index < 0 {
return nil
}
return &i.spans[i.index]
}
// Error implements FragmentIterator.Error.
func (i *Iter) Error() error {
return nil
}
// Close implements FragmentIterator.Close.
func (i *Iter) Close() error {
return nil
}
func (i *Iter) String() string {
return "fragmented-spans"
}

521
vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go generated vendored Normal file
View File

@@ -0,0 +1,521 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
)
// LevelIter provides a merged view of spans from sstables in a level.
// It takes advantage of level invariants to only have one sstable span block
// open at one time, opened using the newIter function passed in.
type LevelIter struct {
cmp base.Compare
// Denotes the kind of key the level iterator should read. If the key type
// is KeyTypePoint, the level iterator will read range tombstones (which
// only affect point keys). If the key type is KeyTypeRange, the level
// iterator will read range keys. It is invalid to configure an iterator
// with the KeyTypePointAndRange key type.
//
// If key type is KeyTypePoint, no straddle spans are emitted between files,
// and point key bounds are used to find files instead of range key bounds.
//
// TODO(bilal): Straddle spans can safely be produced in rangedel mode once
// we can guarantee that we will never read sstables in a level that split
// user keys across them. This might be guaranteed in a future release, but
// as of CockroachDB 22.2 it is not guaranteed, so to be safe disable it when
// keyType == KeyTypePoint
keyType manifest.KeyType
// The LSM level this LevelIter is initialized for. Used in logging.
level manifest.Level
// The below fields are used to fill in gaps between adjacent files' range
// key spaces. This is an optimization to avoid unnecessarily loading files
// in cases where range keys are sparse and rare. dir is set by every
// positioning operation, straddleDir is set to dir whenever a straddling
// Span is synthesized and the last positioning operation returned a
// synthesized straddle span.
//
// Note that when a straddle span is initialized, iterFile is modified to
// point to the next file in the straddleDir direction. A change of direction
// on a straddle key therefore necessitates the value of iterFile to be
// reverted.
dir int
straddle Span
straddleDir int
// The iter for the current file (iterFile). It is nil under any of the
// following conditions:
// - files.Current() == nil
// - err != nil
// - straddleDir != 0, in which case iterFile is not nil and points to the
// next file (in the straddleDir direction).
// - some other constraint, like the bounds in opts, caused the file at index to not
// be relevant to the iteration.
iter FragmentIterator
// iterFile holds the current file.
// INVARIANT: iterFile = files.Current()
iterFile *manifest.FileMetadata
newIter TableNewSpanIter
files manifest.LevelIterator
err error
// The options that were passed in.
tableOpts SpanIterOptions
// TODO(bilal): Add InternalIteratorStats.
}
// LevelIter implements the keyspan.FragmentIterator interface.
var _ FragmentIterator = (*LevelIter)(nil)
// NewLevelIter returns a LevelIter.
func NewLevelIter(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
keyType manifest.KeyType,
) *LevelIter {
l := &LevelIter{}
l.Init(opts, cmp, newIter, files, level, keyType)
return l
}
// Init initializes a LevelIter.
func (l *LevelIter) Init(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
keyType manifest.KeyType,
) {
l.err = nil
l.level = level
l.tableOpts = opts
l.cmp = cmp
l.iterFile = nil
l.newIter = newIter
switch keyType {
case manifest.KeyTypePoint:
l.keyType = keyType
l.files = files.Filter(keyType)
case manifest.KeyTypeRange:
l.keyType = keyType
l.files = files.Filter(keyType)
default:
panic(fmt.Sprintf("unsupported key type: %v", keyType))
}
}
func (l *LevelIter) findFileGE(key []byte) *manifest.FileMetadata {
// Find the earliest file whose largest key is >= key.
//
// If the earliest file has its largest key == key and that largest key is a
// range deletion sentinel, we know that we manufactured this sentinel to convert
// the exclusive range deletion end key into an inclusive key (reminder: [start, end)#seqnum
// is the form of a range deletion sentinel which can contribute a largest key = end#sentinel).
// In this case we don't return this as the earliest file since there is nothing actually
// equal to key in it.
m := l.files.SeekGE(l.cmp, key)
for m != nil {
largestKey := m.LargestRangeKey
if l.keyType == manifest.KeyTypePoint {
largestKey = m.LargestPointKey
}
if !largestKey.IsExclusiveSentinel() || l.cmp(largestKey.UserKey, key) != 0 {
break
}
m = l.files.Next()
}
return m
}
func (l *LevelIter) findFileLT(key []byte) *manifest.FileMetadata {
// Find the last file whose smallest key is < key.
return l.files.SeekLT(l.cmp, key)
}
type loadFileReturnIndicator int8
const (
noFileLoaded loadFileReturnIndicator = iota
fileAlreadyLoaded
newFileLoaded
)
func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileReturnIndicator {
indicator := noFileLoaded
if l.iterFile == file {
if l.err != nil {
return noFileLoaded
}
if l.iter != nil {
// We are already at the file, but we would need to check for bounds.
// Set indicator accordingly.
indicator = fileAlreadyLoaded
}
// We were already at file, but don't have an iterator, probably because the file was
// beyond the iteration bounds. It may still be, but it is also possible that the bounds
// have changed. We handle that below.
}
// Note that LevelIter.Close() can be called multiple times.
if indicator != fileAlreadyLoaded {
if err := l.Close(); err != nil {
return noFileLoaded
}
}
l.iterFile = file
if file == nil {
return noFileLoaded
}
if indicator != fileAlreadyLoaded {
l.iter, l.err = l.newIter(file, l.tableOpts)
indicator = newFileLoaded
}
if l.err != nil {
return noFileLoaded
}
return indicator
}
// SeekGE implements keyspan.FragmentIterator.
func (l *LevelIter) SeekGE(key []byte) *Span {
l.dir = +1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
f := l.findFileGE(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
// Peek at the previous file.
prevFile := l.files.Prev()
l.files.Next()
if prevFile != nil {
// We could unconditionally return an empty span between the seek key and
// f.SmallestRangeKey, however if this span is to the left of all range
// keys on this level, it could lead to inconsistent behaviour in relative
// positioning operations. Consider this example, with a b-c range key:
//
// SeekGE(a) -> a-b:{}
// Next() -> b-c{(#5,RANGEKEYSET,@4,foo)}
// Prev() -> nil
//
// Iterators higher up in the iterator stack rely on this sort of relative
// positioning consistency.
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return l.verify(nil)
}
l.straddleDir = +1
l.straddle = Span{
Start: prevFile.LargestRangeKey.UserKey,
End: f.SmallestRangeKey.UserKey,
Keys: nil,
}
return l.verify(&l.straddle)
}
}
loadFileIndicator := l.loadFile(f, +1)
if loadFileIndicator == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.SeekGE(key); span != nil {
return l.verify(span)
}
return l.skipEmptyFileForward()
}
// SeekLT implements keyspan.FragmentIterator.
func (l *LevelIter) SeekLT(key []byte) *Span {
l.dir = -1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
f := l.findFileLT(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
// Peek at the next file.
nextFile := l.files.Next()
l.files.Prev()
if nextFile != nil {
// We could unconditionally return an empty span between f.LargestRangeKey
// and the seek key, however if this span is to the right of all range keys
// on this level, it could lead to inconsistent behaviour in relative
// positioning operations. Consider this example, with a b-c range key:
//
// SeekLT(d) -> c-d:{}
// Prev() -> b-c{(#5,RANGEKEYSET,@4,foo)}
// Next() -> nil
//
// Iterators higher up in the iterator stack rely on this sort of relative
// positioning consistency.
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return l.verify(nil)
}
l.straddleDir = -1
l.straddle = Span{
Start: f.LargestRangeKey.UserKey,
End: nextFile.SmallestRangeKey.UserKey,
Keys: nil,
}
return l.verify(&l.straddle)
}
}
if l.loadFile(f, -1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.SeekLT(key); span != nil {
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
// First implements keyspan.FragmentIterator.
func (l *LevelIter) First() *Span {
l.dir = +1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
if l.loadFile(l.files.First(), +1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.First(); span != nil {
return l.verify(span)
}
return l.skipEmptyFileForward()
}
// Last implements keyspan.FragmentIterator.
func (l *LevelIter) Last() *Span {
l.dir = -1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
if l.loadFile(l.files.Last(), -1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.Last(); span != nil {
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
// Next implements keyspan.FragmentIterator.
func (l *LevelIter) Next() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) {
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir <= 0
return l.First()
}
l.dir = +1
if l.iter != nil {
if span := l.iter.Next(); span != nil {
return l.verify(span)
}
}
return l.skipEmptyFileForward()
}
// Prev implements keyspan.FragmentIterator.
func (l *LevelIter) Prev() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) {
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir >= 0
return l.Last()
}
l.dir = -1
if l.iter != nil {
if span := l.iter.Prev(); span != nil {
return l.verify(span)
}
}
return l.skipEmptyFileBackward()
}
func (l *LevelIter) skipEmptyFileForward() *Span {
if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
l.iterFile != nil && l.iter != nil {
// We were at a file that had spans. Check if the next file that has
// spans is not directly adjacent to the current file i.e. there is a
// gap in the span keyspace between the two files. In that case, synthesize
// a "straddle span" in l.straddle and return that.
//
// Straddle spans are not created in rangedel mode.
if err := l.Close(); err != nil {
l.err = err
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
// does not change the logic in loadFile() as long as l.iter is also nil;
// which it should be due to the Close() call above.
l.iterFile = l.files.Next()
if l.iterFile == nil {
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
// There is a gap between the two files. Synthesize a straddling span
// to avoid unnecessarily loading the next file.
l.straddle = Span{
Start: startKey,
End: endKey,
}
l.straddleDir = +1
return l.verify(&l.straddle)
}
} else if l.straddleDir < 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
// was already moved backward by skipEmptyFileBackward, so advance it
// forward.
l.iterFile = l.files.Next()
}
l.straddle = Span{}
l.straddleDir = 0
var span *Span
for span.Empty() {
fileToLoad := l.iterFile
if l.keyType == manifest.KeyTypePoint {
// We haven't iterated to the next file yet if we're in point key
// (rangedel) mode.
fileToLoad = l.files.Next()
}
if l.loadFile(fileToLoad, +1) == noFileLoaded {
return l.verify(nil)
}
span = l.iter.First()
// In rangedel mode, we can expect to get empty files that we'd need to
// skip over, but not in range key mode.
if l.keyType == manifest.KeyTypeRange {
break
}
}
return l.verify(span)
}
func (l *LevelIter) skipEmptyFileBackward() *Span {
// We were at a file that had spans. Check if the previous file that has
// spans is not directly adjacent to the current file i.e. there is a
// gap in the span keyspace between the two files. In that case, synthesize
// a "straddle span" in l.straddle and return that.
//
// Straddle spans are not created in rangedel mode.
if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
l.iterFile != nil && l.iter != nil {
if err := l.Close(); err != nil {
l.err = err
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
// does not change the logic in loadFile() as long as l.iter is also nil;
// which it should be due to the Close() call above.
l.iterFile = l.files.Prev()
if l.iterFile == nil {
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
// There is a gap between the two files. Synthesize a straddling span
// to avoid unnecessarily loading the next file.
l.straddle = Span{
Start: startKey,
End: endKey,
}
l.straddleDir = -1
return l.verify(&l.straddle)
}
} else if l.straddleDir > 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
// was already advanced forward by skipEmptyFileForward, so move it
// backward.
l.iterFile = l.files.Prev()
}
l.straddle = Span{}
l.straddleDir = 0
var span *Span
for span.Empty() {
fileToLoad := l.iterFile
if l.keyType == manifest.KeyTypePoint {
fileToLoad = l.files.Prev()
}
if l.loadFile(fileToLoad, -1) == noFileLoaded {
return l.verify(nil)
}
span = l.iter.Last()
// In rangedel mode, we can expect to get empty files that we'd need to
// skip over, but not in range key mode as the filter on the FileMetadata
// should guarantee we always get a non-empty file.
if l.keyType == manifest.KeyTypeRange {
break
}
}
return l.verify(span)
}
// verify is invoked whenever a span is returned from an iterator positioning
// method to a caller. During invariant builds, it asserts invariants to the
// caller.
func (l *LevelIter) verify(s *Span) *Span {
// NB: Do not add any logic outside the invariants.Enabled conditional to
// ensure that verify is always compiled away in production builds.
if invariants.Enabled {
if f := l.files.Current(); f != l.iterFile {
panic(fmt.Sprintf("LevelIter.files.Current (%s) and l.iterFile (%s) diverged",
f, l.iterFile))
}
}
return s
}
// Error implements keyspan.FragmentIterator.
func (l *LevelIter) Error() error {
if l.err != nil || l.iter == nil {
return l.err
}
return l.iter.Error()
}
// Close implements keyspan.FragmentIterator.
func (l *LevelIter) Close() error {
if l.iter != nil {
l.err = l.iter.Close()
l.iter = nil
}
return l.err
}
// String implements keyspan.FragmentIterator.
func (l *LevelIter) String() string {
if l.iterFile != nil {
return fmt.Sprintf("%s: fileNum=%s", l.level, l.iterFile.FileNum)
}
return fmt.Sprintf("%s: fileNum=<nil>", l.level)
}

1209
vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

48
vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go generated vendored Normal file
View File

@@ -0,0 +1,48 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// SeekLE seeks to the span that contains or is before the target key.
func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were to
// use SeekGE then the search key `c` would land on the span [e,i) and
// we'd have to backtrack. The one complexity here is what happens for the
// search key `e`. In that case SeekLT will land us on the span [a,e)
// and we'll have to move forward.
iterSpan := iter.SeekLT(key)
if iterSpan == nil {
// Advance the iterator once to see if the next span has a start key
// equal to key.
iterSpan = iter.Next()
if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
// The iterator is exhausted or we've hit the next span.
return nil
}
} else {
// Invariant: key > iterSpan.Start
if cmp(key, iterSpan.End) >= 0 {
// The current span lies entirely before the search key. Check to see if
// the next span contains the search key. If it doesn't, we'll backup
// and return to our earlier candidate.
iterSpan = iter.Next()
if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
// The next span is past our search key or there is no next span. Go
// back.
iterSpan = iter.Prev()
}
}
}
return iterSpan
}

467
vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go generated vendored Normal file
View File

@@ -0,0 +1,467 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan // import "github.com/cockroachdb/pebble/internal/keyspan"
import (
"bytes"
"fmt"
"sort"
"strconv"
"strings"
"unicode"
"github.com/cockroachdb/pebble/internal/base"
)
// Span represents a set of keys over a span of user key space. All of the keys
// within a Span are applied across the span's key span indicated by Start and
// End. Each internal key applied over the user key span appears as a separate
// Key, with its own kind and sequence number. Optionally, each Key may also
// have a Suffix and/or Value.
//
// Note that the start user key is inclusive and the end user key is exclusive.
//
// Currently the only supported key kinds are:
//
// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
type Span struct {
// Start and End encode the user key range of all the contained items, with
// an inclusive start key and exclusive end key. Both Start and End must be
// non-nil, or both nil if representing an invalid Span.
Start, End []byte
// Keys holds the set of keys applied over the [Start, End) user key range.
// Keys is sorted by (SeqNum, Kind) descending, unless otherwise specified
// by the context. If SeqNum and Kind are equal, the order of Keys is
// undefined. Keys may be empty, even if Start and End are non-nil.
//
// Keys are a decoded representation of the internal keys stored in batches
// or sstable blocks. A single internal key in a range key block may produce
// several decoded Keys.
Keys []Key
KeysOrder KeysOrder
}
// KeysOrder describes the ordering of Keys within a Span.
type KeysOrder int8
const (
// ByTrailerDesc indicates a Span's keys are sorted by Trailer descending.
// This is the default ordering, and the ordering used during physical
// storage.
ByTrailerDesc KeysOrder = iota
// BySuffixAsc indicates a Span's keys are sorted by Suffix ascending. This
// ordering is used during user iteration of range keys.
BySuffixAsc
)
// Key represents a single key applied over a span of user keys. A Key is
// contained by a Span which specifies the span of user keys over which the Key
// is applied.
type Key struct {
// Trailer contains the key kind and sequence number.
Trailer uint64
// Suffix holds an optional suffix associated with the key. This is only
// non-nil for RANGEKEYSET and RANGEKEYUNSET keys.
Suffix []byte
// Value holds a logical value associated with the Key. It is NOT the
// internal value stored in a range key or range deletion block. This is
// only non-nil for RANGEKEYSET keys.
Value []byte
}
// SeqNum returns the sequence number component of the key.
func (k Key) SeqNum() uint64 {
return k.Trailer >> 8
}
// VisibleAt returns true if the provided key is visible at the provided
// snapshot sequence number. It interprets batch sequence numbers as always
// visible, because non-visible batch span keys are filtered when they're
// fragmented.
func (k Key) VisibleAt(snapshot uint64) bool {
seq := k.SeqNum()
return seq < snapshot || seq&base.InternalKeySeqNumBatch != 0
}
// Kind returns the kind component of the key.
func (k Key) Kind() base.InternalKeyKind {
return base.InternalKeyKind(k.Trailer & 0xff)
}
// Equal returns true if this Key is equal to the given key. Two keys are said
// to be equal if the two Keys have equal trailers, suffix and value. Suffix
// comparison uses the provided base.Compare func. Value comparison is bytewise.
func (k Key) Equal(equal base.Equal, b Key) bool {
return k.Trailer == b.Trailer &&
equal(k.Suffix, b.Suffix) &&
bytes.Equal(k.Value, b.Value)
}
// Valid returns true if the span is defined.
func (s *Span) Valid() bool {
return s.Start != nil && s.End != nil
}
// Empty returns true if the span does not contain any keys. An empty span may
// still be Valid. A non-empty span must be Valid.
//
// An Empty span may be produced by Visible, or be produced by iterators in
// order to surface the gaps between keys.
func (s *Span) Empty() bool {
return s == nil || len(s.Keys) == 0
}
// SmallestKey returns the smallest internal key defined by the span's keys.
// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
// contains no keys or its keys are sorted in a different order.
func (s *Span) SmallestKey() base.InternalKey {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
// The first key has the highest (sequence number,kind) tuple.
return base.InternalKey{
UserKey: s.Start,
Trailer: s.Keys[0].Trailer,
}
}
// LargestKey returns the largest internal key defined by the span's keys. The
// returned key will always be a "sentinel key" at the end boundary. The
// "sentinel key" models the exclusive end boundary by returning an InternalKey
// with the maximal sequence number, ensuring all InternalKeys with the same
// user key sort after the sentinel key.
//
// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
// contains no keys or its keys are sorted in a different order.
func (s *Span) LargestKey() base.InternalKey {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
// The last key has the lowest (sequence number,kind) tuple.
kind := s.Keys[len(s.Keys)-1].Kind()
return base.MakeExclusiveSentinelKey(kind, s.End)
}
// SmallestSeqNum returns the smallest sequence number of a key contained within
// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
// the span contains no keys or its keys are sorted in a different order.
func (s *Span) SmallestSeqNum() uint64 {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
return s.Keys[len(s.Keys)-1].SeqNum()
}
// LargestSeqNum returns the largest sequence number of a key contained within
// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
// the span contains no keys or its keys are sorted in a different order.
func (s *Span) LargestSeqNum() uint64 {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
return s.Keys[0].SeqNum()
}
// TODO(jackson): Replace most of the calls to Visible with more targeted calls
// that avoid the need to construct a new Span.
// Visible returns a span with the subset of keys visible at the provided
// sequence number. It requires the Span's keys be in ByTrailerDesc order. It
// panics if the span's keys are sorted in a different order.
//
// Visible may incur an allocation, so callers should prefer targeted,
// non-allocating methods when possible.
func (s Span) Visible(snapshot uint64) Span {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
ret := Span{Start: s.Start, End: s.End}
if len(s.Keys) == 0 {
return ret
}
// Keys from indexed batches may force an allocation. The Keys slice is
// ordered by sequence number, so ordinarily we can return the trailing
// subslice containing keys with sequence numbers less than `seqNum`.
//
// However, batch keys are special. Only visible batch keys are included
// when an Iterator's batch spans are fragmented. They must always be
// visible.
//
// Batch keys can create a sandwich of visible batch keys at the beginning
// of the slice and visible committed keys at the end of the slice, forcing
// us to allocate a new slice and copy the contents.
//
// Care is taking to only incur an allocation only when batch keys and
// visible keys actually sandwich non-visible keys.
// lastBatchIdx and lastNonVisibleIdx are set to the last index of a batch
// key and a non-visible key respectively.
lastBatchIdx := -1
lastNonVisibleIdx := -1
for i := range s.Keys {
if seqNum := s.Keys[i].SeqNum(); seqNum&base.InternalKeySeqNumBatch != 0 {
// Batch key. Always visible.
lastBatchIdx = i
} else if seqNum >= snapshot {
// This key is not visible.
lastNonVisibleIdx = i
}
}
// In the following comments: b = batch, h = hidden, v = visible (committed).
switch {
case lastNonVisibleIdx == -1:
// All keys are visible.
//
// [b b b], [v v v] and [b b b v v v]
ret.Keys = s.Keys
case lastBatchIdx == -1:
// There are no batch keys, so we can return the continuous subslice
// starting after the last non-visible Key.
//
// h h h [v v v]
ret.Keys = s.Keys[lastNonVisibleIdx+1:]
case lastNonVisibleIdx == len(s.Keys)-1:
// While we have a batch key and non-visible keys, there are no
// committed visible keys. The 'sandwich' is missing the bottom layer,
// so we can return the continuous sublice at the beginning.
//
// [b b b] h h h
ret.Keys = s.Keys[0 : lastBatchIdx+1]
default:
// This is the problematic sandwich case. Allocate a new slice, copying
// the batch keys and the visible keys into it.
//
// [b b b] h h h [v v v]
ret.Keys = make([]Key, (lastBatchIdx+1)+(len(s.Keys)-lastNonVisibleIdx-1))
copy(ret.Keys, s.Keys[:lastBatchIdx+1])
copy(ret.Keys[lastBatchIdx+1:], s.Keys[lastNonVisibleIdx+1:])
}
return ret
}
// VisibleAt returns true if the span contains a key visible at the provided
// snapshot. Keys with sequence numbers with the batch bit set are treated as
// always visible.
//
// VisibleAt requires the Span's keys be in ByTrailerDesc order. It panics if
// the span's keys are sorted in a different order.
func (s *Span) VisibleAt(snapshot uint64) bool {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
if len(s.Keys) == 0 {
return false
} else if first := s.Keys[0].SeqNum(); first&base.InternalKeySeqNumBatch != 0 {
// Only visible batch keys are included when an Iterator's batch spans
// are fragmented. They must always be visible.
return true
} else {
// Otherwise we check the last key. Since keys are ordered decreasing in
// sequence number, the last key has the lowest sequence number of any
// of the span's keys. If any of the keys are visible, the last key must
// be visible. Or put differently: if the last key is not visible, then
// no key is visible.
return s.Keys[len(s.Keys)-1].SeqNum() < snapshot
}
}
// ShallowClone returns the span with a Keys slice owned by the span itself.
// None of the key byte slices are cloned (see Span.DeepClone).
func (s *Span) ShallowClone() Span {
c := Span{
Start: s.Start,
End: s.End,
Keys: make([]Key, len(s.Keys)),
KeysOrder: s.KeysOrder,
}
copy(c.Keys, s.Keys)
return c
}
// DeepClone clones the span, creating copies of all contained slices. DeepClone
// is intended for non-production code paths like tests, the level checker, etc
// because it is allocation heavy.
func (s *Span) DeepClone() Span {
c := Span{
Start: make([]byte, len(s.Start)),
End: make([]byte, len(s.End)),
Keys: make([]Key, len(s.Keys)),
KeysOrder: s.KeysOrder,
}
copy(c.Start, s.Start)
copy(c.End, s.End)
for i := range s.Keys {
c.Keys[i].Trailer = s.Keys[i].Trailer
if len(s.Keys[i].Suffix) > 0 {
c.Keys[i].Suffix = make([]byte, len(s.Keys[i].Suffix))
copy(c.Keys[i].Suffix, s.Keys[i].Suffix)
}
if len(s.Keys[i].Value) > 0 {
c.Keys[i].Value = make([]byte, len(s.Keys[i].Value))
copy(c.Keys[i].Value, s.Keys[i].Value)
}
}
return c
}
// Contains returns true if the specified key resides within the span's bounds.
func (s *Span) Contains(cmp base.Compare, key []byte) bool {
return cmp(s.Start, key) <= 0 && cmp(key, s.End) < 0
}
// Covers returns true if the span covers keys at seqNum.
//
// Covers requires the Span's keys be in ByTrailerDesc order. It panics if the
// span's keys are sorted in a different order.
func (s Span) Covers(seqNum uint64) bool {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
return !s.Empty() && s.Keys[0].SeqNum() > seqNum
}
// CoversAt returns true if the span contains a key that is visible at the
// provided snapshot sequence number, and that key's sequence number is higher
// than seqNum.
//
// Keys with sequence numbers with the batch bit set are treated as always
// visible.
//
// CoversAt requires the Span's keys be in ByTrailerDesc order. It panics if the
// span's keys are sorted in a different order.
func (s *Span) CoversAt(snapshot, seqNum uint64) bool {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
// NB: A key is visible at `snapshot` if its sequence number is strictly
// less than `snapshot`. See base.Visible.
for i := range s.Keys {
if kseq := s.Keys[i].SeqNum(); kseq&base.InternalKeySeqNumBatch != 0 {
// Only visible batch keys are included when an Iterator's batch spans
// are fragmented. They must always be visible.
return kseq > seqNum
} else if kseq < snapshot {
return kseq > seqNum
}
}
return false
}
// String returns a string representation of the span.
func (s Span) String() string {
return fmt.Sprint(prettySpan{Span: s, formatKey: base.DefaultFormatter})
}
// Pretty returns a formatter for the span.
func (s Span) Pretty(f base.FormatKey) fmt.Formatter {
// TODO(jackson): Take a base.FormatValue to format Key.Value too.
return prettySpan{s, f}
}
type prettySpan struct {
Span
formatKey base.FormatKey
}
func (s prettySpan) Format(fs fmt.State, c rune) {
if !s.Valid() {
fmt.Fprintf(fs, "<invalid>")
return
}
fmt.Fprintf(fs, "%s-%s:{", s.formatKey(s.Start), s.formatKey(s.End))
for i, k := range s.Keys {
if i > 0 {
fmt.Fprint(fs, " ")
}
fmt.Fprintf(fs, "(#%d,%s", k.SeqNum(), k.Kind())
if len(k.Suffix) > 0 || len(k.Value) > 0 {
fmt.Fprintf(fs, ",%s", k.Suffix)
}
if len(k.Value) > 0 {
fmt.Fprintf(fs, ",%s", k.Value)
}
fmt.Fprint(fs, ")")
}
fmt.Fprintf(fs, "}")
}
// SortKeysByTrailer sorts a keys slice by trailer.
func SortKeysByTrailer(keys *[]Key) {
// NB: keys is a pointer to a slice instead of a slice to avoid `sorted`
// escaping to the heap.
sorted := (*keysBySeqNumKind)(keys)
sort.Sort(sorted)
}
// KeysBySuffix implements sort.Interface, sorting its member Keys slice to by
// Suffix in the order dictated by Cmp.
type KeysBySuffix struct {
Cmp base.Compare
Keys []Key
}
func (s *KeysBySuffix) Len() int { return len(s.Keys) }
func (s *KeysBySuffix) Less(i, j int) bool { return s.Cmp(s.Keys[i].Suffix, s.Keys[j].Suffix) < 0 }
func (s *KeysBySuffix) Swap(i, j int) { s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] }
// ParseSpan parses the string representation of a Span. It's intended for
// tests. ParseSpan panics if passed a malformed span representation.
func ParseSpan(input string) Span {
var s Span
parts := strings.FieldsFunc(input, func(r rune) bool {
switch r {
case '-', ':', '{', '}':
return true
default:
return unicode.IsSpace(r)
}
})
s.Start, s.End = []byte(parts[0]), []byte(parts[1])
// Each of the remaining parts represents a single Key.
s.Keys = make([]Key, 0, len(parts)-2)
for _, p := range parts[2:] {
keyFields := strings.FieldsFunc(p, func(r rune) bool {
switch r {
case '#', ',', '(', ')':
return true
default:
return unicode.IsSpace(r)
}
})
var k Key
// Parse the sequence number.
seqNum, err := strconv.ParseUint(keyFields[0], 10, 64)
if err != nil {
panic(fmt.Sprintf("invalid sequence number: %q: %s", keyFields[0], err))
}
// Parse the key kind.
kind := base.ParseKind(keyFields[1])
k.Trailer = base.MakeTrailer(seqNum, kind)
// Parse the optional suffix.
if len(keyFields) >= 3 {
k.Suffix = []byte(keyFields[2])
}
// Parse the optional value.
if len(keyFields) >= 4 {
k.Value = []byte(keyFields[3])
}
s.Keys = append(s.Keys, k)
}
return s
}

50
vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go generated vendored Normal file
View File

@@ -0,0 +1,50 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
// provided output *Span pointer. The output Span's Keys slice may be reused
// by Transform to reduce allocations.
Transform(cmp base.Compare, in Span, out *Span) error
}
// The TransformerFunc type is an adapter to allow the use of ordinary functions
// as Transformers. If f is a function with the appropriate signature,
// TransformerFunc(f) is a Transformer that calls f.
type TransformerFunc func(base.Compare, Span, *Span) error
// Transform calls f(cmp, in, out).
func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
return tf(cmp, in, out)
}
var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
})
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
for _, k := range s.Keys {
// NB: The InternalKeySeqNumMax value is used for the batch snapshot
// because a batch's visible span keys are filtered when they're
// fragmented. There's no requirement to enforce visibility at
// iteration time.
if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
dst.Keys = append(dst.Keys, k)
}
}
return nil
})
}

73
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored Normal file
View File

@@ -0,0 +1,73 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Truncate creates a new iterator where every span in the supplied iterator is
// truncated to be contained within the range [lower, upper). If start and end
// are specified, filter out any spans that are completely outside those bounds.
func Truncate(
cmp base.Compare,
iter FragmentIterator,
lower, upper []byte,
start, end *base.InternalKey,
panicOnUpperTruncate bool,
) FragmentIterator {
return Filter(iter, func(in *Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
// Ignore this span if it lies completely outside start, end. Note that
// end endInclusive indicated whether end is inclusive.
//
// The comparison between s.End and start is by user key only, as
// the span is exclusive at s.End, so comparing by user keys
// is sufficient.
if start != nil && cmp(in.End, start.UserKey) <= 0 {
return false
}
if end != nil {
v := cmp(in.Start, end.UserKey)
switch {
case v > 0:
// Wholly outside the end bound. Skip it.
return false
case v == 0:
// This span begins at the same user key as `end`. Whether or
// not any of the keys contained within the span are relevant is
// dependent on Trailers. Any keys contained within the span
// with trailers larger than end cover the small sliver of
// keyspace between [k#inf, k#<end-seqnum>]. Since keys are
// sorted descending by Trailer within the span, we need to find
// the prefix of keys with larger trailers.
for i := range in.Keys {
if in.Keys[i].Trailer < end.Trailer {
out.Keys = out.Keys[:i]
break
}
}
default:
// Wholly within the end bound. Keep it.
}
}
var truncated bool
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
out.Start = lower
}
if cmp(in.End, upper) > 0 {
truncated = true
out.End = upper
}
if panicOnUpperTruncate && truncated {
panic("pebble: upper bound should not be truncated")
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}

1312
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

2040
vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

46
vendor/github.com/cockroachdb/pebble/internal/manifest/level.go generated vendored Normal file
View File

@@ -0,0 +1,46 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import "fmt"
const (
// 3 bits are necessary to represent level values from 0-6.
levelBits = 3
levelMask = (1 << levelBits) - 1
// invalidSublevel denotes an invalid or non-applicable sublevel.
invalidSublevel = -1
)
// Level encodes a level and optional sublevel for use in log and error
// messages. The encoding has the property that Level(0) ==
// L0Sublevel(invalidSublevel).
type Level uint32
func makeLevel(level, sublevel int) Level {
return Level(((sublevel + 1) << levelBits) | level)
}
// LevelToInt returns the int representation of a Level
func LevelToInt(l Level) int {
return int(l) & levelMask
}
// L0Sublevel returns a Level representing the specified L0 sublevel.
func L0Sublevel(sublevel int) Level {
if sublevel < 0 {
panic(fmt.Sprintf("invalid L0 sublevel: %d", sublevel))
}
return makeLevel(0, sublevel)
}
func (l Level) String() string {
level := int(l) & levelMask
sublevel := (int(l) >> levelBits) - 1
if sublevel != invalidSublevel {
return fmt.Sprintf("L%d.%d", level, sublevel)
}
return fmt.Sprintf("L%d", level)
}

704
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored Normal file
View File

@@ -0,0 +1,704 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import (
"bytes"
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// LevelMetadata contains metadata for all of the files within
// a level of the LSM.
type LevelMetadata struct {
level int
totalSize uint64
tree btree
}
// clone makes a copy of the level metadata, implicitly increasing the ref
// count of every file contained within lm.
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
totalSize: lm.totalSize,
tree: lm.tree.Clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
return lm.tree.Release()
}
func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetadata {
bcmp := btreeCmpSeqNum
if level > 0 {
bcmp = btreeCmpSmallestKey(cmp)
}
var lm LevelMetadata
lm.level = level
lm.tree, _ = makeBTree(bcmp, files)
for _, f := range files {
lm.totalSize += f.Size
}
return lm
}
func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
var t btree
t.cmp = cmp
for _, f := range files {
t.Insert(f)
}
return t, newLevelSlice(t.Iter())
}
func (lm *LevelMetadata) insert(f *FileMetadata) error {
if err := lm.tree.Insert(f); err != nil {
return err
}
lm.totalSize += f.Size
return nil
}
func (lm *LevelMetadata) remove(f *FileMetadata) bool {
lm.totalSize -= f.Size
return lm.tree.Delete(f)
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.Count() == 0
}
// Len returns the number of files within the level.
func (lm *LevelMetadata) Len() int {
return lm.tree.Count()
}
// Size returns the cumulative size of all the files within the level.
func (lm *LevelMetadata) Size() uint64 {
return lm.totalSize
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.Iter()}
}
// Slice constructs a slice containing the entire level.
func (lm *LevelMetadata) Slice() LevelSlice {
return newLevelSlice(lm.tree.Iter())
}
// Find finds the provided file in the level if it exists.
func (lm *LevelMetadata) Find(cmp base.Compare, m *FileMetadata) *LevelFile {
iter := lm.Iter()
if lm.level != 0 {
// If lm holds files for levels >0, we can narrow our search by binary
// searching by bounds.
o := overlaps(iter, cmp, m.Smallest.UserKey,
m.Largest.UserKey, m.Largest.IsExclusiveSentinel())
iter = o.Iter()
}
for f := iter.First(); f != nil; f = iter.Next() {
if f == m {
lf := iter.Take()
return &lf
}
}
return nil
}
// Annotation lazily calculates and returns the annotation defined by
// Annotator. The Annotator is used as the key for pre-calculated
// values, so equal Annotators must be used to avoid duplicate computations
// and cached annotations. Annotation must not be called concurrently, and in
// practice this is achieved by requiring callers to hold DB.mu.
func (lm *LevelMetadata) Annotation(annotator Annotator) interface{} {
if lm.Empty() {
return annotator.Zero(nil)
}
v, _ := lm.tree.root.Annotation(annotator)
return v
}
// InvalidateAnnotation clears any cached annotations defined by Annotator. The
// Annotator is used as the key for pre-calculated values, so equal Annotators
// must be used to clear the appropriate cached annotation. InvalidateAnnotation
// must not be called concurrently, and in practice this is achieved by
// requiring callers to hold DB.mu.
func (lm *LevelMetadata) InvalidateAnnotation(annotator Annotator) {
if lm.Empty() {
return
}
lm.tree.root.InvalidateAnnotation(annotator)
}
// LevelFile holds a file's metadata along with its position
// within a level of the LSM.
type LevelFile struct {
*FileMetadata
slice LevelSlice
}
// Slice constructs a LevelSlice containing only this file.
func (lf LevelFile) Slice() LevelSlice {
return lf.slice
}
// NewLevelSliceSeqSorted constructs a LevelSlice over the provided files,
// sorted by the L0 sequence number sort order.
// TODO(jackson): Can we improve this interface or avoid needing to export
// a slice constructor like this?
func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSeqNum, files)
tr.Release()
slice.verifyInvariants()
return slice
}
// NewLevelSliceKeySorted constructs a LevelSlice over the provided files,
// sorted by the files smallest keys.
// TODO(jackson): Can we improve this interface or avoid needing to export
// a slice constructor like this?
func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSmallestKey(cmp), files)
tr.Release()
slice.verifyInvariants()
return slice
}
// NewLevelSliceSpecificOrder constructs a LevelSlice over the provided files,
// ordering the files by their order in the provided slice. It's used in
// tests.
// TODO(jackson): Update tests to avoid requiring this and remove it.
func NewLevelSliceSpecificOrder(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSpecificOrder(files), files)
tr.Release()
slice.verifyInvariants()
return slice
}
// newLevelSlice constructs a new LevelSlice backed by iter.
func newLevelSlice(iter iterator) LevelSlice {
s := LevelSlice{iter: iter}
if iter.r != nil {
s.length = iter.r.subtreeCount
}
s.verifyInvariants()
return s
}
// newBoundedLevelSlice constructs a new LevelSlice backed by iter and bounded
// by the provided start and end bounds. The provided startBound and endBound
// iterators must be iterators over the same B-Tree. Both start and end bounds
// are inclusive.
func newBoundedLevelSlice(iter iterator, startBound, endBound *iterator) LevelSlice {
s := LevelSlice{
iter: iter,
start: startBound,
end: endBound,
}
if iter.valid() {
s.length = endBound.countLeft() - startBound.countLeft()
// NB: The +1 is a consequence of the end bound being inclusive.
if endBound.valid() {
s.length++
}
// NB: A slice that's empty due to its bounds may have an endBound
// positioned before the startBound due to the inclusive bounds.
// TODO(jackson): Consider refactoring the end boundary to be exclusive;
// it would simplify some areas (eg, here) and complicate others (eg,
// Reslice-ing to grow compactions).
if s.length < 0 {
s.length = 0
}
}
s.verifyInvariants()
return s
}
// LevelSlice contains a slice of the files within a level of the LSM.
// A LevelSlice is immutable once created, but may be used to construct a
// mutable LevelIterator over the slice's files.
//
// LevelSlices should be constructed through one of the existing constructors,
// not manually initialized.
type LevelSlice struct {
iter iterator
length int
// start and end form the inclusive bounds of a slice of files within a
// level of the LSM. They may be nil if the entire B-Tree backing iter is
// accessible.
start *iterator
end *iterator
}
func (ls LevelSlice) verifyInvariants() {
if invariants.Enabled {
i := ls.Iter()
var length int
for f := i.First(); f != nil; f = i.Next() {
length++
}
if ls.length != length {
panic(fmt.Sprintf("LevelSlice %s has length %d value; actual length is %d", ls, ls.length, length))
}
}
}
// Each invokes fn for each element in the slice.
func (ls LevelSlice) Each(fn func(*FileMetadata)) {
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
fn(f)
}
}
// String implements fmt.Stringer.
func (ls LevelSlice) String() string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d files: ", ls.length)
ls.Each(func(f *FileMetadata) {
if buf.Len() > 0 {
fmt.Fprintf(&buf, " ")
}
fmt.Fprint(&buf, f)
})
return buf.String()
}
// Empty indicates whether the slice contains any files.
func (ls *LevelSlice) Empty() bool {
return emptyWithBounds(ls.iter, ls.start, ls.end)
}
// Iter constructs a LevelIterator that iterates over the slice.
func (ls *LevelSlice) Iter() LevelIterator {
return LevelIterator{
start: ls.start,
end: ls.end,
iter: ls.iter.clone(),
}
}
// Len returns the number of files in the slice. Its runtime is constant.
func (ls *LevelSlice) Len() int {
return ls.length
}
// SizeSum sums the size of all files in the slice. Its runtime is linear in
// the length of the slice.
func (ls *LevelSlice) SizeSum() uint64 {
var sum uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
sum += f.Size
}
return sum
}
// Reslice constructs a new slice backed by the same underlying level, with
// new start and end positions. Reslice invokes the provided function, passing
// two LevelIterators: one positioned to i's inclusive start and one
// positioned to i's inclusive end. The resliceFunc may move either iterator
// forward or backwards, including beyond the callee's original bounds to
// capture additional files from the underlying level. Reslice constructs and
// returns a new LevelSlice with the final bounds of the iterators after
// calling resliceFunc.
func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelSlice {
if ls.iter.r == nil {
return ls
}
var start, end LevelIterator
if ls.start == nil {
start.iter = ls.iter.clone()
start.iter.first()
} else {
start.iter = ls.start.clone()
}
if ls.end == nil {
end.iter = ls.iter.clone()
end.iter.last()
} else {
end.iter = ls.end.clone()
}
resliceFunc(&start, &end)
return newBoundedLevelSlice(start.iter.clone(), &start.iter, &end.iter)
}
// KeyType is used to specify the type of keys we're looking for in
// LevelIterator positioning operations. Files not containing any keys of the
// desired type are skipped.
type KeyType int8
const (
// KeyTypePointAndRange denotes a search among the entire keyspace, including
// both point keys and range keys. No sstables are skipped.
KeyTypePointAndRange KeyType = iota
// KeyTypePoint denotes a search among the point keyspace. SSTables with no
// point keys will be skipped. Note that the point keyspace includes rangedels.
KeyTypePoint
// KeyTypeRange denotes a search among the range keyspace. SSTables with no
// range keys will be skipped.
KeyTypeRange
)
type keyTypeAnnotator struct{}
var _ Annotator = keyTypeAnnotator{}
func (k keyTypeAnnotator) Zero(dst interface{}) interface{} {
var val *KeyType
if dst != nil {
val = dst.(*KeyType)
} else {
val = new(KeyType)
}
*val = KeyTypePoint
return val
}
func (k keyTypeAnnotator) Accumulate(m *FileMetadata, dst interface{}) (interface{}, bool) {
v := dst.(*KeyType)
switch *v {
case KeyTypePoint:
if m.HasRangeKeys {
*v = KeyTypePointAndRange
}
case KeyTypePointAndRange:
// Do nothing.
default:
panic("unexpected key type")
}
return v, true
}
func (k keyTypeAnnotator) Merge(src interface{}, dst interface{}) interface{} {
v := dst.(*KeyType)
srcVal := src.(*KeyType)
switch *v {
case KeyTypePoint:
if *srcVal == KeyTypePointAndRange {
*v = KeyTypePointAndRange
}
case KeyTypePointAndRange:
// Do nothing.
default:
panic("unexpected key type")
}
return v
}
// LevelIterator iterates over a set of files' metadata. Its zero value is an
// empty iterator.
type LevelIterator struct {
iter iterator
start *iterator
end *iterator
filter KeyType
}
func (i LevelIterator) String() string {
var buf bytes.Buffer
iter := i.iter.clone()
iter.first()
iter.prev()
if i.iter.pos == -1 {
fmt.Fprint(&buf, "(<start>)*")
}
iter.next()
for ; iter.valid(); iter.next() {
if buf.Len() > 0 {
fmt.Fprint(&buf, " ")
}
if i.start != nil && cmpIter(iter, *i.start) == 0 {
fmt.Fprintf(&buf, " [ ")
}
isCurrentPos := cmpIter(iter, i.iter) == 0
if isCurrentPos {
fmt.Fprint(&buf, " ( ")
}
fmt.Fprint(&buf, iter.cur().String())
if isCurrentPos {
fmt.Fprint(&buf, " )*")
}
if i.end != nil && cmpIter(iter, *i.end) == 0 {
fmt.Fprintf(&buf, " ]")
}
}
if i.iter.n != nil && i.iter.pos >= i.iter.n.count {
if buf.Len() > 0 {
fmt.Fprint(&buf, " ")
}
fmt.Fprint(&buf, "(<end>)*")
}
return buf.String()
}
// Clone copies the iterator, returning an independent iterator at the same
// position.
func (i *LevelIterator) Clone() LevelIterator {
if i.iter.r == nil {
return *i
}
// The start and end iterators are not cloned and are treated as
// immutable.
return LevelIterator{
iter: i.iter.clone(),
start: i.start,
end: i.end,
filter: i.filter,
}
}
// Current returns the item at the current iterator position.
//
// Current is deprecated. Callers should instead use the return value of a
// positioning operation.
func (i *LevelIterator) Current() *FileMetadata {
if !i.iter.valid() ||
(i.end != nil && cmpIter(i.iter, *i.end) > 0) ||
(i.start != nil && cmpIter(i.iter, *i.start) < 0) {
return nil
}
return i.iter.cur()
}
func (i *LevelIterator) empty() bool {
return emptyWithBounds(i.iter, i.start, i.end)
}
// Filter clones the iterator and sets the desired KeyType as the key to filter
// files on.
func (i *LevelIterator) Filter(keyType KeyType) LevelIterator {
l := i.Clone()
l.filter = keyType
return l
}
func emptyWithBounds(i iterator, start, end *iterator) bool {
// If i.r is nil, the iterator was constructed from an empty btree.
// If the end bound is before the start bound, the bounds represent an
// empty slice of the B-Tree.
return i.r == nil || (start != nil && end != nil && cmpIter(*end, *start) < 0)
}
// First seeks to the first file in the iterator and returns it.
func (i *LevelIterator) First() *FileMetadata {
if i.empty() {
return nil
}
if i.start != nil {
i.iter = i.start.clone()
} else {
i.iter.first()
}
if !i.iter.valid() {
return nil
}
return i.skipFilteredForward(i.iter.cur())
}
// Last seeks to the last file in the iterator and returns it.
func (i *LevelIterator) Last() *FileMetadata {
if i.empty() {
return nil
}
if i.end != nil {
i.iter = i.end.clone()
} else {
i.iter.last()
}
if !i.iter.valid() {
return nil
}
return i.skipFilteredBackward(i.iter.cur())
}
// Next advances the iterator to the next file and returns it.
func (i *LevelIterator) Next() *FileMetadata {
if i.iter.r == nil {
return nil
}
if invariants.Enabled && (i.iter.pos >= i.iter.n.count || (i.end != nil && cmpIter(i.iter, *i.end) > 0)) {
panic("pebble: cannot next forward-exhausted iterator")
}
i.iter.next()
if !i.iter.valid() {
return nil
}
return i.skipFilteredForward(i.iter.cur())
}
// Prev moves the iterator the previous file and returns it.
func (i *LevelIterator) Prev() *FileMetadata {
if i.iter.r == nil {
return nil
}
if invariants.Enabled && (i.iter.pos < 0 || (i.start != nil && cmpIter(i.iter, *i.start) < 0)) {
panic("pebble: cannot prev backward-exhausted iterator")
}
i.iter.prev()
if !i.iter.valid() {
return nil
}
return i.skipFilteredBackward(i.iter.cur())
}
// SeekGE seeks to the first file in the iterator's file set with a largest
// user key greater than or equal to the provided user key. The iterator must
// have been constructed from L1+, because it requires the underlying files to
// be sorted by user keys and non-overlapping.
func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.iter.r == nil {
return nil
}
m := i.seek(func(m *FileMetadata) bool {
return cmp(m.Largest.UserKey, userKey) >= 0
})
if i.filter != KeyTypePointAndRange && m != nil {
b, ok := m.LargestBound(i.filter)
if !ok {
m = i.Next()
} else if c := cmp(b.UserKey, userKey); c < 0 || c == 0 && b.IsExclusiveSentinel() {
// This file does not contain any keys of the type ≥ lower. It
// should be filtered, even though it does contain point keys.
m = i.Next()
}
}
return i.skipFilteredForward(m)
}
// SeekLT seeks to the last file in the iterator's file set with a smallest
// user key less than the provided user key. The iterator must have been
// constructed from L1+, because it requires the underlying files to be sorted
// by user keys and non-overlapping.
func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.iter.r == nil {
return nil
}
i.seek(func(m *FileMetadata) bool {
return cmp(m.Smallest.UserKey, userKey) >= 0
})
m := i.Prev()
// Although i.Prev() guarantees that the current file contains keys of the
// relevant type, it doesn't guarantee that the keys of the relevant type
// are < userKey.
if i.filter != KeyTypePointAndRange && m != nil {
b, ok := m.SmallestBound(i.filter)
if !ok {
panic("unreachable")
}
if c := cmp(b.UserKey, userKey); c >= 0 {
// This file does not contain any keys of the type ≥ lower. It
// should be filtered, even though it does contain point keys.
m = i.Prev()
}
}
return i.skipFilteredBackward(m)
}
// skipFilteredForward takes the file metadata at the iterator's current
// position, and skips forward if the current key-type filter (i.filter)
// excludes the file. It skips until it finds an unfiltered file or exhausts the
// level. If lower is != nil, skipFilteredForward skips any files that do not
// contain keys with the provided key-type ≥ lower.
//
// skipFilteredForward also enforces the upper bound, returning nil if at any
// point the upper bound is exceeded.
func (i *LevelIterator) skipFilteredForward(meta *FileMetadata) *FileMetadata {
for meta != nil && !meta.ContainsKeyType(i.filter) {
i.iter.next()
if !i.iter.valid() {
meta = nil
} else {
meta = i.iter.cur()
}
}
if meta != nil && i.end != nil && cmpIter(i.iter, *i.end) > 0 {
// Exceeded upper bound.
meta = nil
}
return meta
}
// skipFilteredBackward takes the file metadata at the iterator's current
// position, and skips backward if the current key-type filter (i.filter)
// excludes the file. It skips until it finds an unfiltered file or exhausts the
// level. If upper is != nil, skipFilteredBackward skips any files that do not
// contain keys with the provided key-type < upper.
//
// skipFilteredBackward also enforces the lower bound, returning nil if at any
// point the lower bound is exceeded.
func (i *LevelIterator) skipFilteredBackward(meta *FileMetadata) *FileMetadata {
for meta != nil && !meta.ContainsKeyType(i.filter) {
i.iter.prev()
if !i.iter.valid() {
meta = nil
} else {
meta = i.iter.cur()
}
}
if meta != nil && i.start != nil && cmpIter(i.iter, *i.start) < 0 {
// Exceeded lower bound.
meta = nil
}
return meta
}
func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
i.iter.seek(fn)
// i.iter.seek seeked in the unbounded underlying B-Tree. If the iterator
// has start or end bounds, we may have exceeded them. Reset to the bounds
// if necessary.
//
// NB: The LevelIterator and LevelSlice semantics require that a bounded
// LevelIterator/LevelSlice containing files x0, x1, ..., xn behave
// identically to an unbounded LevelIterator/LevelSlice of a B-Tree
// containing x0, x1, ..., xn. In other words, any files outside the
// LevelIterator's bounds should not influence the iterator's behavior.
// When seeking, this means a SeekGE that seeks beyond the end bound,
// followed by a Prev should return the last element within bounds.
if i.end != nil && cmpIter(i.iter, *i.end) > 0 {
i.iter = i.end.clone()
// Since seek(fn) positioned beyond i.end, we know there is nothing to
// return within bounds.
i.iter.next()
return nil
} else if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
i.iter = i.start.clone()
}
if !i.iter.valid() {
return nil
}
return i.iter.cur()
}
// Take constructs a LevelFile containing the file at the iterator's current
// position. Take panics if the iterator is not currently positioned over a
// file.
func (i *LevelIterator) Take() LevelFile {
m := i.Current()
if m == nil {
panic("Take called on invalid LevelIterator")
}
// LevelSlice's start and end fields are immutable and are positioned to
// the same position for a LevelFile because they're inclusive, so we can
// share one iterator stack between the two bounds.
boundsIter := i.iter.clone()
s := newBoundedLevelSlice(i.iter.clone(), &boundsIter, &boundsIter)
return LevelFile{
FileMetadata: m,
slice: s,
}
}

1540
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1112
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

60
vendor/github.com/cockroachdb/pebble/internal/manual/manual.go generated vendored Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manual
// #include <stdlib.h>
import "C"
import "unsafe"
// The go:linkname directives provides backdoor access to private functions in
// the runtime. Below we're accessing the throw function.
//go:linkname throw runtime.throw
func throw(s string)
// TODO(peter): Rather than relying an C malloc/free, we could fork the Go
// runtime page allocator and allocate large chunks of memory using mmap or
// similar.
// New allocates a slice of size n. The returned slice is from manually managed
// memory and MUST be released by calling Free. Failure to do so will result in
// a memory leak.
func New(n int) []byte {
if n == 0 {
return make([]byte, 0)
}
// We need to be conscious of the Cgo pointer passing rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// ...
// Note: the current implementation has a bug. While Go code is permitted
// to write nil or a C pointer (but not a Go pointer) to C memory, the
// current implementation may sometimes cause a runtime error if the
// contents of the C memory appear to be a Go pointer. Therefore, avoid
// passing uninitialized C memory to Go code if the Go code is going to
// store pointer values in it. Zero out the memory in C before passing it
// to Go.
ptr := C.calloc(C.size_t(n), 1)
if ptr == nil {
// NB: throw is like panic, except it guarantees the process will be
// terminated. The call below is exactly what the Go runtime invokes when
// it cannot allocate memory.
throw("out of memory")
}
// Interpret the C pointer as a pointer to a Go array, then slice.
return (*[MaxArrayLen]byte)(unsafe.Pointer(ptr))[:n:n]
}
// Free frees the specified slice.
func Free(b []byte) {
if cap(b) != 0 {
if len(b) == 0 {
b = b[:cap(b)]
}
ptr := unsafe.Pointer(&b[0])
C.free(ptr)
}
}

13
vendor/github.com/cockroachdb/pebble/internal/manual/manual_32bit.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build 386 || amd64p32 || arm || armbe || mips || mipsle || mips64p32 || mips64p32le || ppc || sparc
// +build 386 amd64p32 arm armbe mips mipsle mips64p32 mips64p32le ppc sparc
package manual
const (
// MaxArrayLen is a safe maximum length for slices on this architecture.
MaxArrayLen = 1<<31 - 1
)

13
vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64
package manual
const (
// MaxArrayLen is a safe maximum length for slices on this architecture.
MaxArrayLen = 1<<50 - 1
)

20
vendor/github.com/cockroachdb/pebble/internal/manual/manual_nocgo.go generated vendored Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !cgo
// +build !cgo
package manual
// Provides versions of New and Free when cgo is not available (e.g. cross
// compilation).
// New allocates a slice of size n.
func New(n int) []byte {
return make([]byte, n)
}
// Free frees the specified slice.
func Free(b []byte) {
}

19
vendor/github.com/cockroachdb/pebble/internal/private/batch.go generated vendored Normal file
View File

@@ -0,0 +1,19 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// BatchSort is a hook for constructing iterators over the point and range
// mutations contained in a batch in sorted order. It is intended for testing
// use only.
var BatchSort func(interface{}) (
points base.InternalIterator,
rangeDels keyspan.FragmentIterator,
rangeKeys keyspan.FragmentIterator,
)

29
vendor/github.com/cockroachdb/pebble/internal/private/sstable.go generated vendored Normal file
View File

@@ -0,0 +1,29 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import "github.com/cockroachdb/pebble/internal/base"
// SSTableCacheOpts is a hook for specifying cache options to
// sstable.NewReader.
var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{}
// SSTableRawTombstonesOpt is a sstable.Reader option for disabling
// fragmentation of the range tombstones returned by
// sstable.Reader.NewRangeDelIter(). Used by debug tools to get a raw view of
// the tombstones contained in an sstable.
var SSTableRawTombstonesOpt interface{}
// SSTableWriterDisableKeyOrderChecks is a hook for disabling the key ordering
// invariant check performed by sstable.Writer. It is intended for internal use
// only in the construction of invalid sstables for testing. See
// tool/make_test_sstables.go.
var SSTableWriterDisableKeyOrderChecks func(interface{})
// SSTableInternalProperties is a func(*sstable.Writer) *sstable.Properties
// function that allows Pebble-internal code to mutate properties that external
// sstable writers are not permitted to edit. It's an untyped interface{} to
// avoid a cyclic dependency.
var SSTableInternalProperties interface{}

43
vendor/github.com/cockroachdb/pebble/internal/rangedel/rangedel.go generated vendored Normal file
View File

@@ -0,0 +1,43 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package rangedel
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// Encode takes a Span containing only range deletions. It invokes the provided
// closure with the encoded internal keys that represent the Span's state. The
// keys and values passed to emit are only valid until the closure returns. If
// emit returns an error, Encode stops and returns the error.
func Encode(s *keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
for _, k := range s.Keys {
if k.Kind() != base.InternalKeyKindRangeDelete {
return base.CorruptionErrorf("pebble: rangedel.Encode cannot encode %s key", k.Kind())
}
ik := base.InternalKey{
UserKey: s.Start,
Trailer: k.Trailer,
}
if err := emit(ik, s.End); err != nil {
return err
}
}
return nil
}
// Decode takes an internal key pair encoding a range deletion and returns a
// decoded keyspan containing the key. If keysDst is provided, the key will be
// appended to keysDst, avoiding an allocation.
func Decode(ik base.InternalKey, v []byte, keysDst []keyspan.Key) keyspan.Span {
return keyspan.Span{
Start: ik.UserKey,
End: v,
Keys: append(keysDst, keyspan.Key{
Trailer: ik.Trailer,
}),
}
}

Some files were not shown because too many files have changed in this diff Show More