Initial commit

vendor/github.com/cockroachdb/pebble/.editorconfig

@@ -0,0 +1,10 @@
+# See http://editorconfig.org
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+
+# For non-go files, we indent with two spaces. In go files we indent
+# with tabs but still set indent_size to control the github web viewer.
+indent_size=2

vendor/github.com/cockroachdb/pebble/.gitignore

@@ -0,0 +1,9 @@
+# Github action artifacts.
+artifacts
+# Profiling artifacts.
+cpu.*.prof
+heap.prof
+mutex.prof
+coverprofile.out
+# Testing artifacts
+meta.*.test

vendor/github.com/cockroachdb/pebble/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 The LevelDB-Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/cockroachdb/pebble/Makefile

@@ -0,0 +1,131 @@
+GO := go
+PKG := ./...
+GOFLAGS :=
+STRESSFLAGS :=
+TAGS := invariants
+TESTS := .
+PREV_RELEASE := crl-release-23.1
+COVER_PROFILE := coverprofile.out
+
+.PHONY: all
+all:
+	@echo usage:
+	@echo "  make test"
+	@echo "  make testrace"
+	@echo "  make stress"
+	@echo "  make stressrace"
+	@echo "  make stressmeta"
+	@echo "  make crossversion-meta"
+	@echo "  make testcoverage"
+	@echo "  make mod-update"
+	@echo "  make generate"
+	@echo "  make generate-test-data"
+	@echo "  make clean"
+
+override testflags :=
+.PHONY: test
+test:
+	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG}
+
+.PHONY: testcoverage
+testcoverage:
+	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} -coverprofile ${COVER_PROFILE}
+
+.PHONY: testrace
+testrace: testflags += -race -timeout 20m
+testrace: test
+
+testasan: testflags += -asan -timeout 20m
+testasan: test
+
+testmsan: export CC=clang
+testmsan: testflags += -msan -timeout 20m
+testmsan: test
+
+.PHONY: testobjiotracing
+testobjiotracing:
+	${GO} test -tags '$(TAGS) pebble_obj_io_tracing' ${testflags} -run ${TESTS} ./objstorage/objstorageprovider/objiotracing
+
+.PHONY: lint
+lint:
+	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ./internal/lint
+
+.PHONY: stress stressrace
+stressrace: testflags += -race
+stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 -test.v
+stress stressrace: test
+
+.PHONY: stressmeta
+stressmeta: override PKG = ./internal/metamorphic
+stressmeta: override STRESSFLAGS += -p 1
+stressmeta: override TESTS = TestMeta$$
+stressmeta: stress
+
+.PHONY: crossversion-meta
+crossversion-meta:
+	git checkout ${PREV_RELEASE}; \
+		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${PREV_RELEASE}.test'; \
+		git checkout -; \
+		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
+		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${PREV_RELEASE},${PREV_RELEASE},${PREV_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
+
+.PHONY: stress-crossversion
+stress-crossversion:
+	STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 crl-release-23.1 master
+
+.PHONY: generate
+generate:
+	${GO} generate ${PKG}
+
+generate:
+
+# Note that the output of generate-test-data is not deterministic. This should
+# only be run manually as needed.
+.PHONY: generate-test-data
+generate-test-data:
+	${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
+	${GO} run -tags make_test_find_db ./tool/make_test_find_db.go
+	${GO} run -tags make_test_sstables ./tool/make_test_sstables.go
+	${GO} run -tags make_test_remotecat ./tool/make_test_remotecat.go
+
+mod-update:
+	${GO} get -u
+	${GO} mod tidy
+
+.PHONY: clean
+clean:
+	rm -f $(patsubst %,%.test,$(notdir $(shell go list ${PKG})))
+
+git_dirty := $(shell git status -s)
+
+.PHONY: git-clean-check
+git-clean-check:
+ifneq ($(git_dirty),)
+	@echo "Git repository is dirty!"
+	@false
+else
+	@echo "Git repository is clean."
+endif
+
+.PHONY: mod-tidy-check
+mod-tidy-check:
+ifneq ($(git_dirty),)
+	$(error mod-tidy-check must be invoked on a clean repository)
+endif
+	@${GO} mod tidy
+	$(MAKE) git-clean-check
+
+# TODO(radu): switch back to @latest once bogus doc changes are
+# addressed; see https://github.com/cockroachdb/crlfmt/pull/44
+.PHONY: format
+format:
+	go install github.com/cockroachdb/crlfmt@44a36ec7 && crlfmt -w -tab 2 .
+
+.PHONY: format-check
+format-check:
+ifneq ($(git_dirty),)
+	$(error format-check must be invoked on a clean repository)
+endif
+	$(MAKE) format
+	git diff
+	$(MAKE) git-clean-check

vendor/github.com/cockroachdb/pebble/README.md

@@ -0,0 +1,225 @@
+# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) <sup><sub><sub>[Coverage](https://storage.googleapis.com/crl-codecover-public/pebble/index.html)</sub></sub></sup>
+
+#### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
+
+Pebble is a LevelDB/RocksDB inspired key-value store focused on
+performance and internal usage by CockroachDB. Pebble inherits the
+RocksDB file formats and a few extensions such as range deletion
+tombstones, table-level bloom filters, and updates to the MANIFEST
+format.
+
+Pebble intentionally does not aspire to include every feature in RocksDB and
+specifically targets the use case and feature set needed by CockroachDB:
+
+* Block-based tables
+* Checkpoints
+* Indexed batches
+* Iterator options (lower/upper bound, table filter)
+* Level-based compaction
+* Manual compaction
+* Merge operator
+* Prefix bloom filters
+* Prefix iteration
+* Range deletion tombstones
+* Reverse iteration
+* SSTable ingestion
+* Single delete
+* Snapshots
+* Table-level bloom filters
+
+RocksDB has a large number of features that are not implemented in
+Pebble:
+
+* Backups
+* Column families
+* Delete files in range
+* FIFO compaction style
+* Forward iterator / tailing iterator
+* Hash table format
+* Memtable bloom filter
+* Persistent cache
+* Pin iterator key / value
+* Plain table format
+* SSTable ingest-behind
+* Sub-compactions
+* Transactions
+* Universal compaction style
+
+***WARNING***: Pebble may silently corrupt data or behave incorrectly if
+used with a RocksDB database that uses a feature Pebble doesn't
+support. Caveat emptor!
+
+## Production Ready
+
+Pebble was introduced as an alternative storage engine to RocksDB in
+CockroachDB v20.1 (released May 2020) and was used in production
+successfully at that time. Pebble was made the default storage engine
+in CockroachDB v20.2 (released Nov 2020). Pebble is being used in
+production by users of CockroachDB at scale and is considered stable
+and production ready.
+
+## Advantages
+
+Pebble offers several improvements over RocksDB:
+
+* Faster reverse iteration via backwards links in the memtable's
+  skiplist.
+* Faster commit pipeline that achieves better concurrency.
+* Seamless merged iteration of indexed batches. The mutations in the
+  batch conceptually occupy another memtable level.
+* L0 sublevels and flush splitting for concurrent compactions out of L0 and
+  reduced read-amplification during heavy write load.
+* Faster LSM edits in LSMs with large numbers of sstables through use of a
+  copy-on-write B-tree to hold file metadata.
+* Delete-only compactions that drop whole sstables that fall within the bounds
+  of a range deletion.
+* Block-property collectors and filters that enable iterators to skip tables,
+  index blocks and data blocks that are irrelevant, according to user-defined
+  properties over key-value pairs.
+* Range keys API, allowing KV pairs defined over a range of keyspace with
+  user-defined semantics and interleaved during iteration.
+* Smaller, more approachable code base.
+
+See the [Pebble vs RocksDB: Implementation
+Differences](docs/rocksdb.md) doc for more details on implementation
+differences.
+
+## RocksDB Compatibility
+
+Pebble strives for forward compatibility with RocksDB 6.2.1 (the latest
+version of RocksDB used by CockroachDB). Forward compatibility means
+that a DB generated by RocksDB can be used by Pebble. Currently, Pebble
+provides bidirectional compatibility with RocksDB (a Pebble generated DB
+can be used by RocksDB) when using its FormatMostCompatible format. New
+functionality that is backwards incompatible is gated behind new format
+major versions. In general, Pebble only provides compatibility with the
+subset of functionality and configuration used by CockroachDB. The scope
+of RocksDB functionality and configuration is too large to adequately
+test and document all the incompatibilities. The list below contains
+known incompatibilities.
+
+* Pebble's use of WAL recycling is only compatible with RocksDB's
+  `kTolerateCorruptedTailRecords` WAL recovery mode. Older versions of
+  RocksDB would automatically map incompatible WAL recovery modes to
+  `kTolerateCorruptedTailRecords`. New versions of RocksDB will
+  disable WAL recycling.
+* Column families. Pebble does not support column families, nor does
+  it attempt to detect their usage when opening a DB that may contain
+  them.
+* Hash table format. Pebble does not support the hash table sstable
+  format.
+* Plain table format. Pebble does not support the plain table sstable
+  format.
+* SSTable format version 3 and 4. Pebble does not support version 3
+  and version 4 format sstables. The sstable format version is
+  controlled by the `BlockBasedTableOptions::format_version` option.
+  See [#97](https://github.com/cockroachdb/pebble/issues/97).
+
+## Format major versions
+
+Over time Pebble has introduced new physical file formats.  Backwards
+incompatible changes are made through the introduction of 'format major
+versions'. By default, when Pebble opens a database, it defaults to
+`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB
+6.2.1 (with the caveats described above).
+
+To opt into new formats, a user may set `FormatMajorVersion` on the
+[`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options)
+supplied to
+[`Open`](https://pkg.go.dev/github.com/cockroachdb/pebble#Open), or
+upgrade the format major version at runtime using
+[`DB.RatchetFormatMajorVersion`](https://pkg.go.dev/github.com/cockroachdb/pebble#DB.RatchetFormatMajorVersion).
+Format major version upgrades are permanent; There is no option to
+return to an earlier format.
+
+The table below outlines the history of format major versions:
+
+| Name                               | Value | Migration  |
+|------------------------------------|-------|------------|
+| FormatMostCompatible               |   1   | No         |
+| FormatVersioned                    |   3   | No         |
+| FormatSetWithDelete                |   4   | No         |
+| FormatBlockPropertyCollector       |   5   | No         |
+| FormatSplitUserKeysMarked          |   6   | Background |
+| FormatSplitUserKeysMarkedCompacted |   7   | Blocking   |
+| FormatRangeKeys                    |   8   | No         |
+| FormatMinTableFormatPebblev1       |   9   | No         |
+| FormatPrePebblev1Marked            |  10   | Background |
+| FormatSSTableValueBlocks           |  12   | No         |
+| FormatFlushableIngest              |  13   | No         |
+| FormatPrePebblev1MarkedCompacted   |  14   | Blocking   |
+| FormatDeleteSizedAndObsolete       |  15   | No         |
+| FormatVirtualSSTables              |  16   | No         |
+
+Upgrading to a format major version with 'Background' in the migration
+column may trigger background activity to rewrite physical file
+formats, typically through compactions. Upgrading to a format major
+version with 'Blocking' in the migration column will block until a
+migration is complete. The database may continue to serve reads and
+writes if upgrading a live database through
+`RatchetFormatMajorVersion`, but the method call will not return until
+the migration is complete.
+
+For reference, the table below lists the range of supported Pebble format major
+versions for CockroachDB releases.
+
+| CockroachDB release | Earliest supported                 | Latest supported          |
+|---------------------|------------------------------------|---------------------------|
+| 20.1 through 21.1   | FormatMostCompatible               | FormatMostCompatible      |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 22.1                | FormatMostCompatible               | FormatSplitUserKeysMarked |
+| 22.2                | FormatMostCompatible               | FormatPrePebblev1Marked   |
+| 23.1                | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest     |
+| 23.2                | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables     |
+
+## Pedigree
+
+Pebble is based on the incomplete Go version of LevelDB:
+
+https://github.com/golang/leveldb
+
+The Go version of LevelDB is based on the C++ original:
+
+https://github.com/google/leveldb
+
+Optimizations and inspiration were drawn from RocksDB:
+
+https://github.com/facebook/rocksdb
+
+## Getting Started
+
+### Example Code
+
+```go
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/cockroachdb/pebble"
+)
+
+func main() {
+	db, err := pebble.Open("demo", &pebble.Options{})
+	if err != nil {
+		log.Fatal(err)
+	}
+	key := []byte("hello")
+	if err := db.Set(key, []byte("world"), pebble.Sync); err != nil {
+		log.Fatal(err)
+	}
+	value, closer, err := db.Get(key)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%s %s\n", key, value)
+	if err := closer.Close(); err != nil {
+		log.Fatal(err)
+	}
+	if err := db.Close(); err != nil {
+		log.Fatal(err)
+	}
+}
+```

vendor/github.com/cockroachdb/pebble/cache.go

@@ -0,0 +1,23 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import "github.com/cockroachdb/pebble/internal/cache"
+
+// Cache exports the cache.Cache type.
+type Cache = cache.Cache
+
+// NewCache creates a new cache of the specified size. Memory for the cache is
+// allocated on demand, not during initialization. The cache is created with a
+// reference count of 1. Each DB it is associated with adds a reference, so the
+// creator of the cache should usually release their reference after the DB is
+// created.
+//
+//	c := pebble.NewCache(...)
+//	defer c.Unref()
+//	d, err := pebble.Open(pebble.Options{Cache: c})
+func NewCache(size int64) *cache.Cache {
+	return cache.New(size)
+}

vendor/github.com/cockroachdb/pebble/checkpoint.go

@@ -0,0 +1,425 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"io"
+	"os"
+
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/record"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/pebble/vfs/atomicfs"
+)
+
+// checkpointOptions hold the optional parameters to construct checkpoint
+// snapshots.
+type checkpointOptions struct {
+	// flushWAL set to true will force a flush and sync of the WAL prior to
+	// checkpointing.
+	flushWAL bool
+
+	// If set, any SSTs that don't overlap with these spans are excluded from a checkpoint.
+	restrictToSpans []CheckpointSpan
+}
+
+// CheckpointOption set optional parameters used by `DB.Checkpoint`.
+type CheckpointOption func(*checkpointOptions)
+
+// WithFlushedWAL enables flushing and syncing the WAL prior to constructing a
+// checkpoint. This guarantees that any writes committed before calling
+// DB.Checkpoint will be part of that checkpoint.
+//
+// Note that this setting can only be useful in cases when some writes are
+// performed with Sync = false. Otherwise, the guarantee will already be met.
+//
+// Passing this option is functionally equivalent to calling
+// DB.LogData(nil, Sync) right before DB.Checkpoint.
+func WithFlushedWAL() CheckpointOption {
+	return func(opt *checkpointOptions) {
+		opt.flushWAL = true
+	}
+}
+
+// WithRestrictToSpans specifies spans of interest for the checkpoint. Any SSTs
+// that don't overlap with any of these spans are excluded from the checkpoint.
+//
+// Note that the checkpoint can still surface keys outside of these spans (from
+// the WAL and from SSTs that partially overlap with these spans). Moreover,
+// these surface keys aren't necessarily "valid" in that they could have been
+// modified but the SST containing the modification is excluded.
+func WithRestrictToSpans(spans []CheckpointSpan) CheckpointOption {
+	return func(opt *checkpointOptions) {
+		opt.restrictToSpans = spans
+	}
+}
+
+// CheckpointSpan is a key range [Start, End) (inclusive on Start, exclusive on
+// End) of interest for a checkpoint.
+type CheckpointSpan struct {
+	Start []byte
+	End   []byte
+}
+
+// excludeFromCheckpoint returns true if an SST file should be excluded from the
+// checkpoint because it does not overlap with the spans of interest
+// (opt.restrictToSpans).
+func excludeFromCheckpoint(f *fileMetadata, opt *checkpointOptions, cmp Compare) bool {
+	if len(opt.restrictToSpans) == 0 {
+		// Option not set; don't exclude anything.
+		return false
+	}
+	for _, s := range opt.restrictToSpans {
+		if f.Overlaps(cmp, s.Start, s.End, true /* exclusiveEnd */) {
+			return false
+		}
+	}
+	// None of the restrictToSpans overlapped; we can exclude this file.
+	return true
+}
+
+// mkdirAllAndSyncParents creates destDir and any of its missing parents.
+// Those missing parents, as well as the closest existing ancestor, are synced.
+// Returns a handle to the directory created at destDir.
+func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
+	// Collect paths for all directories between destDir (excluded) and its
+	// closest existing ancestor (included).
+	var parentPaths []string
+	for parentPath := fs.PathDir(destDir); ; parentPath = fs.PathDir(parentPath) {
+		parentPaths = append(parentPaths, parentPath)
+		if fs.PathDir(parentPath) == parentPath {
+			break
+		}
+		_, err := fs.Stat(parentPath)
+		if err == nil {
+			// Exit loop at the closest existing ancestor.
+			break
+		}
+		if !oserror.IsNotExist(err) {
+			return nil, err
+		}
+	}
+	// Create destDir and any of its missing parents.
+	if err := fs.MkdirAll(destDir, 0755); err != nil {
+		return nil, err
+	}
+	// Sync all the parent directories up to the closest existing ancestor,
+	// included.
+	for _, parentPath := range parentPaths {
+		parentDir, err := fs.OpenDir(parentPath)
+		if err != nil {
+			return nil, err
+		}
+		err = parentDir.Sync()
+		if err != nil {
+			_ = parentDir.Close()
+			return nil, err
+		}
+		err = parentDir.Close()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return fs.OpenDir(destDir)
+}
+
+// Checkpoint constructs a snapshot of the DB instance in the specified
+// directory. The WAL, MANIFEST, OPTIONS, and sstables will be copied into the
+// snapshot. Hard links will be used when possible. Beware of the significant
+// space overhead for a checkpoint if hard links are disabled. Also beware that
+// even if hard links are used, the space overhead for the checkpoint will
+// increase over time as the DB performs compactions.
+func (d *DB) Checkpoint(
+	destDir string, opts ...CheckpointOption,
+) (
+	ckErr error, /* used in deferred cleanup */
+) {
+	opt := &checkpointOptions{}
+	for _, fn := range opts {
+		fn(opt)
+	}
+
+	if _, err := d.opts.FS.Stat(destDir); !oserror.IsNotExist(err) {
+		if err == nil {
+			return &os.PathError{
+				Op:   "checkpoint",
+				Path: destDir,
+				Err:  oserror.ErrExist,
+			}
+		}
+		return err
+	}
+
+	if opt.flushWAL && !d.opts.DisableWAL {
+		// Write an empty log-data record to flush and sync the WAL.
+		if err := d.LogData(nil /* data */, Sync); err != nil {
+			return err
+		}
+	}
+
+	// Disable file deletions.
+	d.mu.Lock()
+	d.disableFileDeletions()
+	defer func() {
+		d.mu.Lock()
+		defer d.mu.Unlock()
+		d.enableFileDeletions()
+	}()
+
+	// TODO(peter): RocksDB provides the option to roll the manifest if the
+	// MANIFEST size is too large. Should we do this too?
+
+	// Lock the manifest before getting the current version. We need the
+	// length of the manifest that we read to match the current version that
+	// we read, otherwise we might copy a versionEdit not reflected in the
+	// sstables we copy/link.
+	d.mu.versions.logLock()
+	// Get the unflushed log files, the current version, and the current manifest
+	// file number.
+	memQueue := d.mu.mem.queue
+	current := d.mu.versions.currentVersion()
+	formatVers := d.FormatMajorVersion()
+	manifestFileNum := d.mu.versions.manifestFileNum
+	manifestSize := d.mu.versions.manifest.Size()
+	optionsFileNum := d.optionsFileNum
+	virtualBackingFiles := make(map[base.DiskFileNum]struct{})
+	for diskFileNum := range d.mu.versions.backingState.fileBackingMap {
+		virtualBackingFiles[diskFileNum] = struct{}{}
+	}
+	// Release the manifest and DB.mu so we don't block other operations on
+	// the database.
+	d.mu.versions.logUnlock()
+	d.mu.Unlock()
+
+	// Wrap the normal filesystem with one which wraps newly created files with
+	// vfs.NewSyncingFile.
+	fs := vfs.NewSyncingFS(d.opts.FS, vfs.SyncingFileOptions{
+		NoSyncOnClose: d.opts.NoSyncOnClose,
+		BytesPerSync:  d.opts.BytesPerSync,
+	})
+
+	// Create the dir and its parents (if necessary), and sync them.
+	var dir vfs.File
+	defer func() {
+		if dir != nil {
+			_ = dir.Close()
+		}
+		if ckErr != nil {
+			// Attempt to cleanup on error.
+			_ = fs.RemoveAll(destDir)
+		}
+	}()
+	dir, ckErr = mkdirAllAndSyncParents(fs, destDir)
+	if ckErr != nil {
+		return ckErr
+	}
+
+	{
+		// Link or copy the OPTIONS.
+		srcPath := base.MakeFilepath(fs, d.dirname, fileTypeOptions, optionsFileNum)
+		destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
+		ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
+		if ckErr != nil {
+			return ckErr
+		}
+	}
+
+	{
+		// Set the format major version in the destination directory.
+		var versionMarker *atomicfs.Marker
+		versionMarker, _, ckErr = atomicfs.LocateMarker(fs, destDir, formatVersionMarkerName)
+		if ckErr != nil {
+			return ckErr
+		}
+
+		// We use the marker to encode the active format version in the
+		// marker filename. Unlike other uses of the atomic marker,
+		// there is no file with the filename `formatVers.String()` on
+		// the filesystem.
+		ckErr = versionMarker.Move(formatVers.String())
+		if ckErr != nil {
+			return ckErr
+		}
+		ckErr = versionMarker.Close()
+		if ckErr != nil {
+			return ckErr
+		}
+	}
+
+	var excludedFiles map[deletedFileEntry]*fileMetadata
+	// Set of FileBacking.DiskFileNum which will be required by virtual sstables
+	// in the checkpoint.
+	requiredVirtualBackingFiles := make(map[base.DiskFileNum]struct{})
+	// Link or copy the sstables.
+	for l := range current.Levels {
+		iter := current.Levels[l].Iter()
+		for f := iter.First(); f != nil; f = iter.Next() {
+			if excludeFromCheckpoint(f, opt, d.cmp) {
+				if excludedFiles == nil {
+					excludedFiles = make(map[deletedFileEntry]*fileMetadata)
+				}
+				excludedFiles[deletedFileEntry{
+					Level:   l,
+					FileNum: f.FileNum,
+				}] = f
+				continue
+			}
+
+			fileBacking := f.FileBacking
+			if f.Virtual {
+				if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok {
+					continue
+				}
+				requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{}
+			}
+
+			srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum)
+			destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
+			ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
+			if ckErr != nil {
+				return ckErr
+			}
+		}
+	}
+
+	var removeBackingTables []base.DiskFileNum
+	for diskFileNum := range virtualBackingFiles {
+		if _, ok := requiredVirtualBackingFiles[diskFileNum]; !ok {
+			// The backing sstable associated with fileNum is no longer
+			// required.
+			removeBackingTables = append(removeBackingTables, diskFileNum)
+		}
+	}
+
+	ckErr = d.writeCheckpointManifest(
+		fs, formatVers, destDir, dir, manifestFileNum.DiskFileNum(), manifestSize,
+		excludedFiles, removeBackingTables,
+	)
+	if ckErr != nil {
+		return ckErr
+	}
+
+	// Copy the WAL files. We copy rather than link because WAL file recycling
+	// will cause the WAL files to be reused which would invalidate the
+	// checkpoint.
+	for i := range memQueue {
+		logNum := memQueue[i].logNum
+		if logNum == 0 {
+			continue
+		}
+		srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum.DiskFileNum())
+		destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
+		ckErr = vfs.Copy(fs, srcPath, destPath)
+		if ckErr != nil {
+			return ckErr
+		}
+	}
+
+	// Sync and close the checkpoint directory.
+	ckErr = dir.Sync()
+	if ckErr != nil {
+		return ckErr
+	}
+	ckErr = dir.Close()
+	dir = nil
+	return ckErr
+}
+
+func (d *DB) writeCheckpointManifest(
+	fs vfs.FS,
+	formatVers FormatMajorVersion,
+	destDirPath string,
+	destDir vfs.File,
+	manifestFileNum base.DiskFileNum,
+	manifestSize int64,
+	excludedFiles map[deletedFileEntry]*fileMetadata,
+	removeBackingTables []base.DiskFileNum,
+) error {
+	// Copy the MANIFEST, and create a pointer to it. We copy rather
+	// than link because additional version edits added to the
+	// MANIFEST after we took our snapshot of the sstables will
+	// reference sstables that aren't in our checkpoint. For a
+	// similar reason, we need to limit how much of the MANIFEST we
+	// copy.
+	// If some files are excluded from the checkpoint, also append a block that
+	// records those files as deleted.
+	if err := func() error {
+		srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum)
+		destPath := fs.PathJoin(destDirPath, fs.PathBase(srcPath))
+		src, err := fs.Open(srcPath, vfs.SequentialReadsOption)
+		if err != nil {
+			return err
+		}
+		defer src.Close()
+
+		dst, err := fs.Create(destPath)
+		if err != nil {
+			return err
+		}
+		defer dst.Close()
+
+		// Copy all existing records. We need to copy at the record level in case we
+		// need to append another record with the excluded files (we cannot simply
+		// append a record after a raw data copy; see
+		// https://github.com/cockroachdb/cockroach/issues/100935).
+		r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum.FileNum())
+		w := record.NewWriter(dst)
+		for {
+			rr, err := r.Next()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return err
+			}
+
+			rw, err := w.Next()
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(rw, rr); err != nil {
+				return err
+			}
+		}
+
+		if len(excludedFiles) > 0 {
+			// Write out an additional VersionEdit that deletes the excluded SST files.
+			ve := versionEdit{
+				DeletedFiles:         excludedFiles,
+				RemovedBackingTables: removeBackingTables,
+			}
+
+			rw, err := w.Next()
+			if err != nil {
+				return err
+			}
+			if err := ve.Encode(rw); err != nil {
+				return err
+			}
+		}
+		if err := w.Close(); err != nil {
+			return err
+		}
+		return dst.Sync()
+	}(); err != nil {
+		return err
+	}
+
+	// Recent format versions use an atomic marker for setting the
+	// active manifest. Older versions use the CURRENT file. The
+	// setCurrentFunc function will return a closure that will
+	// take the appropriate action for the database's format
+	// version.
+	var manifestMarker *atomicfs.Marker
+	manifestMarker, _, err := atomicfs.LocateMarker(fs, destDirPath, manifestMarkerName)
+	if err != nil {
+		return err
+	}
+	if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum.FileNum()); err != nil {
+		return err
+	}
+	return manifestMarker.Close()
+}

vendor/github.com/cockroachdb/pebble/cleaner.go

@@ -0,0 +1,295 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"context"
+	"runtime/pprof"
+	"sync"
+	"time"
+
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/tokenbucket"
+)
+
+// Cleaner exports the base.Cleaner type.
+type Cleaner = base.Cleaner
+
+// DeleteCleaner exports the base.DeleteCleaner type.
+type DeleteCleaner = base.DeleteCleaner
+
+// ArchiveCleaner exports the base.ArchiveCleaner type.
+type ArchiveCleaner = base.ArchiveCleaner
+
+type cleanupManager struct {
+	opts            *Options
+	objProvider     objstorage.Provider
+	onTableDeleteFn func(fileSize uint64)
+	deletePacer     *deletionPacer
+
+	// jobsCh is used as the cleanup job queue.
+	jobsCh chan *cleanupJob
+	// waitGroup is used to wait for the background goroutine to exit.
+	waitGroup sync.WaitGroup
+
+	mu struct {
+		sync.Mutex
+		// totalJobs is the total number of enqueued jobs (completed or in progress).
+		totalJobs              int
+		completedJobs          int
+		completedJobsCond      sync.Cond
+		jobsQueueWarningIssued bool
+	}
+}
+
+// We can queue this many jobs before we have to block EnqueueJob.
+const jobsQueueDepth = 1000
+
+// obsoleteFile holds information about a file that needs to be deleted soon.
+type obsoleteFile struct {
+	dir      string
+	fileNum  base.DiskFileNum
+	fileType fileType
+	fileSize uint64
+}
+
+type cleanupJob struct {
+	jobID         int
+	obsoleteFiles []obsoleteFile
+}
+
+// openCleanupManager creates a cleanupManager and starts its background goroutine.
+// The cleanupManager must be Close()d.
+func openCleanupManager(
+	opts *Options,
+	objProvider objstorage.Provider,
+	onTableDeleteFn func(fileSize uint64),
+	getDeletePacerInfo func() deletionPacerInfo,
+) *cleanupManager {
+	cm := &cleanupManager{
+		opts:            opts,
+		objProvider:     objProvider,
+		onTableDeleteFn: onTableDeleteFn,
+		deletePacer:     newDeletionPacer(time.Now(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo),
+		jobsCh:          make(chan *cleanupJob, jobsQueueDepth),
+	}
+	cm.mu.completedJobsCond.L = &cm.mu.Mutex
+	cm.waitGroup.Add(1)
+
+	go func() {
+		pprof.Do(context.Background(), gcLabels, func(context.Context) {
+			cm.mainLoop()
+		})
+	}()
+
+	return cm
+}
+
+// Close stops the background goroutine, waiting until all queued jobs are completed.
+// Delete pacing is disabled for the remaining jobs.
+func (cm *cleanupManager) Close() {
+	close(cm.jobsCh)
+	cm.waitGroup.Wait()
+}
+
+// EnqueueJob adds a cleanup job to the manager's queue.
+func (cm *cleanupManager) EnqueueJob(jobID int, obsoleteFiles []obsoleteFile) {
+	job := &cleanupJob{
+		jobID:         jobID,
+		obsoleteFiles: obsoleteFiles,
+	}
+
+	// Report deleted bytes to the pacer, which can use this data to potentially
+	// increase the deletion rate to keep up. We want to do this at enqueue time
+	// rather than when we get to the job, otherwise the reported bytes will be
+	// subject to the throttling rate which defeats the purpose.
+	var pacingBytes uint64
+	for _, of := range obsoleteFiles {
+		if cm.needsPacing(of.fileType, of.fileNum) {
+			pacingBytes += of.fileSize
+		}
+	}
+	if pacingBytes > 0 {
+		cm.deletePacer.ReportDeletion(time.Now(), pacingBytes)
+	}
+
+	cm.mu.Lock()
+	cm.mu.totalJobs++
+	cm.maybeLogLocked()
+	cm.mu.Unlock()
+
+	if invariants.Enabled && len(cm.jobsCh) >= cap(cm.jobsCh)-2 {
+		panic("cleanup jobs queue full")
+	}
+
+	cm.jobsCh <- job
+}
+
+// Wait until the completion of all jobs that were already queued.
+//
+// Does not wait for jobs that are enqueued during the call.
+//
+// Note that DB.mu should not be held while calling this method; the background
+// goroutine needs to acquire DB.mu to update deleted table metrics.
+func (cm *cleanupManager) Wait() {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	n := cm.mu.totalJobs
+	for cm.mu.completedJobs < n {
+		cm.mu.completedJobsCond.Wait()
+	}
+}
+
+// mainLoop runs the manager's background goroutine.
+func (cm *cleanupManager) mainLoop() {
+	defer cm.waitGroup.Done()
+
+	var tb tokenbucket.TokenBucket
+	// Use a token bucket with 1 token / second refill rate and 1 token burst.
+	tb.Init(1.0, 1.0)
+	for job := range cm.jobsCh {
+		for _, of := range job.obsoleteFiles {
+			if of.fileType != fileTypeTable {
+				path := base.MakeFilepath(cm.opts.FS, of.dir, of.fileType, of.fileNum)
+				cm.deleteObsoleteFile(of.fileType, job.jobID, path, of.fileNum, of.fileSize)
+			} else {
+				cm.maybePace(&tb, of.fileType, of.fileNum, of.fileSize)
+				cm.onTableDeleteFn(of.fileSize)
+				cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.fileNum)
+			}
+		}
+		cm.mu.Lock()
+		cm.mu.completedJobs++
+		cm.mu.completedJobsCond.Broadcast()
+		cm.maybeLogLocked()
+		cm.mu.Unlock()
+	}
+}
+
+func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNum base.DiskFileNum) bool {
+	if fileType != fileTypeTable {
+		return false
+	}
+	meta, err := cm.objProvider.Lookup(fileType, fileNum)
+	if err != nil {
+		// The object was already removed from the provider; we won't actually
+		// delete anything, so we don't need to pace.
+		return false
+	}
+	// Don't throttle deletion of remote objects.
+	return !meta.IsRemote()
+}
+
+// maybePace sleeps before deleting an object if appropriate. It is always
+// called from the background goroutine.
+func (cm *cleanupManager) maybePace(
+	tb *tokenbucket.TokenBucket, fileType base.FileType, fileNum base.DiskFileNum, fileSize uint64,
+) {
+	if !cm.needsPacing(fileType, fileNum) {
+		return
+	}
+
+	tokens := cm.deletePacer.PacingDelay(time.Now(), fileSize)
+	if tokens == 0.0 {
+		// The token bucket might be in debt; it could make us wait even for 0
+		// tokens. We don't want that if the pacer decided throttling should be
+		// disabled.
+		return
+	}
+	// Wait for tokens. We use a token bucket instead of sleeping outright because
+	// the token bucket accumulates up to one second of unused tokens.
+	for {
+		ok, d := tb.TryToFulfill(tokenbucket.Tokens(tokens))
+		if ok {
+			break
+		}
+		time.Sleep(d)
+	}
+}
+
+// deleteObsoleteFile deletes a (non-object) file that is no longer needed.
+func (cm *cleanupManager) deleteObsoleteFile(
+	fileType fileType, jobID int, path string, fileNum base.DiskFileNum, fileSize uint64,
+) {
+	// TODO(peter): need to handle this error, probably by re-adding the
+	// file that couldn't be deleted to one of the obsolete slices map.
+	err := cm.opts.Cleaner.Clean(cm.opts.FS, fileType, path)
+	if oserror.IsNotExist(err) {
+		return
+	}
+
+	switch fileType {
+	case fileTypeLog:
+		cm.opts.EventListener.WALDeleted(WALDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	case fileTypeManifest:
+		cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	case fileTypeTable:
+		panic("invalid deletion of object file")
+	}
+}
+
+func (cm *cleanupManager) deleteObsoleteObject(
+	fileType fileType, jobID int, fileNum base.DiskFileNum,
+) {
+	if fileType != fileTypeTable {
+		panic("not an object")
+	}
+
+	var path string
+	meta, err := cm.objProvider.Lookup(fileType, fileNum)
+	if err != nil {
+		path = "<nil>"
+	} else {
+		path = cm.objProvider.Path(meta)
+		err = cm.objProvider.Remove(fileType, fileNum)
+	}
+	if cm.objProvider.IsNotExistError(err) {
+		return
+	}
+
+	switch fileType {
+	case fileTypeTable:
+		cm.opts.EventListener.TableDeleted(TableDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	}
+}
+
+// maybeLogLocked issues a log if the job queue gets 75% full and issues a log
+// when the job queue gets back to less than 10% full.
+//
+// Must be called with cm.mu locked.
+func (cm *cleanupManager) maybeLogLocked() {
+	const highThreshold = jobsQueueDepth * 3 / 4
+	const lowThreshold = jobsQueueDepth / 10
+
+	jobsInQueue := cm.mu.totalJobs - cm.mu.completedJobs
+
+	if !cm.mu.jobsQueueWarningIssued && jobsInQueue > highThreshold {
+		cm.mu.jobsQueueWarningIssued = true
+		cm.opts.Logger.Infof("cleanup falling behind; job queue has over %d jobs", highThreshold)
+	}
+
+	if cm.mu.jobsQueueWarningIssued && jobsInQueue < lowThreshold {
+		cm.mu.jobsQueueWarningIssued = false
+		cm.opts.Logger.Infof("cleanup back to normal; job queue has under %d jobs", lowThreshold)
+	}
+}

vendor/github.com/cockroachdb/pebble/commit.go

@@ -0,0 +1,517 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/cockroachdb/pebble/record"
+)
+
+// commitQueue is a lock-free fixed-size single-producer, multi-consumer
+// queue. The single producer can enqueue (push) to the head, and consumers can
+// dequeue (pop) from the tail.
+//
+// It has the added feature that it nils out unused slots to avoid unnecessary
+// retention of objects.
+type commitQueue struct {
+	// headTail packs together a 32-bit head index and a 32-bit tail index. Both
+	// are indexes into slots modulo len(slots)-1.
+	//
+	// tail = index of oldest data in queue
+	// head = index of next slot to fill
+	//
+	// Slots in the range [tail, head) are owned by consumers.  A consumer
+	// continues to own a slot outside this range until it nils the slot, at
+	// which point ownership passes to the producer.
+	//
+	// The head index is stored in the most-significant bits so that we can
+	// atomically add to it and the overflow is harmless.
+	headTail atomic.Uint64
+
+	// slots is a ring buffer of values stored in this queue. The size must be a
+	// power of 2. A slot is in use until *both* the tail index has moved beyond
+	// it and the slot value has been set to nil. The slot value is set to nil
+	// atomically by the consumer and read atomically by the producer.
+	slots [record.SyncConcurrency]atomic.Pointer[Batch]
+}
+
+const dequeueBits = 32
+
+func (q *commitQueue) unpack(ptrs uint64) (head, tail uint32) {
+	const mask = 1<<dequeueBits - 1
+	head = uint32((ptrs >> dequeueBits) & mask)
+	tail = uint32(ptrs & mask)
+	return
+}
+
+func (q *commitQueue) pack(head, tail uint32) uint64 {
+	const mask = 1<<dequeueBits - 1
+	return (uint64(head) << dequeueBits) |
+		uint64(tail&mask)
+}
+
+func (q *commitQueue) enqueue(b *Batch) {
+	ptrs := q.headTail.Load()
+	head, tail := q.unpack(ptrs)
+	if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
+		// Queue is full. This should never be reached because commitPipeline.commitQueueSem
+		// limits the number of concurrent operations.
+		panic("pebble: not reached")
+	}
+	slot := &q.slots[head&uint32(len(q.slots)-1)]
+
+	// Check if the head slot has been released by dequeueApplied.
+	for slot.Load() != nil {
+		// Another goroutine is still cleaning up the tail, so the queue is
+		// actually still full. We spin because this should resolve itself
+		// momentarily.
+		runtime.Gosched()
+	}
+
+	// The head slot is free, so we own it.
+	slot.Store(b)
+
+	// Increment head. This passes ownership of slot to dequeueApplied and acts as a
+	// store barrier for writing the slot.
+	q.headTail.Add(1 << dequeueBits)
+}
+
+// dequeueApplied removes the earliest enqueued Batch, if it is applied.
+//
+// Returns nil if the commit queue is empty or the earliest Batch is not yet
+// applied.
+func (q *commitQueue) dequeueApplied() *Batch {
+	for {
+		ptrs := q.headTail.Load()
+		head, tail := q.unpack(ptrs)
+		if tail == head {
+			// Queue is empty.
+			return nil
+		}
+
+		slot := &q.slots[tail&uint32(len(q.slots)-1)]
+		b := slot.Load()
+		if b == nil || !b.applied.Load() {
+			// The batch is not ready to be dequeued, or another goroutine has
+			// already dequeued it.
+			return nil
+		}
+
+		// Confirm head and tail (for our speculative check above) and increment
+		// tail. If this succeeds, then we own the slot at tail.
+		ptrs2 := q.pack(head, tail+1)
+		if q.headTail.CompareAndSwap(ptrs, ptrs2) {
+			// We now own slot.
+			//
+			// Tell enqueue that we're done with this slot. Zeroing the slot is also
+			// important so we don't leave behind references that could keep this object
+			// live longer than necessary.
+			slot.Store(nil)
+			// At this point enqueue owns the slot.
+			return b
+		}
+	}
+}
+
+// commitEnv contains the environment that a commitPipeline interacts
+// with. This allows fine-grained testing of commitPipeline behavior without
+// construction of an entire DB.
+type commitEnv struct {
+	// The next sequence number to give to a batch. Protected by
+	// commitPipeline.mu.
+	logSeqNum *atomic.Uint64
+	// The visible sequence number at which reads should be performed. Ratcheted
+	// upwards atomically as batches are applied to the memtable.
+	visibleSeqNum *atomic.Uint64
+
+	// Apply the batch to the specified memtable. Called concurrently.
+	apply func(b *Batch, mem *memTable) error
+	// Write the batch to the WAL. If wg != nil, the data will be persisted
+	// asynchronously and done will be called on wg upon completion. If wg != nil
+	// and err != nil, a failure to persist the WAL will populate *err. Returns
+	// the memtable the batch should be applied to. Serial execution enforced by
+	// commitPipeline.mu.
+	write func(b *Batch, wg *sync.WaitGroup, err *error) (*memTable, error)
+}
+
+// A commitPipeline manages the stages of committing a set of mutations
+// (contained in a single Batch) atomically to the DB. The steps are
+// conceptually:
+//
+//  1. Write the batch to the WAL and optionally sync the WAL
+//  2. Apply the mutations in the batch to the memtable
+//
+// These two simple steps are made complicated by the desire for high
+// performance. In the absence of concurrency, performance is limited by how
+// fast a batch can be written (and synced) to the WAL and then added to the
+// memtable, both of which are outside the purview of the commit
+// pipeline. Performance under concurrency is the primary concern of the commit
+// pipeline, though it also needs to maintain two invariants:
+//
+//  1. Batches need to be written to the WAL in sequence number order.
+//  2. Batches need to be made visible for reads in sequence number order. This
+//     invariant arises from the use of a single sequence number which
+//     indicates which mutations are visible.
+//
+// Taking these invariants into account, let's revisit the work the commit
+// pipeline needs to perform. Writing the batch to the WAL is necessarily
+// serialized as there is a single WAL object. The order of the entries in the
+// WAL defines the sequence number order. Note that writing to the WAL is
+// extremely fast, usually just a memory copy. Applying the mutations in a
+// batch to the memtable can occur concurrently as the underlying skiplist
+// supports concurrent insertions. Publishing the visible sequence number is
+// another serialization point, but one with a twist: the visible sequence
+// number cannot be bumped until the mutations for earlier batches have
+// finished applying to the memtable (the visible sequence number only ratchets
+// up). Lastly, if requested, the commit waits for the WAL to sync. Note that
+// waiting for the WAL sync after ratcheting the visible sequence number allows
+// another goroutine to read committed data before the WAL has synced. This is
+// similar behavior to RocksDB's manual WAL flush functionality. Application
+// code needs to protect against this if necessary.
+//
+// The full outline of the commit pipeline operation is as follows:
+//
+//	with commitPipeline mutex locked:
+//	  assign batch sequence number
+//	  write batch to WAL
+//	(optionally) add batch to WAL sync list
+//	apply batch to memtable (concurrently)
+//	wait for earlier batches to apply
+//	ratchet read sequence number
+//	(optionally) wait for the WAL to sync
+//
+// As soon as a batch has been written to the WAL, the commitPipeline mutex is
+// released allowing another batch to write to the WAL. Each commit operation
+// individually applies its batch to the memtable providing concurrency. The
+// WAL sync happens concurrently with applying to the memtable (see
+// commitPipeline.syncLoop).
+//
+// The "waits for earlier batches to apply" work is more complicated than might
+// be expected. The obvious approach would be to keep a queue of pending
+// batches and for each batch to wait for the previous batch to finish
+// committing. This approach was tried initially and turned out to be too
+// slow. The problem is that it causes excessive goroutine activity as each
+// committing goroutine needs to wake up in order for the next goroutine to be
+// unblocked. The approach taken in the current code is conceptually similar,
+// though it avoids waking a goroutine to perform work that another goroutine
+// can perform. A commitQueue (a single-producer, multiple-consumer queue)
+// holds the ordered list of committing batches. Addition to the queue is done
+// while holding commitPipeline.mutex ensuring the same ordering of batches in
+// the queue as the ordering in the WAL. When a batch finishes applying to the
+// memtable, it atomically updates its Batch.applied field. Ratcheting of the
+// visible sequence number is done by commitPipeline.publish which loops
+// dequeueing "applied" batches and ratcheting the visible sequence number. If
+// we hit an unapplied batch at the head of the queue we can block as we know
+// that committing of that unapplied batch will eventually find our (applied)
+// batch in the queue. See commitPipeline.publish for additional commentary.
+type commitPipeline struct {
+	// WARNING: The following struct `commitQueue` contains fields which will
+	// be accessed atomically.
+	//
+	// Go allocations are guaranteed to be 64-bit aligned which we take advantage
+	// of by placing the 64-bit fields which we access atomically at the beginning
+	// of the commitPipeline struct.
+	// For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
+	// Queue of pending batches to commit.
+	pending commitQueue
+	env     commitEnv
+	// The commit path has two queues:
+	// - commitPipeline.pending contains batches whose seqnums have not yet been
+	//   published. It is a lock-free single producer multi consumer queue.
+	// - LogWriter.flusher.syncQ contains state for batches that have asked for
+	//   a sync. It is a lock-free single producer single consumer queue.
+	// These lock-free queues have a fixed capacity. And since they are
+	// lock-free, we cannot do blocking waits when pushing onto these queues, in
+	// case they are full. Additionally, adding to these queues happens while
+	// holding commitPipeline.mu, and we don't want to block while holding that
+	// mutex since it is also needed by other code.
+	//
+	// Popping from these queues is independent and for a particular batch can
+	// occur in either order, though it is more common that popping from the
+	// commitPipeline.pending will happen first.
+	//
+	// Due to these constraints, we reserve a unit of space in each queue before
+	// acquiring commitPipeline.mu, which also ensures that the push operation
+	// is guaranteed to have space in the queue. The commitQueueSem and
+	// logSyncQSem are used for this reservation.
+	commitQueueSem chan struct{}
+	logSyncQSem    chan struct{}
+	ingestSem      chan struct{}
+	// The mutex to use for synchronizing access to logSeqNum and serializing
+	// calls to commitEnv.write().
+	mu sync.Mutex
+}
+
+func newCommitPipeline(env commitEnv) *commitPipeline {
+	p := &commitPipeline{
+		env: env,
+		// The capacity of both commitQueue.slots and syncQueue.slots is set to
+		// record.SyncConcurrency, which also determines the value of these
+		// semaphores. We used to have a single semaphore, which required that the
+		// capacity of these queues be the same. Now that we have two semaphores,
+		// the capacity of these queues could be changed to be different. Say half
+		// of the batches asked to be synced, but syncing took 5x the latency of
+		// adding to the memtable and publishing. Then syncQueue.slots could be
+		// sized as 0.5*5 of the commitQueue.slots. We can explore this if we find
+		// that LogWriterMetrics.SyncQueueLen has high utilization under some
+		// workloads.
+		//
+		// NB: the commit concurrency is one less than SyncConcurrency because we
+		// have to allow one "slot" for a concurrent WAL rotation which will close
+		// and sync the WAL.
+		commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
+		logSyncQSem:    make(chan struct{}, record.SyncConcurrency-1),
+		ingestSem:      make(chan struct{}, 1),
+	}
+	return p
+}
+
+// directWrite is used to directly write to the WAL. commitPipeline.mu must be
+// held while this is called. DB.mu must not be held. directWrite will only
+// return once the WAL sync is complete. Note that DirectWrite is a special case
+// function which is currently only used when ingesting sstables as a flushable.
+// Reason carefully about the correctness argument when calling this function
+// from any context.
+func (p *commitPipeline) directWrite(b *Batch) error {
+	var syncWG sync.WaitGroup
+	var syncErr error
+	syncWG.Add(1)
+	p.logSyncQSem <- struct{}{}
+	_, err := p.env.write(b, &syncWG, &syncErr)
+	syncWG.Wait()
+	err = firstError(err, syncErr)
+	return err
+}
+
+// Commit the specified batch, writing it to the WAL, optionally syncing the
+// WAL, and applying the batch to the memtable. Upon successful return the
+// batch's mutations will be visible for reading.
+// REQUIRES: noSyncWait => syncWAL
+func (p *commitPipeline) Commit(b *Batch, syncWAL bool, noSyncWait bool) error {
+	if b.Empty() {
+		return nil
+	}
+
+	commitStartTime := time.Now()
+	// Acquire semaphores.
+	p.commitQueueSem <- struct{}{}
+	if syncWAL {
+		p.logSyncQSem <- struct{}{}
+	}
+	b.commitStats.SemaphoreWaitDuration = time.Since(commitStartTime)
+
+	// Prepare the batch for committing: enqueuing the batch in the pending
+	// queue, determining the batch sequence number and writing the data to the
+	// WAL.
+	//
+	// NB: We set Batch.commitErr on error so that the batch won't be a candidate
+	// for reuse. See Batch.release().
+	mem, err := p.prepare(b, syncWAL, noSyncWait)
+	if err != nil {
+		b.db = nil // prevent batch reuse on error
+		// NB: we are not doing <-p.commitQueueSem since the batch is still
+		// sitting in the pending queue. We should consider fixing this by also
+		// removing the batch from the pending queue.
+		return err
+	}
+
+	// Apply the batch to the memtable.
+	if err := p.env.apply(b, mem); err != nil {
+		b.db = nil // prevent batch reuse on error
+		// NB: we are not doing <-p.commitQueueSem since the batch is still
+		// sitting in the pending queue. We should consider fixing this by also
+		// removing the batch from the pending queue.
+		return err
+	}
+
+	// Publish the batch sequence number.
+	p.publish(b)
+
+	<-p.commitQueueSem
+
+	if !noSyncWait {
+		// Already waited for commit, so look at the error.
+		if b.commitErr != nil {
+			b.db = nil // prevent batch reuse on error
+			err = b.commitErr
+		}
+	}
+	// Else noSyncWait. The LogWriter can be concurrently writing to
+	// b.commitErr. We will read b.commitErr in Batch.SyncWait after the
+	// LogWriter is done writing.
+
+	b.commitStats.TotalDuration = time.Since(commitStartTime)
+
+	return err
+}
+
+// AllocateSeqNum allocates count sequence numbers, invokes the prepare
+// callback, then the apply callback, and then publishes the sequence
+// numbers. AllocateSeqNum does not write to the WAL or add entries to the
+// memtable. AllocateSeqNum can be used to sequence an operation such as
+// sstable ingestion within the commit pipeline. The prepare callback is
+// invoked with commitPipeline.mu held, but note that DB.mu is not held and
+// must be locked if necessary.
+func (p *commitPipeline) AllocateSeqNum(
+	count int, prepare func(seqNum uint64), apply func(seqNum uint64),
+) {
+	// This method is similar to Commit and prepare. Be careful about trying to
+	// share additional code with those methods because Commit and prepare are
+	// performance critical code paths.
+
+	b := newBatch(nil)
+	defer b.release()
+
+	// Give the batch a count of 1 so that the log and visible sequence number
+	// are incremented correctly.
+	b.data = make([]byte, batchHeaderLen)
+	b.setCount(uint32(count))
+	b.commit.Add(1)
+
+	p.commitQueueSem <- struct{}{}
+
+	p.mu.Lock()
+
+	// Enqueue the batch in the pending queue. Note that while the pending queue
+	// is lock-free, we want the order of batches to be the same as the sequence
+	// number order.
+	p.pending.enqueue(b)
+
+	// Assign the batch a sequence number. Note that we use atomic operations
+	// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
+	// mutual exclusion for other goroutines writing to logSeqNum.
+	logSeqNum := p.env.logSeqNum.Add(uint64(count)) - uint64(count)
+	seqNum := logSeqNum
+	if seqNum == 0 {
+		// We can't use the value 0 for the global seqnum during ingestion, because
+		// 0 indicates no global seqnum. So allocate one more seqnum.
+		p.env.logSeqNum.Add(1)
+		seqNum++
+	}
+	b.setSeqNum(seqNum)
+
+	// Wait for any outstanding writes to the memtable to complete. This is
+	// necessary for ingestion so that the check for memtable overlap can see any
+	// writes that were sequenced before the ingestion. The spin loop is
+	// unfortunate, but obviates the need for additional synchronization.
+	for {
+		visibleSeqNum := p.env.visibleSeqNum.Load()
+		if visibleSeqNum == logSeqNum {
+			break
+		}
+		runtime.Gosched()
+	}
+
+	// Invoke the prepare callback. Note the lack of error reporting. Even if the
+	// callback internally fails, the sequence number needs to be published in
+	// order to allow the commit pipeline to proceed.
+	prepare(b.SeqNum())
+
+	p.mu.Unlock()
+
+	// Invoke the apply callback.
+	apply(b.SeqNum())
+
+	// Publish the sequence number.
+	p.publish(b)
+
+	<-p.commitQueueSem
+}
+
+func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memTable, error) {
+	n := uint64(b.Count())
+	if n == invalidBatchCount {
+		return nil, ErrInvalidBatch
+	}
+	var syncWG *sync.WaitGroup
+	var syncErr *error
+	switch {
+	case !syncWAL:
+		// Only need to wait for the publish.
+		b.commit.Add(1)
+	// Remaining cases represent syncWAL=true.
+	case noSyncWait:
+		syncErr = &b.commitErr
+		syncWG = &b.fsyncWait
+		// Only need to wait synchronously for the publish. The user will
+		// (asynchronously) wait on the batch's fsyncWait.
+		b.commit.Add(1)
+		b.fsyncWait.Add(1)
+	case !noSyncWait:
+		syncErr = &b.commitErr
+		syncWG = &b.commit
+		// Must wait for both the publish and the WAL fsync.
+		b.commit.Add(2)
+	}
+
+	p.mu.Lock()
+
+	// Enqueue the batch in the pending queue. Note that while the pending queue
+	// is lock-free, we want the order of batches to be the same as the sequence
+	// number order.
+	p.pending.enqueue(b)
+
+	// Assign the batch a sequence number. Note that we use atomic operations
+	// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
+	// mutual exclusion for other goroutines writing to logSeqNum.
+	b.setSeqNum(p.env.logSeqNum.Add(n) - n)
+
+	// Write the data to the WAL.
+	mem, err := p.env.write(b, syncWG, syncErr)
+
+	p.mu.Unlock()
+
+	return mem, err
+}
+
+func (p *commitPipeline) publish(b *Batch) {
+	// Mark the batch as applied.
+	b.applied.Store(true)
+
+	// Loop dequeuing applied batches from the pending queue. If our batch was
+	// the head of the pending queue we are guaranteed that either we'll publish
+	// it or someone else will dequeueApplied and publish it. If our batch is not the
+	// head of the queue then either we'll dequeueApplied applied batches and reach our
+	// batch or there is an unapplied batch blocking us. When that unapplied
+	// batch applies it will go through the same process and publish our batch
+	// for us.
+	for {
+		t := p.pending.dequeueApplied()
+		if t == nil {
+			// Wait for another goroutine to publish us. We might also be waiting for
+			// the WAL sync to finish.
+			now := time.Now()
+			b.commit.Wait()
+			b.commitStats.CommitWaitDuration += time.Since(now)
+			break
+		}
+		if !t.applied.Load() {
+			panic("not reached")
+		}
+
+		// We're responsible for publishing the sequence number for batch t, but
+		// another concurrent goroutine might sneak in and publish the sequence
+		// number for a subsequent batch. That's ok as all we're guaranteeing is
+		// that the sequence number ratchets up.
+		for {
+			curSeqNum := p.env.visibleSeqNum.Load()
+			newSeqNum := t.SeqNum() + uint64(t.Count())
+			if newSeqNum <= curSeqNum {
+				// t's sequence number has already been published.
+				break
+			}
+			if p.env.visibleSeqNum.CompareAndSwap(curSeqNum, newSeqNum) {
+				// We successfully published t's sequence number.
+				break
+			}
+		}
+
+		t.commit.Done()
+	}
+}

vendor/github.com/cockroachdb/pebble/comparer.go

@@ -0,0 +1,31 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Compare exports the base.Compare type.
+type Compare = base.Compare
+
+// Equal exports the base.Equal type.
+type Equal = base.Equal
+
+// AbbreviatedKey exports the base.AbbreviatedKey type.
+type AbbreviatedKey = base.AbbreviatedKey
+
+// Separator exports the base.Separator type.
+type Separator = base.Separator
+
+// Successor exports the base.Successor type.
+type Successor = base.Successor
+
+// Split exports the base.Split type.
+type Split = base.Split
+
+// Comparer exports the base.Comparer type.
+type Comparer = base.Comparer
+
+// DefaultComparer exports the base.DefaultComparer variable.
+var DefaultComparer = base.DefaultComparer

vendor/github.com/cockroachdb/pebble/error_iter.go

@@ -0,0 +1,90 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/keyspan"
+)
+
+type errorIter struct {
+	err error
+}
+
+// errorIter implements the base.InternalIterator interface.
+var _ internalIterator = (*errorIter)(nil)
+
+func newErrorIter(err error) *errorIter {
+	return &errorIter{err: err}
+}
+
+func (c *errorIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) First() (*InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) Last() (*InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) Next() (*InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) Prev() (*InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) {
+	return nil, base.LazyValue{}
+}
+
+func (c *errorIter) Error() error {
+	return c.err
+}
+
+func (c *errorIter) Close() error {
+	return c.err
+}
+
+func (c *errorIter) String() string {
+	return "error"
+}
+
+func (c *errorIter) SetBounds(lower, upper []byte) {}
+
+type errorKeyspanIter struct {
+	err error
+}
+
+// errorKeyspanIter implements the keyspan.FragmentIterator interface.
+var _ keyspan.FragmentIterator = (*errorKeyspanIter)(nil)
+
+func newErrorKeyspanIter(err error) *errorKeyspanIter {
+	return &errorKeyspanIter{err: err}
+}
+
+func (*errorKeyspanIter) SeekGE(key []byte) *keyspan.Span { return nil }
+func (*errorKeyspanIter) SeekLT(key []byte) *keyspan.Span { return nil }
+func (*errorKeyspanIter) First() *keyspan.Span            { return nil }
+func (*errorKeyspanIter) Last() *keyspan.Span             { return nil }
+func (*errorKeyspanIter) Next() *keyspan.Span             { return nil }
+func (*errorKeyspanIter) Prev() *keyspan.Span             { return nil }
+func (i *errorKeyspanIter) Error() error                  { return i.err }
+func (i *errorKeyspanIter) Close() error                  { return i.err }
+func (*errorKeyspanIter) String() string                  { return "error" }

vendor/github.com/cockroachdb/pebble/event.go

@@ -0,0 +1,766 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/redact"
+)
+
+// TableInfo exports the manifest.TableInfo type.
+type TableInfo = manifest.TableInfo
+
+func tablesTotalSize(tables []TableInfo) uint64 {
+	var size uint64
+	for i := range tables {
+		size += tables[i].Size
+	}
+	return size
+}
+
+func formatFileNums(tables []TableInfo) string {
+	var buf strings.Builder
+	for i := range tables {
+		if i > 0 {
+			buf.WriteString(" ")
+		}
+		buf.WriteString(tables[i].FileNum.String())
+	}
+	return buf.String()
+}
+
+// LevelInfo contains info pertaining to a particular level.
+type LevelInfo struct {
+	Level  int
+	Tables []TableInfo
+	Score  float64
+}
+
+func (i LevelInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("L%d [%s] (%s) Score=%.2f",
+		redact.Safe(i.Level),
+		redact.Safe(formatFileNums(i.Tables)),
+		redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))),
+		redact.Safe(i.Score))
+}
+
+// CompactionInfo contains the info for a compaction event.
+type CompactionInfo struct {
+	// JobID is the ID of the compaction job.
+	JobID int
+	// Reason is the reason for the compaction.
+	Reason string
+	// Input contains the input tables for the compaction organized by level.
+	Input []LevelInfo
+	// Output contains the output tables generated by the compaction. The output
+	// tables are empty for the compaction begin event.
+	Output LevelInfo
+	// Duration is the time spent compacting, including reading and writing
+	// sstables.
+	Duration time.Duration
+	// TotalDuration is the total wall-time duration of the compaction,
+	// including applying the compaction to the database. TotalDuration is
+	// always ≥ Duration.
+	TotalDuration time.Duration
+	Done          bool
+	Err           error
+
+	SingleLevelOverlappingRatio float64
+	MultiLevelOverlappingRatio  float64
+
+	// Annotations specifies additional info to appear in a compaction's event log line
+	Annotations compactionAnnotations
+}
+
+type compactionAnnotations []string
+
+// SafeFormat implements redact.SafeFormatter.
+func (ca compactionAnnotations) SafeFormat(w redact.SafePrinter, _ rune) {
+	if len(ca) == 0 {
+		return
+	}
+	for i := range ca {
+		if i != 0 {
+			w.Print(" ")
+		}
+		w.Printf("%s", redact.SafeString(ca[i]))
+	}
+}
+
+func (i CompactionInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] compaction(%s) to L%d error: %s",
+			redact.Safe(i.JobID), redact.SafeString(i.Reason), redact.Safe(i.Output.Level), i.Err)
+		return
+	}
+
+	if !i.Done {
+		w.Printf("[JOB %d] compacting(%s) ",
+			redact.Safe(i.JobID),
+			redact.SafeString(i.Reason))
+		w.Printf("%s", i.Annotations)
+		w.Printf("%s; ", levelInfos(i.Input))
+		w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio)
+		return
+	}
+	outputSize := tablesTotalSize(i.Output.Tables)
+	w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
+	w.Printf("%s", i.Annotations)
+	w.Print(levelInfos(i.Input))
+	w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
+		redact.Safe(i.Output.Level),
+		redact.Safe(formatFileNums(i.Output.Tables)),
+		redact.Safe(humanize.Bytes.Uint64(outputSize)),
+		redact.Safe(i.Duration.Seconds()),
+		redact.Safe(i.TotalDuration.Seconds()),
+		redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+}
+
+type levelInfos []LevelInfo
+
+func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) {
+	for j, levelInfo := range i {
+		if j > 0 {
+			w.Printf(" + ")
+		}
+		w.Print(levelInfo)
+	}
+}
+
+// DiskSlowInfo contains the info for a disk slowness event when writing to a
+// file.
+type DiskSlowInfo = vfs.DiskSlowInfo
+
+// FlushInfo contains the info for a flush event.
+type FlushInfo struct {
+	// JobID is the ID of the flush job.
+	JobID int
+	// Reason is the reason for the flush.
+	Reason string
+	// Input contains the count of input memtables that were flushed.
+	Input int
+	// InputBytes contains the total in-memory size of the memtable(s) that were
+	// flushed. This size includes skiplist indexing data structures.
+	InputBytes uint64
+	// Output contains the ouptut table generated by the flush. The output info
+	// is empty for the flush begin event.
+	Output []TableInfo
+	// Duration is the time spent flushing. This duration includes writing and
+	// syncing all of the flushed keys to sstables.
+	Duration time.Duration
+	// TotalDuration is the total wall-time duration of the flush, including
+	// applying the flush to the database. TotalDuration is always ≥ Duration.
+	TotalDuration time.Duration
+	// Ingest is set to true if the flush is handling tables that were added to
+	// the flushable queue via an ingestion operation.
+	Ingest bool
+	// IngestLevels are the output levels for each ingested table in the flush.
+	// This field is only populated when Ingest is true.
+	IngestLevels []int
+	Done         bool
+	Err          error
+}
+
+func (i FlushInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] flush error: %s", redact.Safe(i.JobID), i.Err)
+		return
+	}
+
+	plural := redact.SafeString("s")
+	if i.Input == 1 {
+		plural = ""
+	}
+	if !i.Done {
+		w.Printf("[JOB %d] ", redact.Safe(i.JobID))
+		if !i.Ingest {
+			w.Printf("flushing %d memtable", redact.Safe(i.Input))
+			w.SafeString(plural)
+			w.Printf(" (%s) to L0", redact.Safe(humanize.Bytes.Uint64(i.InputBytes)))
+		} else {
+			w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural)
+		}
+		return
+	}
+
+	outputSize := tablesTotalSize(i.Output)
+	if !i.Ingest {
+		if invariants.Enabled && len(i.IngestLevels) > 0 {
+			panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0"))
+		}
+		w.Printf("[JOB %d] flushed %d memtable%s (%s) to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
+			redact.Safe(i.JobID), redact.Safe(i.Input), plural,
+			redact.Safe(humanize.Bytes.Uint64(i.InputBytes)),
+			redact.Safe(formatFileNums(i.Output)),
+			redact.Safe(humanize.Bytes.Uint64(outputSize)),
+			redact.Safe(i.Duration.Seconds()),
+			redact.Safe(i.TotalDuration.Seconds()),
+			redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+	} else {
+		if invariants.Enabled && len(i.IngestLevels) == 0 {
+			panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0"))
+		}
+		w.Printf("[JOB %d] flushed %d ingested flushable%s",
+			redact.Safe(i.JobID), redact.Safe(len(i.Output)), plural)
+		for j, level := range i.IngestLevels {
+			file := i.Output[j]
+			if j > 0 {
+				w.Printf(" +")
+			}
+			w.Printf(" L%d:%s (%s)", level, file.FileNum, humanize.Bytes.Uint64(file.Size))
+		}
+		w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
+			redact.Safe(i.Duration.Seconds()),
+			redact.Safe(i.TotalDuration.Seconds()),
+			redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+	}
+}
+
+// ManifestCreateInfo contains info about a manifest creation event.
+type ManifestCreateInfo struct {
+	// JobID is the ID of the job the caused the manifest to be created.
+	JobID int
+	Path  string
+	// The file number of the new Manifest.
+	FileNum FileNum
+	Err     error
+}
+
+func (i ManifestCreateInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
+		return
+	}
+	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), i.FileNum)
+}
+
+// ManifestDeleteInfo contains the info for a Manifest deletion event.
+type ManifestDeleteInfo struct {
+	// JobID is the ID of the job the caused the Manifest to be deleted.
+	JobID   int
+	Path    string
+	FileNum FileNum
+	Err     error
+}
+
+func (i ManifestDeleteInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
+		return
+	}
+	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), i.FileNum)
+}
+
+// TableCreateInfo contains the info for a table creation event.
+type TableCreateInfo struct {
+	JobID int
+	// Reason is the reason for the table creation: "compacting", "flushing", or
+	// "ingesting".
+	Reason  string
+	Path    string
+	FileNum FileNum
+}
+
+func (i TableCreateInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("[JOB %d] %s: sstable created %s",
+		redact.Safe(i.JobID), redact.Safe(i.Reason), i.FileNum)
+}
+
+// TableDeleteInfo contains the info for a table deletion event.
+type TableDeleteInfo struct {
+	JobID   int
+	Path    string
+	FileNum FileNum
+	Err     error
+}
+
+func (i TableDeleteInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] sstable delete error %s: %s",
+			redact.Safe(i.JobID), i.FileNum, i.Err)
+		return
+	}
+	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), i.FileNum)
+}
+
+// TableIngestInfo contains the info for a table ingestion event.
+type TableIngestInfo struct {
+	// JobID is the ID of the job the caused the table to be ingested.
+	JobID  int
+	Tables []struct {
+		TableInfo
+		Level int
+	}
+	// GlobalSeqNum is the sequence number that was assigned to all entries in
+	// the ingested table.
+	GlobalSeqNum uint64
+	// flushable indicates whether the ingested sstable was treated as a
+	// flushable.
+	flushable bool
+	Err       error
+}
+
+func (i TableIngestInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] ingest error: %s", redact.Safe(i.JobID), i.Err)
+		return
+	}
+
+	if i.flushable {
+		w.Printf("[JOB %d] ingested as flushable", redact.Safe(i.JobID))
+	} else {
+		w.Printf("[JOB %d] ingested", redact.Safe(i.JobID))
+	}
+
+	for j := range i.Tables {
+		t := &i.Tables[j]
+		if j > 0 {
+			w.Printf(",")
+		}
+		levelStr := ""
+		if !i.flushable {
+			levelStr = fmt.Sprintf("L%d:", t.Level)
+		}
+		w.Printf(" %s%s (%s)", redact.Safe(levelStr), t.FileNum,
+			redact.Safe(humanize.Bytes.Uint64(t.Size)))
+	}
+}
+
+// TableStatsInfo contains the info for a table stats loaded event.
+type TableStatsInfo struct {
+	// JobID is the ID of the job that finished loading the initial tables'
+	// stats.
+	JobID int
+}
+
+func (i TableStatsInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i TableStatsInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("[JOB %d] all initial table stats loaded", redact.Safe(i.JobID))
+}
+
+// TableValidatedInfo contains information on the result of a validation run
+// on an sstable.
+type TableValidatedInfo struct {
+	JobID int
+	Meta  *fileMetadata
+}
+
+func (i TableValidatedInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i TableValidatedInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("[JOB %d] validated table: %s", redact.Safe(i.JobID), i.Meta)
+}
+
+// WALCreateInfo contains info about a WAL creation event.
+type WALCreateInfo struct {
+	// JobID is the ID of the job the caused the WAL to be created.
+	JobID int
+	Path  string
+	// The file number of the new WAL.
+	FileNum FileNum
+	// The file number of a previous WAL which was recycled to create this
+	// one. Zero if recycling did not take place.
+	RecycledFileNum FileNum
+	Err             error
+}
+
+func (i WALCreateInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] WAL create error: %s", redact.Safe(i.JobID), i.Err)
+		return
+	}
+
+	if i.RecycledFileNum == 0 {
+		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), i.FileNum)
+		return
+	}
+
+	w.Printf("[JOB %d] WAL created %s (recycled %s)",
+		redact.Safe(i.JobID), i.FileNum, i.RecycledFileNum)
+}
+
+// WALDeleteInfo contains the info for a WAL deletion event.
+type WALDeleteInfo struct {
+	// JobID is the ID of the job the caused the WAL to be deleted.
+	JobID   int
+	Path    string
+	FileNum FileNum
+	Err     error
+}
+
+func (i WALDeleteInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	if i.Err != nil {
+		w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
+		return
+	}
+	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), i.FileNum)
+}
+
+// WriteStallBeginInfo contains the info for a write stall begin event.
+type WriteStallBeginInfo struct {
+	Reason string
+}
+
+func (i WriteStallBeginInfo) String() string {
+	return redact.StringWithoutMarkers(i)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (i WriteStallBeginInfo) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("write stall beginning: %s", redact.Safe(i.Reason))
+}
+
+// EventListener contains a set of functions that will be invoked when various
+// significant DB events occur. Note that the functions should not run for an
+// excessive amount of time as they are invoked synchronously by the DB and may
+// block continued DB work. For a similar reason it is advisable to not perform
+// any synchronous calls back into the DB.
+type EventListener struct {
+	// BackgroundError is invoked whenever an error occurs during a background
+	// operation such as flush or compaction.
+	BackgroundError func(error)
+
+	// CompactionBegin is invoked after the inputs to a compaction have been
+	// determined, but before the compaction has produced any output.
+	CompactionBegin func(CompactionInfo)
+
+	// CompactionEnd is invoked after a compaction has completed and the result
+	// has been installed.
+	CompactionEnd func(CompactionInfo)
+
+	// DiskSlow is invoked after a disk write operation on a file created with a
+	// disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks) is
+	// observed to exceed the specified disk slowness threshold duration. DiskSlow
+	// is called on a goroutine that is monitoring slowness/stuckness. The callee
+	// MUST return without doing any IO, or blocking on anything (like a mutex)
+	// that is waiting on IO. This is imperative in order to reliably monitor for
+	// slowness, since if this goroutine gets stuck, the monitoring will stop
+	// working.
+	DiskSlow func(DiskSlowInfo)
+
+	// FlushBegin is invoked after the inputs to a flush have been determined,
+	// but before the flush has produced any output.
+	FlushBegin func(FlushInfo)
+
+	// FlushEnd is invoked after a flush has complated and the result has been
+	// installed.
+	FlushEnd func(FlushInfo)
+
+	// FormatUpgrade is invoked after the database's FormatMajorVersion
+	// is upgraded.
+	FormatUpgrade func(FormatMajorVersion)
+
+	// ManifestCreated is invoked after a manifest has been created.
+	ManifestCreated func(ManifestCreateInfo)
+
+	// ManifestDeleted is invoked after a manifest has been deleted.
+	ManifestDeleted func(ManifestDeleteInfo)
+
+	// TableCreated is invoked when a table has been created.
+	TableCreated func(TableCreateInfo)
+
+	// TableDeleted is invoked after a table has been deleted.
+	TableDeleted func(TableDeleteInfo)
+
+	// TableIngested is invoked after an externally created table has been
+	// ingested via a call to DB.Ingest().
+	TableIngested func(TableIngestInfo)
+
+	// TableStatsLoaded is invoked at most once, when the table stats
+	// collector has loaded statistics for all tables that existed at Open.
+	TableStatsLoaded func(TableStatsInfo)
+
+	// TableValidated is invoked after validation runs on an sstable.
+	TableValidated func(TableValidatedInfo)
+
+	// WALCreated is invoked after a WAL has been created.
+	WALCreated func(WALCreateInfo)
+
+	// WALDeleted is invoked after a WAL has been deleted.
+	WALDeleted func(WALDeleteInfo)
+
+	// WriteStallBegin is invoked when writes are intentionally delayed.
+	WriteStallBegin func(WriteStallBeginInfo)
+
+	// WriteStallEnd is invoked when delayed writes are released.
+	WriteStallEnd func()
+}
+
+// EnsureDefaults ensures that background error events are logged to the
+// specified logger if a handler for those events hasn't been otherwise
+// specified. Ensure all handlers are non-nil so that we don't have to check
+// for nil-ness before invoking.
+func (l *EventListener) EnsureDefaults(logger Logger) {
+	if l.BackgroundError == nil {
+		if logger != nil {
+			l.BackgroundError = func(err error) {
+				logger.Infof("background error: %s", err)
+			}
+		} else {
+			l.BackgroundError = func(error) {}
+		}
+	}
+	if l.CompactionBegin == nil {
+		l.CompactionBegin = func(info CompactionInfo) {}
+	}
+	if l.CompactionEnd == nil {
+		l.CompactionEnd = func(info CompactionInfo) {}
+	}
+	if l.DiskSlow == nil {
+		l.DiskSlow = func(info DiskSlowInfo) {}
+	}
+	if l.FlushBegin == nil {
+		l.FlushBegin = func(info FlushInfo) {}
+	}
+	if l.FlushEnd == nil {
+		l.FlushEnd = func(info FlushInfo) {}
+	}
+	if l.FormatUpgrade == nil {
+		l.FormatUpgrade = func(v FormatMajorVersion) {}
+	}
+	if l.ManifestCreated == nil {
+		l.ManifestCreated = func(info ManifestCreateInfo) {}
+	}
+	if l.ManifestDeleted == nil {
+		l.ManifestDeleted = func(info ManifestDeleteInfo) {}
+	}
+	if l.TableCreated == nil {
+		l.TableCreated = func(info TableCreateInfo) {}
+	}
+	if l.TableDeleted == nil {
+		l.TableDeleted = func(info TableDeleteInfo) {}
+	}
+	if l.TableIngested == nil {
+		l.TableIngested = func(info TableIngestInfo) {}
+	}
+	if l.TableStatsLoaded == nil {
+		l.TableStatsLoaded = func(info TableStatsInfo) {}
+	}
+	if l.TableValidated == nil {
+		l.TableValidated = func(validated TableValidatedInfo) {}
+	}
+	if l.WALCreated == nil {
+		l.WALCreated = func(info WALCreateInfo) {}
+	}
+	if l.WALDeleted == nil {
+		l.WALDeleted = func(info WALDeleteInfo) {}
+	}
+	if l.WriteStallBegin == nil {
+		l.WriteStallBegin = func(info WriteStallBeginInfo) {}
+	}
+	if l.WriteStallEnd == nil {
+		l.WriteStallEnd = func() {}
+	}
+}
+
+// MakeLoggingEventListener creates an EventListener that logs all events to the
+// specified logger.
+func MakeLoggingEventListener(logger Logger) EventListener {
+	if logger == nil {
+		logger = DefaultLogger
+	}
+
+	return EventListener{
+		BackgroundError: func(err error) {
+			logger.Infof("background error: %s", err)
+		},
+		CompactionBegin: func(info CompactionInfo) {
+			logger.Infof("%s", info)
+		},
+		CompactionEnd: func(info CompactionInfo) {
+			logger.Infof("%s", info)
+		},
+		DiskSlow: func(info DiskSlowInfo) {
+			logger.Infof("%s", info)
+		},
+		FlushBegin: func(info FlushInfo) {
+			logger.Infof("%s", info)
+		},
+		FlushEnd: func(info FlushInfo) {
+			logger.Infof("%s", info)
+		},
+		FormatUpgrade: func(v FormatMajorVersion) {
+			logger.Infof("upgraded to format version: %s", v)
+		},
+		ManifestCreated: func(info ManifestCreateInfo) {
+			logger.Infof("%s", info)
+		},
+		ManifestDeleted: func(info ManifestDeleteInfo) {
+			logger.Infof("%s", info)
+		},
+		TableCreated: func(info TableCreateInfo) {
+			logger.Infof("%s", info)
+		},
+		TableDeleted: func(info TableDeleteInfo) {
+			logger.Infof("%s", info)
+		},
+		TableIngested: func(info TableIngestInfo) {
+			logger.Infof("%s", info)
+		},
+		TableStatsLoaded: func(info TableStatsInfo) {
+			logger.Infof("%s", info)
+		},
+		TableValidated: func(info TableValidatedInfo) {
+			logger.Infof("%s", info)
+		},
+		WALCreated: func(info WALCreateInfo) {
+			logger.Infof("%s", info)
+		},
+		WALDeleted: func(info WALDeleteInfo) {
+			logger.Infof("%s", info)
+		},
+		WriteStallBegin: func(info WriteStallBeginInfo) {
+			logger.Infof("%s", info)
+		},
+		WriteStallEnd: func() {
+			logger.Infof("write stall ending")
+		},
+	}
+}
+
+// TeeEventListener wraps two EventListeners, forwarding all events to both.
+func TeeEventListener(a, b EventListener) EventListener {
+	a.EnsureDefaults(nil)
+	b.EnsureDefaults(nil)
+	return EventListener{
+		BackgroundError: func(err error) {
+			a.BackgroundError(err)
+			b.BackgroundError(err)
+		},
+		CompactionBegin: func(info CompactionInfo) {
+			a.CompactionBegin(info)
+			b.CompactionBegin(info)
+		},
+		CompactionEnd: func(info CompactionInfo) {
+			a.CompactionEnd(info)
+			b.CompactionEnd(info)
+		},
+		DiskSlow: func(info DiskSlowInfo) {
+			a.DiskSlow(info)
+			b.DiskSlow(info)
+		},
+		FlushBegin: func(info FlushInfo) {
+			a.FlushBegin(info)
+			b.FlushBegin(info)
+		},
+		FlushEnd: func(info FlushInfo) {
+			a.FlushEnd(info)
+			b.FlushEnd(info)
+		},
+		FormatUpgrade: func(v FormatMajorVersion) {
+			a.FormatUpgrade(v)
+			b.FormatUpgrade(v)
+		},
+		ManifestCreated: func(info ManifestCreateInfo) {
+			a.ManifestCreated(info)
+			b.ManifestCreated(info)
+		},
+		ManifestDeleted: func(info ManifestDeleteInfo) {
+			a.ManifestDeleted(info)
+			b.ManifestDeleted(info)
+		},
+		TableCreated: func(info TableCreateInfo) {
+			a.TableCreated(info)
+			b.TableCreated(info)
+		},
+		TableDeleted: func(info TableDeleteInfo) {
+			a.TableDeleted(info)
+			b.TableDeleted(info)
+		},
+		TableIngested: func(info TableIngestInfo) {
+			a.TableIngested(info)
+			b.TableIngested(info)
+		},
+		TableStatsLoaded: func(info TableStatsInfo) {
+			a.TableStatsLoaded(info)
+			b.TableStatsLoaded(info)
+		},
+		TableValidated: func(info TableValidatedInfo) {
+			a.TableValidated(info)
+			b.TableValidated(info)
+		},
+		WALCreated: func(info WALCreateInfo) {
+			a.WALCreated(info)
+			b.WALCreated(info)
+		},
+		WALDeleted: func(info WALDeleteInfo) {
+			a.WALDeleted(info)
+			b.WALDeleted(info)
+		},
+		WriteStallBegin: func(info WriteStallBeginInfo) {
+			a.WriteStallBegin(info)
+			b.WriteStallBegin(info)
+		},
+		WriteStallEnd: func() {
+			a.WriteStallEnd()
+			b.WriteStallEnd()
+		},
+	}
+}

vendor/github.com/cockroachdb/pebble/external_iterator.go

@@ -0,0 +1,555 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/sstable"
+)
+
+// ExternalIterOption provide an interface to specify open-time options to
+// NewExternalIter.
+type ExternalIterOption interface {
+	// iterApply is called on the iterator during opening in order to set internal
+	// parameters.
+	iterApply(*Iterator)
+	// readerOptions returns any reader options added by this iter option.
+	readerOptions() []sstable.ReaderOption
+}
+
+type externalIterReaderOptions struct {
+	opts []sstable.ReaderOption
+}
+
+func (e *externalIterReaderOptions) iterApply(iterator *Iterator) {
+	// Do nothing.
+}
+
+func (e *externalIterReaderOptions) readerOptions() []sstable.ReaderOption {
+	return e.opts
+}
+
+// ExternalIterReaderOptions returns an ExternalIterOption that specifies
+// sstable.ReaderOptions to be applied on sstable readers in NewExternalIter.
+func ExternalIterReaderOptions(opts ...sstable.ReaderOption) ExternalIterOption {
+	return &externalIterReaderOptions{opts: opts}
+}
+
+// ExternalIterForwardOnly is an ExternalIterOption that specifies this iterator
+// will only be used for forward positioning operations (First, SeekGE, Next).
+// This could enable optimizations that take advantage of this invariant.
+// Behaviour when a reverse positioning operation is done on an iterator
+// opened with this option is unpredictable, though in most cases it should.
+type ExternalIterForwardOnly struct{}
+
+func (e ExternalIterForwardOnly) iterApply(iter *Iterator) {
+	iter.forwardOnly = true
+}
+
+func (e ExternalIterForwardOnly) readerOptions() []sstable.ReaderOption {
+	return nil
+}
+
+// NewExternalIter takes an input 2d array of sstable files which may overlap
+// across subarrays but not within a subarray (at least as far as points are
+// concerned; range keys are allowed to overlap arbitrarily even within a
+// subarray), and returns an Iterator over the merged contents of the sstables.
+// Input sstables may contain point keys, range keys, range deletions, etc. The
+// input files slice must be sorted in reverse chronological ordering. A key in a
+// file at a lower index subarray will shadow a key with an identical user key
+// contained within a file at a higher index subarray. Each subarray must be
+// sorted in internal key order, where lower index files contain keys that sort
+// left of files with higher indexes.
+//
+// Input sstables must only contain keys with the zero sequence number.
+//
+// Iterators constructed through NewExternalIter do not support all iterator
+// options, including block-property and table filters. NewExternalIter errors
+// if an incompatible option is set.
+func NewExternalIter(
+	o *Options,
+	iterOpts *IterOptions,
+	files [][]sstable.ReadableFile,
+	extraOpts ...ExternalIterOption,
+) (it *Iterator, err error) {
+	return NewExternalIterWithContext(context.Background(), o, iterOpts, files, extraOpts...)
+}
+
+// NewExternalIterWithContext is like NewExternalIter, and additionally
+// accepts a context for tracing.
+func NewExternalIterWithContext(
+	ctx context.Context,
+	o *Options,
+	iterOpts *IterOptions,
+	files [][]sstable.ReadableFile,
+	extraOpts ...ExternalIterOption,
+) (it *Iterator, err error) {
+	if iterOpts != nil {
+		if err := validateExternalIterOpts(iterOpts); err != nil {
+			return nil, err
+		}
+	}
+
+	var readers [][]*sstable.Reader
+
+	// Ensure we close all the opened readers if we error out.
+	defer func() {
+		if err != nil {
+			for i := range readers {
+				for j := range readers[i] {
+					_ = readers[i][j].Close()
+				}
+			}
+		}
+	}()
+	seqNumOffset := 0
+	var extraReaderOpts []sstable.ReaderOption
+	for i := range extraOpts {
+		extraReaderOpts = append(extraReaderOpts, extraOpts[i].readerOptions()...)
+	}
+	for _, levelFiles := range files {
+		seqNumOffset += len(levelFiles)
+	}
+	for _, levelFiles := range files {
+		var subReaders []*sstable.Reader
+		seqNumOffset -= len(levelFiles)
+		subReaders, err = openExternalTables(o, levelFiles, seqNumOffset, o.MakeReaderOptions(), extraReaderOpts...)
+		readers = append(readers, subReaders)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	buf := iterAllocPool.Get().(*iterAlloc)
+	dbi := &buf.dbi
+	*dbi = Iterator{
+		ctx:                 ctx,
+		alloc:               buf,
+		merge:               o.Merger.Merge,
+		comparer:            *o.Comparer,
+		readState:           nil,
+		keyBuf:              buf.keyBuf,
+		prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
+		boundsBuf:           buf.boundsBuf,
+		batch:               nil,
+		// Add the readers to the Iterator so that Close closes them, and
+		// SetOptions can re-construct iterators from them.
+		externalReaders: readers,
+		newIters: func(
+			ctx context.Context, f *manifest.FileMetadata, opts *IterOptions,
+			internalOpts internalIterOpts) (internalIterator, keyspan.FragmentIterator, error) {
+			// NB: External iterators are currently constructed without any
+			// `levelIters`. newIters should never be called. When we support
+			// organizing multiple non-overlapping files into a single level
+			// (see TODO below), we'll need to adjust this tableNewIters
+			// implementation to open iterators by looking up f in a map
+			// of readers indexed by *fileMetadata.
+			panic("unreachable")
+		},
+		seqNum: base.InternalKeySeqNumMax,
+	}
+	if iterOpts != nil {
+		dbi.opts = *iterOpts
+		dbi.processBounds(iterOpts.LowerBound, iterOpts.UpperBound)
+	}
+	for i := range extraOpts {
+		extraOpts[i].iterApply(dbi)
+	}
+	finishInitializingExternal(ctx, dbi)
+	return dbi, nil
+}
+
+func validateExternalIterOpts(iterOpts *IterOptions) error {
+	switch {
+	case iterOpts.TableFilter != nil:
+		return errors.Errorf("pebble: external iterator: TableFilter unsupported")
+	case iterOpts.PointKeyFilters != nil:
+		return errors.Errorf("pebble: external iterator: PointKeyFilters unsupported")
+	case iterOpts.RangeKeyFilters != nil:
+		return errors.Errorf("pebble: external iterator: RangeKeyFilters unsupported")
+	case iterOpts.OnlyReadGuaranteedDurable:
+		return errors.Errorf("pebble: external iterator: OnlyReadGuaranteedDurable unsupported")
+	case iterOpts.UseL6Filters:
+		return errors.Errorf("pebble: external iterator: UseL6Filters unsupported")
+	}
+	return nil
+}
+
+func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterator, error) {
+	// TODO(jackson): In some instances we could generate fewer levels by using
+	// L0Sublevels code to organize nonoverlapping files into the same level.
+	// This would allow us to use levelIters and keep a smaller set of data and
+	// files in-memory. However, it would also require us to identify the bounds
+	// of all the files upfront.
+
+	if !it.opts.pointKeys() {
+		return emptyIter, nil
+	} else if it.pointIter != nil {
+		return it.pointIter, nil
+	}
+	mlevels := it.alloc.mlevels[:0]
+
+	if len(it.externalReaders) > cap(mlevels) {
+		mlevels = make([]mergingIterLevel, 0, len(it.externalReaders))
+	}
+	for _, readers := range it.externalReaders {
+		var combinedIters []internalIterator
+		for _, r := range readers {
+			var (
+				rangeDelIter keyspan.FragmentIterator
+				pointIter    internalIterator
+				err          error
+			)
+			// We could set hideObsoletePoints=true, since we are reading at
+			// InternalKeySeqNumMax, but we don't bother since these sstables should
+			// not have obsolete points (so the performance optimization is
+			// unnecessary), and we don't want to bother constructing a
+			// BlockPropertiesFilterer that includes obsoleteKeyBlockPropertyFilter.
+			pointIter, err = r.NewIterWithBlockPropertyFiltersAndContextEtc(
+				ctx, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */
+				false /* hideObsoletePoints */, false, /* useFilterBlock */
+				&it.stats.InternalStats, sstable.TrivialReaderProvider{Reader: r})
+			if err != nil {
+				return nil, err
+			}
+			rangeDelIter, err = r.NewRawRangeDelIter()
+			if err != nil {
+				return nil, err
+			}
+			if rangeDelIter == nil && pointIter != nil && it.forwardOnly {
+				// TODO(bilal): Consider implementing range key pausing in
+				// simpleLevelIter so we can reduce mergingIterLevels even more by
+				// sending all sstable iterators to combinedIters, not just those
+				// corresponding to sstables without range deletes.
+				combinedIters = append(combinedIters, pointIter)
+				continue
+			}
+			mlevels = append(mlevels, mergingIterLevel{
+				iter:         pointIter,
+				rangeDelIter: rangeDelIter,
+			})
+		}
+		if len(combinedIters) == 1 {
+			mlevels = append(mlevels, mergingIterLevel{
+				iter: combinedIters[0],
+			})
+		} else if len(combinedIters) > 1 {
+			sli := &simpleLevelIter{
+				cmp:   it.cmp,
+				iters: combinedIters,
+			}
+			sli.init(it.opts)
+			mlevels = append(mlevels, mergingIterLevel{
+				iter:         sli,
+				rangeDelIter: nil,
+			})
+		}
+	}
+	if len(mlevels) == 1 && mlevels[0].rangeDelIter == nil {
+		// Set closePointIterOnce to true. This is because we're bypassing the
+		// merging iter, which turns Close()s on it idempotent for any child
+		// iterators. The outer Iterator could call Close() on a point iter twice,
+		// which sstable iterators do not support (as they release themselves to
+		// a pool).
+		it.closePointIterOnce = true
+		return mlevels[0].iter, nil
+	}
+
+	it.alloc.merging.init(&it.opts, &it.stats.InternalStats, it.comparer.Compare, it.comparer.Split, mlevels...)
+	it.alloc.merging.snapshot = base.InternalKeySeqNumMax
+	if len(mlevels) <= cap(it.alloc.levelsPositioned) {
+		it.alloc.merging.levelsPositioned = it.alloc.levelsPositioned[:len(mlevels)]
+	}
+	return &it.alloc.merging, nil
+}
+
+func finishInitializingExternal(ctx context.Context, it *Iterator) {
+	pointIter, err := createExternalPointIter(ctx, it)
+	if err != nil {
+		it.pointIter = &errorIter{err: err}
+	} else {
+		it.pointIter = pointIter
+	}
+	it.iter = it.pointIter
+
+	if it.opts.rangeKeys() {
+		it.rangeKeyMasking.init(it, it.comparer.Compare, it.comparer.Split)
+		var rangeKeyIters []keyspan.FragmentIterator
+		if it.rangeKey == nil {
+			// We could take advantage of the lack of overlaps in range keys within
+			// each slice in it.externalReaders, and generate keyspan.LevelIters
+			// out of those. However, since range keys are expected to be sparse to
+			// begin with, the performance gain might not be significant enough to
+			// warrant it.
+			//
+			// TODO(bilal): Explore adding a simpleRangeKeyLevelIter that does not
+			// operate on FileMetadatas (similar to simpleLevelIter), and implements
+			// this optimization.
+			for _, readers := range it.externalReaders {
+				for _, r := range readers {
+					if rki, err := r.NewRawRangeKeyIter(); err != nil {
+						rangeKeyIters = append(rangeKeyIters, &errorKeyspanIter{err: err})
+					} else if rki != nil {
+						rangeKeyIters = append(rangeKeyIters, rki)
+					}
+				}
+			}
+			if len(rangeKeyIters) > 0 {
+				it.rangeKey = iterRangeKeyStateAllocPool.Get().(*iteratorRangeKeyState)
+				it.rangeKey.init(it.comparer.Compare, it.comparer.Split, &it.opts)
+				it.rangeKey.rangeKeyIter = it.rangeKey.iterConfig.Init(
+					&it.comparer,
+					base.InternalKeySeqNumMax,
+					it.opts.LowerBound, it.opts.UpperBound,
+					&it.hasPrefix, &it.prefixOrFullSeekKey,
+					false /* internalKeys */, &it.rangeKey.internal,
+				)
+				for i := range rangeKeyIters {
+					it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])
+				}
+			}
+		}
+		if it.rangeKey != nil {
+			it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter,
+				keyspan.InterleavingIterOpts{
+					Mask:       &it.rangeKeyMasking,
+					LowerBound: it.opts.LowerBound,
+					UpperBound: it.opts.UpperBound,
+				})
+			it.iter = &it.rangeKey.iiter
+		}
+	}
+}
+
+func openExternalTables(
+	o *Options,
+	files []sstable.ReadableFile,
+	seqNumOffset int,
+	readerOpts sstable.ReaderOptions,
+	extraReaderOpts ...sstable.ReaderOption,
+) (readers []*sstable.Reader, err error) {
+	readers = make([]*sstable.Reader, 0, len(files))
+	for i := range files {
+		readable, err := sstable.NewSimpleReadable(files[i])
+		if err != nil {
+			return readers, err
+		}
+		r, err := sstable.NewReader(readable, readerOpts, extraReaderOpts...)
+		if err != nil {
+			return readers, err
+		}
+		// Use the index of the file in files as the sequence number for all of
+		// its keys.
+		r.Properties.GlobalSeqNum = uint64(len(files) - i + seqNumOffset)
+		readers = append(readers, r)
+	}
+	return readers, err
+}
+
+// simpleLevelIter is similar to a levelIter in that it merges the points
+// from multiple point iterators that are non-overlapping in the key ranges
+// they return. It is only expected to support forward iteration and forward
+// regular seeking; reverse iteration and prefix seeking is not supported.
+// Intended to be a low-overhead, non-FileMetadata dependent option for
+// NewExternalIter. To optimize seeking and forward iteration, it maintains
+// two slices of child iterators; one of all iterators, and a subset of it that
+// contains just the iterators that contain point keys within the current
+// bounds.
+//
+// Note that this levelIter does not support pausing at file boundaries
+// in case of range tombstones in this file that could apply to points outside
+// of this file (and outside of this level). This is sufficient for optimizing
+// the main use cases of NewExternalIter, however for completeness it would make
+// sense to build this pausing functionality in.
+type simpleLevelIter struct {
+	cmp          Compare
+	err          error
+	lowerBound   []byte
+	iters        []internalIterator
+	filtered     []internalIterator
+	firstKeys    [][]byte
+	firstKeysBuf []byte
+	currentIdx   int
+}
+
+var _ internalIterator = &simpleLevelIter{}
+
+// init initializes this simpleLevelIter.
+func (s *simpleLevelIter) init(opts IterOptions) {
+	s.currentIdx = 0
+	s.lowerBound = opts.LowerBound
+	s.resetFilteredIters()
+}
+
+func (s *simpleLevelIter) resetFilteredIters() {
+	s.filtered = s.filtered[:0]
+	s.firstKeys = s.firstKeys[:0]
+	s.firstKeysBuf = s.firstKeysBuf[:0]
+	s.err = nil
+	for i := range s.iters {
+		var iterKey *base.InternalKey
+		if s.lowerBound != nil {
+			iterKey, _ = s.iters[i].SeekGE(s.lowerBound, base.SeekGEFlagsNone)
+		} else {
+			iterKey, _ = s.iters[i].First()
+		}
+		if iterKey != nil {
+			s.filtered = append(s.filtered, s.iters[i])
+			bufStart := len(s.firstKeysBuf)
+			s.firstKeysBuf = append(s.firstKeysBuf, iterKey.UserKey...)
+			s.firstKeys = append(s.firstKeys, s.firstKeysBuf[bufStart:bufStart+len(iterKey.UserKey)])
+		} else if err := s.iters[i].Error(); err != nil {
+			s.err = err
+		}
+	}
+}
+
+func (s *simpleLevelIter) SeekGE(
+	key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	if s.err != nil {
+		return nil, base.LazyValue{}
+	}
+	// Find the first file that is entirely >= key. The file before that could
+	// contain the key we're looking for.
+	n := sort.Search(len(s.firstKeys), func(i int) bool {
+		return s.cmp(key, s.firstKeys[i]) <= 0
+	})
+	if n > 0 {
+		s.currentIdx = n - 1
+	} else {
+		s.currentIdx = n
+	}
+	if s.currentIdx < len(s.filtered) {
+		if iterKey, val := s.filtered[s.currentIdx].SeekGE(key, flags); iterKey != nil {
+			return iterKey, val
+		}
+		if err := s.filtered[s.currentIdx].Error(); err != nil {
+			s.err = err
+		}
+		s.currentIdx++
+	}
+	return s.skipEmptyFileForward(key, flags)
+}
+
+func (s *simpleLevelIter) skipEmptyFileForward(
+	seekKey []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	var iterKey *base.InternalKey
+	var val base.LazyValue
+	for s.currentIdx >= 0 && s.currentIdx < len(s.filtered) && s.err == nil {
+		if seekKey != nil {
+			iterKey, val = s.filtered[s.currentIdx].SeekGE(seekKey, flags)
+		} else if s.lowerBound != nil {
+			iterKey, val = s.filtered[s.currentIdx].SeekGE(s.lowerBound, flags)
+		} else {
+			iterKey, val = s.filtered[s.currentIdx].First()
+		}
+		if iterKey != nil {
+			return iterKey, val
+		}
+		if err := s.filtered[s.currentIdx].Error(); err != nil {
+			s.err = err
+		}
+		s.currentIdx++
+	}
+	return nil, base.LazyValue{}
+}
+
+func (s *simpleLevelIter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+func (s *simpleLevelIter) SeekLT(
+	key []byte, flags base.SeekLTFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+func (s *simpleLevelIter) First() (*base.InternalKey, base.LazyValue) {
+	if s.err != nil {
+		return nil, base.LazyValue{}
+	}
+	s.currentIdx = 0
+	return s.skipEmptyFileForward(nil /* seekKey */, base.SeekGEFlagsNone)
+}
+
+func (s *simpleLevelIter) Last() (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+func (s *simpleLevelIter) Next() (*base.InternalKey, base.LazyValue) {
+	if s.err != nil {
+		return nil, base.LazyValue{}
+	}
+	if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) {
+		return nil, base.LazyValue{}
+	}
+	if iterKey, val := s.filtered[s.currentIdx].Next(); iterKey != nil {
+		return iterKey, val
+	}
+	s.currentIdx++
+	return s.skipEmptyFileForward(nil /* seekKey */, base.SeekGEFlagsNone)
+}
+
+func (s *simpleLevelIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	if s.err != nil {
+		return nil, base.LazyValue{}
+	}
+	if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) {
+		return nil, base.LazyValue{}
+	}
+	if iterKey, val := s.filtered[s.currentIdx].NextPrefix(succKey); iterKey != nil {
+		return iterKey, val
+	}
+	s.currentIdx++
+	return s.skipEmptyFileForward(succKey /* seekKey */, base.SeekGEFlagsNone)
+}
+
+func (s *simpleLevelIter) Prev() (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+func (s *simpleLevelIter) Error() error {
+	if s.currentIdx >= 0 && s.currentIdx < len(s.filtered) {
+		s.err = firstError(s.err, s.filtered[s.currentIdx].Error())
+	}
+	return s.err
+}
+
+func (s *simpleLevelIter) Close() error {
+	var err error
+	for i := range s.iters {
+		err = firstError(err, s.iters[i].Close())
+	}
+	return err
+}
+
+func (s *simpleLevelIter) SetBounds(lower, upper []byte) {
+	s.currentIdx = -1
+	s.lowerBound = lower
+	for i := range s.iters {
+		s.iters[i].SetBounds(lower, upper)
+	}
+	s.resetFilteredIters()
+}
+
+func (s *simpleLevelIter) String() string {
+	if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) {
+		return "simpleLevelIter: current=<nil>"
+	}
+	return fmt.Sprintf("simpleLevelIter: current=%s", s.filtered[s.currentIdx])
+}
+
+var _ internalIterator = &simpleLevelIter{}

vendor/github.com/cockroachdb/pebble/filenames.go

@@ -0,0 +1,54 @@
+// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+type fileType = base.FileType
+
+// FileNum is an identifier for a file within a database.
+type FileNum = base.FileNum
+
+const (
+	fileTypeLog      = base.FileTypeLog
+	fileTypeLock     = base.FileTypeLock
+	fileTypeTable    = base.FileTypeTable
+	fileTypeManifest = base.FileTypeManifest
+	fileTypeCurrent  = base.FileTypeCurrent
+	fileTypeOptions  = base.FileTypeOptions
+	fileTypeTemp     = base.FileTypeTemp
+	fileTypeOldTemp  = base.FileTypeOldTemp
+)
+
+// setCurrentFile sets the CURRENT file to point to the manifest with
+// provided file number.
+//
+// NB: This is a low-level routine and typically not what you want to
+// use. Newer versions of Pebble running newer format major versions do
+// not use the CURRENT file. See setCurrentFunc in version_set.go.
+func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error {
+	newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum)
+	oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum)
+	fs.Remove(oldFilename)
+	f, err := fs.Create(oldFilename)
+	if err != nil {
+		return err
+	}
+	if _, err := fmt.Fprintf(f, "MANIFEST-%s\n", fileNum); err != nil {
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return fs.Rename(oldFilename, newFilename)
+}

vendor/github.com/cockroachdb/pebble/flushable.go

@@ -0,0 +1,249 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manifest"
+)
+
+// flushable defines the interface for immutable memtables.
+type flushable interface {
+	newIter(o *IterOptions) internalIterator
+	newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator
+	newRangeDelIter(o *IterOptions) keyspan.FragmentIterator
+	newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator
+	containsRangeKeys() bool
+	// inuseBytes returns the number of inuse bytes by the flushable.
+	inuseBytes() uint64
+	// totalBytes returns the total number of bytes allocated by the flushable.
+	totalBytes() uint64
+	// readyForFlush returns true when the flushable is ready for flushing. See
+	// memTable.readyForFlush for one implementation which needs to check whether
+	// there are any outstanding write references.
+	readyForFlush() bool
+}
+
+// flushableEntry wraps a flushable and adds additional metadata and
+// functionality that is common to all flushables.
+type flushableEntry struct {
+	flushable
+	// Channel which is closed when the flushable has been flushed.
+	flushed chan struct{}
+	// flushForced indicates whether a flush was forced on this memtable (either
+	// manual, or due to ingestion). Protected by DB.mu.
+	flushForced bool
+	// delayedFlushForcedAt indicates whether a timer has been set to force a
+	// flush on this memtable at some point in the future. Protected by DB.mu.
+	// Holds the timestamp of when the flush will be issued.
+	delayedFlushForcedAt time.Time
+	// logNum corresponds to the WAL that contains the records present in the
+	// receiver.
+	logNum FileNum
+	// logSize is the size in bytes of the associated WAL. Protected by DB.mu.
+	logSize uint64
+	// The current logSeqNum at the time the memtable was created. This is
+	// guaranteed to be less than or equal to any seqnum stored in the memtable.
+	logSeqNum uint64
+	// readerRefs tracks the read references on the flushable. The two sources of
+	// reader references are DB.mu.mem.queue and readState.memtables. The memory
+	// reserved by the flushable in the cache is released when the reader refs
+	// drop to zero. If the flushable is referencing sstables, then the file
+	// refount is also decreased once the reader refs drops to 0. If the
+	// flushable is a memTable, when the reader refs drops to zero, the writer
+	// refs will already be zero because the memtable will have been flushed and
+	// that only occurs once the writer refs drops to zero.
+	readerRefs atomic.Int32
+	// Closure to invoke to release memory accounting.
+	releaseMemAccounting func()
+	// unrefFiles, if not nil, should be invoked to decrease the ref count of
+	// files which are backing the flushable.
+	unrefFiles func() []*fileBacking
+	// deleteFnLocked should be called if the caller is holding DB.mu.
+	deleteFnLocked func(obsolete []*fileBacking)
+	// deleteFn should be called if the caller is not holding DB.mu.
+	deleteFn func(obsolete []*fileBacking)
+}
+
+func (e *flushableEntry) readerRef() {
+	switch v := e.readerRefs.Add(1); {
+	case v <= 1:
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+	}
+}
+
+// db.mu must not be held when this is called.
+func (e *flushableEntry) readerUnref(deleteFiles bool) {
+	e.readerUnrefHelper(deleteFiles, e.deleteFn)
+}
+
+// db.mu must be held when this is called.
+func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) {
+	e.readerUnrefHelper(deleteFiles, e.deleteFnLocked)
+}
+
+func (e *flushableEntry) readerUnrefHelper(
+	deleteFiles bool, deleteFn func(obsolete []*fileBacking),
+) {
+	switch v := e.readerRefs.Add(-1); {
+	case v < 0:
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+	case v == 0:
+		if e.releaseMemAccounting == nil {
+			panic("pebble: memtable reservation already released")
+		}
+		e.releaseMemAccounting()
+		e.releaseMemAccounting = nil
+		if e.unrefFiles != nil {
+			obsolete := e.unrefFiles()
+			e.unrefFiles = nil
+			if deleteFiles {
+				deleteFn(obsolete)
+			}
+		}
+	}
+}
+
+type flushableList []*flushableEntry
+
+// ingestedFlushable is the implementation of the flushable interface for the
+// ingesting sstables which are added to the flushable list.
+type ingestedFlushable struct {
+	files            []physicalMeta
+	comparer         *Comparer
+	newIters         tableNewIters
+	newRangeKeyIters keyspan.TableNewSpanIter
+
+	// Since the level slice is immutable, we construct and set it once. It
+	// should be safe to read from slice in future reads.
+	slice manifest.LevelSlice
+	// hasRangeKeys is set on ingestedFlushable construction.
+	hasRangeKeys bool
+}
+
+func newIngestedFlushable(
+	files []*fileMetadata,
+	comparer *Comparer,
+	newIters tableNewIters,
+	newRangeKeyIters keyspan.TableNewSpanIter,
+) *ingestedFlushable {
+	var physicalFiles []physicalMeta
+	var hasRangeKeys bool
+	for _, f := range files {
+		if f.HasRangeKeys {
+			hasRangeKeys = true
+		}
+		physicalFiles = append(physicalFiles, f.PhysicalMeta())
+	}
+
+	ret := &ingestedFlushable{
+		files:            physicalFiles,
+		comparer:         comparer,
+		newIters:         newIters,
+		newRangeKeyIters: newRangeKeyIters,
+		// slice is immutable and can be set once and used many times.
+		slice:        manifest.NewLevelSliceKeySorted(comparer.Compare, files),
+		hasRangeKeys: hasRangeKeys,
+	}
+
+	return ret
+}
+
+// TODO(sumeer): ingestedFlushable iters also need to plumb context for
+// tracing.
+
+// newIter is part of the flushable interface.
+func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
+	var opts IterOptions
+	if o != nil {
+		opts = *o
+	}
+	// TODO(bananabrick): The manifest.Level in newLevelIter is only used for
+	// logging. Update the manifest.Level encoding to account for levels which
+	// aren't truly levels in the lsm. Right now, the encoding only supports
+	// L0 sublevels, and the rest of the levels in the lsm.
+	return newLevelIter(
+		opts, s.comparer, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
+	)
+}
+
+// newFlushIter is part of the flushable interface.
+func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
+	// newFlushIter is only used for writing memtables to disk as sstables.
+	// Since ingested sstables are already present on disk, they don't need to
+	// make use of a flush iter.
+	panic("pebble: not implemented")
+}
+
+func (s *ingestedFlushable) constructRangeDelIter(
+	file *manifest.FileMetadata, _ keyspan.SpanIterOptions,
+) (keyspan.FragmentIterator, error) {
+	// Note that the keyspan level iter expects a non-nil iterator to be
+	// returned even if there is an error. So, we return the emptyKeyspanIter.
+	iter, rangeDelIter, err := s.newIters(context.Background(), file, nil, internalIterOpts{})
+	if err != nil {
+		return emptyKeyspanIter, err
+	}
+	iter.Close()
+	if rangeDelIter == nil {
+		return emptyKeyspanIter, nil
+	}
+	return rangeDelIter, nil
+}
+
+// newRangeDelIter is part of the flushable interface.
+// TODO(bananabrick): Using a level iter instead of a keyspan level iter to
+// surface range deletes is more efficient.
+func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
+	return keyspan.NewLevelIter(
+		keyspan.SpanIterOptions{}, s.comparer.Compare,
+		s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
+		manifest.KeyTypePoint,
+	)
+}
+
+// newRangeKeyIter is part of the flushable interface.
+func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
+	if !s.containsRangeKeys() {
+		return nil
+	}
+
+	return keyspan.NewLevelIter(
+		keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters,
+		s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
+	)
+}
+
+// containsRangeKeys is part of the flushable interface.
+func (s *ingestedFlushable) containsRangeKeys() bool {
+	return s.hasRangeKeys
+}
+
+// inuseBytes is part of the flushable interface.
+func (s *ingestedFlushable) inuseBytes() uint64 {
+	// inuseBytes is only used when memtables are flushed to disk as sstables.
+	panic("pebble: not implemented")
+}
+
+// totalBytes is part of the flushable interface.
+func (s *ingestedFlushable) totalBytes() uint64 {
+	// We don't allocate additional bytes for the ingestedFlushable.
+	return 0
+}
+
+// readyForFlush is part of the flushable interface.
+func (s *ingestedFlushable) readyForFlush() bool {
+	// ingestedFlushable should always be ready to flush. However, note that
+	// memtables before the ingested sstables in the memtable queue must be
+	// flushed before an ingestedFlushable can be flushed. This is because the
+	// ingested sstables need an updated view of the Version to
+	// determine where to place the files in the lsm.
+	return true
+}

vendor/github.com/cockroachdb/pebble/format_major_version.go

@@ -0,0 +1,678 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/sstable"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/pebble/vfs/atomicfs"
+)
+
+// FormatMajorVersion is a constant controlling the format of persisted
+// data. Backwards incompatible changes to durable formats are gated
+// behind new format major versions.
+//
+// At any point, a database's format major version may be bumped.
+// However, once a database's format major version is increased,
+// previous versions of Pebble will refuse to open the database.
+//
+// The zero value format is the FormatDefault constant. The exact
+// FormatVersion that the default corresponds to may change with time.
+type FormatMajorVersion uint64
+
+// SafeValue implements redact.SafeValue.
+func (v FormatMajorVersion) SafeValue() {}
+
+// String implements fmt.Stringer.
+func (v FormatMajorVersion) String() string {
+	// NB: This must not change. It's used as the value for the on-disk
+	// version marker file.
+	//
+	// Specifically, this value must always parse as a base 10 integer
+	// that fits in a uint64. We format it as zero-padded, 3-digit
+	// number today, but the padding may change.
+	return fmt.Sprintf("%03d", v)
+}
+
+const (
+	// 21.2 versions.
+
+	// FormatDefault leaves the format version unspecified. The
+	// FormatDefault constant may be ratcheted upwards over time.
+	FormatDefault FormatMajorVersion = iota
+	// FormatMostCompatible maintains the most backwards compatibility,
+	// maintaining bi-directional compatibility with RocksDB 6.2.1 in
+	// the particular configuration described in the Pebble README.
+	FormatMostCompatible
+	// formatVersionedManifestMarker is the first
+	// backwards-incompatible change made to Pebble, introducing the
+	// format-version marker file for handling backwards-incompatible
+	// changes more broadly, and replacing the `CURRENT` file with a
+	// marker file.
+	//
+	// This format version is intended as an intermediary version state.
+	// It is deliberately unexported to discourage direct use of this
+	// format major version.  Clients should use FormatVersioned which
+	// also ensures earlier versions of Pebble fail to open a database
+	// written in a future format major version.
+	formatVersionedManifestMarker
+	// FormatVersioned is a new format major version that replaces the
+	// old `CURRENT` file with a new 'marker' file scheme.  Previous
+	// Pebble versions will be unable to open the database unless
+	// they're aware of format versions.
+	FormatVersioned
+	// FormatSetWithDelete is a format major version that introduces a new key
+	// kind, base.InternalKeyKindSetWithDelete. Previous Pebble versions will be
+	// unable to open this database.
+	FormatSetWithDelete
+
+	// 22.1 versions.
+
+	// FormatBlockPropertyCollector is a format major version that introduces
+	// BlockPropertyCollectors.
+	FormatBlockPropertyCollector
+	// FormatSplitUserKeysMarked is a format major version that guarantees that
+	// all files that share user keys with neighbors are marked for compaction
+	// in the manifest. Ratcheting to FormatSplitUserKeysMarked will block
+	// (without holding mutexes) until the scan of the LSM is complete and the
+	// manifest has been rotated.
+	FormatSplitUserKeysMarked
+
+	// 22.2 versions.
+
+	// FormatSplitUserKeysMarkedCompacted is a format major version that
+	// guarantees that all files explicitly marked for compaction in the manifest
+	// have been compacted. Combined with the FormatSplitUserKeysMarked format
+	// major version, this version guarantees that there are no user keys split
+	// across multiple files within a level L1+. Ratcheting to this format version
+	// will block (without holding mutexes) until all necessary compactions for
+	// files marked for compaction are complete.
+	FormatSplitUserKeysMarkedCompacted
+	// FormatRangeKeys is a format major version that introduces range keys.
+	FormatRangeKeys
+	// FormatMinTableFormatPebblev1 is a format major version that guarantees that
+	// tables created by or ingested into the DB at or above this format major
+	// version will have a table format version of at least Pebblev1 (Block
+	// Properties).
+	FormatMinTableFormatPebblev1
+	// FormatPrePebblev1Marked is a format major version that guarantees that all
+	// sstables with a table format version pre-Pebblev1 (i.e. those that are
+	// guaranteed to not contain block properties) are marked for compaction in
+	// the manifest. Ratcheting to FormatPrePebblev1Marked will block (without
+	// holding mutexes) until the scan of the LSM is complete and the manifest has
+	// been rotated.
+	FormatPrePebblev1Marked
+
+	// 23.1 versions.
+
+	// formatUnusedPrePebblev1MarkedCompacted is an unused format major version.
+	// This format major version was originally intended to ship in the 23.1
+	// release. It was later decided that this should be deferred until a
+	// subsequent release. The original ordering is preserved so as not to
+	// introduce breaking changes in Cockroach.
+	formatUnusedPrePebblev1MarkedCompacted
+
+	// FormatSSTableValueBlocks is a format major version that adds support for
+	// storing values in value blocks in the sstable. Value block support is not
+	// necessarily enabled when writing sstables, when running with this format
+	// major version.
+	//
+	// WARNING: In development, so no production code should upgrade to this
+	// format, since a DB with this format major version will not actually
+	// interoperate correctly with another DB with the same format major
+	// version. This format major version is introduced so that tests can start
+	// being executed up to this version. Note that these tests succeed despite
+	// the incomplete support since they do not enable value blocks and use
+	// TableFormatPebblev2.
+	FormatSSTableValueBlocks
+
+	// FormatFlushableIngest is a format major version that enables lazy
+	// addition of ingested sstables into the LSM structure. When an ingest
+	// overlaps with a memtable, a record of the ingest is written to the WAL
+	// without waiting for a flush. Subsequent reads treat the ingested files as
+	// a level above the overlapping memtable. Once the memtable is flushed, the
+	// ingested files are moved into the lowest possible levels.
+	//
+	// This feature is behind a format major version because it required
+	// breaking changes to the WAL format.
+	FormatFlushableIngest
+
+	// 23.2 versions.
+
+	// FormatPrePebblev1MarkedCompacted is a format major version that guarantees
+	// that all sstables explicitly marked for compaction in the manifest (see
+	// FormatPrePebblev1Marked) have been compacted. Ratcheting to this format
+	// version will block (without holding mutexes) until all necessary
+	// compactions for files marked for compaction are complete.
+	FormatPrePebblev1MarkedCompacted
+
+	// FormatDeleteSizedAndObsolete is a format major version that adds support
+	// for deletion tombstones that encode the size of the value they're
+	// expected to delete. This format major version is required before the
+	// associated key kind may be committed through batch applications or
+	// ingests. It also adds support for keys that are marked obsolete (see
+	// sstable/format.go for details).
+	FormatDeleteSizedAndObsolete
+
+	// FormatVirtualSSTables is a format major version that adds support for
+	// virtual sstables that can reference a sub-range of keys in an underlying
+	// physical sstable. This information is persisted through new,
+	// backward-incompatible fields in the Manifest, and therefore requires
+	// a format major version.
+	FormatVirtualSSTables
+
+	// internalFormatNewest holds the newest format major version, including
+	// experimental ones excluded from the exported FormatNewest constant until
+	// they've stabilized. Used in tests.
+	internalFormatNewest FormatMajorVersion = iota - 1
+
+	// FormatNewest always contains the most recent format major version.
+	FormatNewest FormatMajorVersion = internalFormatNewest
+)
+
+// MaxTableFormat returns the maximum sstable.TableFormat that can be used at
+// this FormatMajorVersion.
+func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
+	switch v {
+	case FormatDefault, FormatMostCompatible, formatVersionedManifestMarker,
+		FormatVersioned, FormatSetWithDelete:
+		return sstable.TableFormatRocksDBv2
+	case FormatBlockPropertyCollector, FormatSplitUserKeysMarked,
+		FormatSplitUserKeysMarkedCompacted:
+		return sstable.TableFormatPebblev1
+	case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
+		formatUnusedPrePebblev1MarkedCompacted:
+		return sstable.TableFormatPebblev2
+	case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
+		return sstable.TableFormatPebblev3
+	case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
+		return sstable.TableFormatPebblev4
+	default:
+		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
+	}
+}
+
+// MinTableFormat returns the minimum sstable.TableFormat that can be used at
+// this FormatMajorVersion.
+func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
+	switch v {
+	case FormatDefault, FormatMostCompatible, formatVersionedManifestMarker,
+		FormatVersioned, FormatSetWithDelete, FormatBlockPropertyCollector,
+		FormatSplitUserKeysMarked, FormatSplitUserKeysMarkedCompacted,
+		FormatRangeKeys:
+		return sstable.TableFormatLevelDB
+	case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
+		formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
+		FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
+		FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
+		return sstable.TableFormatPebblev1
+	default:
+		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
+	}
+}
+
+// orderingInvariants returns an enum encoding the set of invariants that must
+// hold within the receiver format major version. Invariants only get stricter
+// as the format major version advances, so it is okay to retrieve the
+// invariants from the current format major version and by the time the
+// invariants are enforced, the format major version has advanced.
+func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants {
+	if v < FormatSplitUserKeysMarkedCompacted {
+		return manifest.AllowSplitUserKeys
+	}
+	return manifest.ProhibitSplitUserKeys
+}
+
+// formatMajorVersionMigrations defines the migrations from one format
+// major version to the next. Each migration is defined as a closure
+// which will be invoked on the database before the new format major
+// version is committed. Migrations must be idempotent. Migrations are
+// invoked with d.mu locked.
+//
+// Each migration is responsible for invoking finalizeFormatVersUpgrade
+// to set the new format major version.  RatchetFormatMajorVersion will
+// panic if a migration returns a nil error but fails to finalize the
+// new format major version.
+var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
+	FormatMostCompatible: func(d *DB) error { return nil },
+	formatVersionedManifestMarker: func(d *DB) error {
+		// formatVersionedManifestMarker introduces the use of a marker
+		// file for pointing to the current MANIFEST file.
+
+		// Lock the manifest.
+		d.mu.versions.logLock()
+		defer d.mu.versions.logUnlock()
+
+		// Construct the filename of the currently active manifest and
+		// move the manifest marker to that filename. The marker is
+		// guaranteed to exist, because we unconditionally locate it
+		// during Open.
+		manifestFileNum := d.mu.versions.manifestFileNum
+		filename := base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum())
+		if err := d.mu.versions.manifestMarker.Move(filename); err != nil {
+			return errors.Wrap(err, "moving manifest marker")
+		}
+
+		// Now that we have a manifest marker file in place and pointing
+		// to the current MANIFEST, finalize the upgrade. If we fail for
+		// some reason, a retry of this migration is guaranteed to again
+		// move the manifest marker file to the latest manifest. If
+		// we're unable to finalize the upgrade, a subsequent call to
+		// Open will ignore the manifest marker.
+		if err := d.finalizeFormatVersUpgrade(formatVersionedManifestMarker); err != nil {
+			return err
+		}
+
+		// We've finalized the upgrade. All subsequent Open calls will
+		// ignore the CURRENT file and instead read the manifest marker.
+		// Before we unlock the manifest, we need to update versionSet
+		// to use the manifest marker on future rotations.
+		d.mu.versions.setCurrent = setCurrentFuncMarker(
+			d.mu.versions.manifestMarker,
+			d.mu.versions.fs,
+			d.mu.versions.dirname)
+		return nil
+	},
+	// The FormatVersioned version is split into two, each with their
+	// own migration to ensure the post-migration cleanup happens even
+	// if there's a crash immediately after finalizing the version. Once
+	// a new format major version is finalized, its migration will never
+	// run again. Post-migration cleanup like the one in the migration
+	// below must be performed in a separate migration or every time the
+	// database opens.
+	FormatVersioned: func(d *DB) error {
+		// Replace the `CURRENT` file with one that points to the
+		// nonexistent `MANIFEST-000000` file. If an earlier Pebble
+		// version that does not know about format major versions
+		// attempts to open the database, it will error avoiding
+		// accidental corruption.
+		if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil {
+			return err
+		}
+		return d.finalizeFormatVersUpgrade(FormatVersioned)
+	},
+	// As SetWithDelete is a new key kind, there is nothing to migrate. We can
+	// simply finalize the format version and we're done.
+	FormatSetWithDelete: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatSetWithDelete)
+	},
+	FormatBlockPropertyCollector: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatBlockPropertyCollector)
+	},
+	FormatSplitUserKeysMarked: func(d *DB) error {
+		// Mark any unmarked files with split-user keys. Note all format major
+		// versions migrations are invoked with DB.mu locked.
+		if err := d.markFilesLocked(markFilesWithSplitUserKeys(d.opts.Comparer.Equal)); err != nil {
+			return err
+		}
+		return d.finalizeFormatVersUpgrade(FormatSplitUserKeysMarked)
+	},
+	FormatSplitUserKeysMarkedCompacted: func(d *DB) error {
+		// Before finalizing the format major version, rewrite any sstables
+		// still marked for compaction. Note all format major versions
+		// migrations are invoked with DB.mu locked.
+		if err := d.compactMarkedFilesLocked(); err != nil {
+			return err
+		}
+		return d.finalizeFormatVersUpgrade(FormatSplitUserKeysMarkedCompacted)
+	},
+	FormatRangeKeys: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatRangeKeys)
+	},
+	FormatMinTableFormatPebblev1: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatMinTableFormatPebblev1)
+	},
+	FormatPrePebblev1Marked: func(d *DB) error {
+		// Mark any unmarked files that contain only table properties. Note all
+		// format major versions migrations are invoked with DB.mu locked.
+		if err := d.markFilesLocked(markFilesPrePebblev1(d.tableCache)); err != nil {
+			return err
+		}
+		return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
+	},
+	formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
+		// Intentional no-op.
+		return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted)
+	},
+	FormatSSTableValueBlocks: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
+	},
+	FormatFlushableIngest: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatFlushableIngest)
+	},
+	FormatPrePebblev1MarkedCompacted: func(d *DB) error {
+		// Before finalizing the format major version, rewrite any sstables
+		// still marked for compaction. Note all format major versions
+		// migrations are invoked with DB.mu locked.
+		if err := d.compactMarkedFilesLocked(); err != nil {
+			return err
+		}
+		return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
+	},
+	FormatDeleteSizedAndObsolete: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete)
+	},
+	FormatVirtualSSTables: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
+	},
+}
+
+const formatVersionMarkerName = `format-version`
+
+func lookupFormatMajorVersion(
+	fs vfs.FS, dirname string,
+) (FormatMajorVersion, *atomicfs.Marker, error) {
+	m, versString, err := atomicfs.LocateMarker(fs, dirname, formatVersionMarkerName)
+	if err != nil {
+		return 0, nil, err
+	}
+	if versString == "" {
+		return FormatMostCompatible, m, nil
+	}
+	v, err := strconv.ParseUint(versString, 10, 64)
+	if err != nil {
+		return 0, nil, errors.Wrap(err, "parsing format major version")
+	}
+	vers := FormatMajorVersion(v)
+	if vers == FormatDefault {
+		return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers)
+	}
+	if vers > internalFormatNewest {
+		return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers)
+	}
+	return vers, m, nil
+}
+
+// FormatMajorVersion returns the database's active format major
+// version. The format major version may be higher than the one
+// provided in Options when the database was opened if the existing
+// database was written with a higher format version.
+func (d *DB) FormatMajorVersion() FormatMajorVersion {
+	return FormatMajorVersion(d.mu.formatVers.vers.Load())
+}
+
+// RatchetFormatMajorVersion ratchets the opened database's format major
+// version to the provided version. It errors if the provided format
+// major version is below the database's current version. Once a
+// database's format major version is upgraded, previous Pebble versions
+// that do not know of the format version will be unable to open the
+// database.
+func (d *DB) RatchetFormatMajorVersion(fmv FormatMajorVersion) error {
+	if err := d.closed.Load(); err != nil {
+		panic(err)
+	}
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.ratchetFormatMajorVersionLocked(fmv)
+}
+
+func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) error {
+	if d.opts.ReadOnly {
+		return ErrReadOnly
+	}
+	if formatVers > internalFormatNewest {
+		// Guard against accidentally forgetting to update internalFormatNewest.
+		return errors.Errorf("pebble: unknown format version %d", formatVers)
+	}
+	if currentVers := d.FormatMajorVersion(); currentVers > formatVers {
+		return errors.Newf("pebble: database already at format major version %d; cannot reduce to %d",
+			currentVers, formatVers)
+	}
+	if d.mu.formatVers.ratcheting {
+		return errors.Newf("pebble: database format major version upgrade is in-progress")
+	}
+	d.mu.formatVers.ratcheting = true
+	defer func() { d.mu.formatVers.ratcheting = false }()
+
+	for nextVers := d.FormatMajorVersion() + 1; nextVers <= formatVers; nextVers++ {
+		if err := formatMajorVersionMigrations[nextVers](d); err != nil {
+			return errors.Wrapf(err, "migrating to version %d", nextVers)
+		}
+
+		// NB: The migration is responsible for calling
+		// finalizeFormatVersUpgrade to finalize the upgrade. This
+		// structure is necessary because some migrations may need to
+		// update in-memory state (without ever dropping locks) after
+		// the upgrade is finalized. Here we assert that the upgrade
+		// did occur.
+		if d.FormatMajorVersion() != nextVers {
+			d.opts.Logger.Fatalf("pebble: successful migration to format version %d never finalized the upgrade", nextVers)
+		}
+	}
+	return nil
+}
+
+// finalizeFormatVersUpgrade is typically only be called from within a
+// format major version migration.
+//
+// See formatMajorVersionMigrations.
+func (d *DB) finalizeFormatVersUpgrade(formatVers FormatMajorVersion) error {
+	// We use the marker to encode the active format version in the
+	// marker filename. Unlike other uses of the atomic marker, there is
+	// no file with the filename `formatVers.String()` on the
+	// filesystem.
+	if err := d.mu.formatVers.marker.Move(formatVers.String()); err != nil {
+		return err
+	}
+	d.mu.formatVers.vers.Store(uint64(formatVers))
+	d.opts.EventListener.FormatUpgrade(formatVers)
+	return nil
+}
+
+// compactMarkedFilesLocked performs a migration that schedules rewrite
+// compactions to compact away any sstables marked for compaction.
+// compactMarkedFilesLocked is run while ratcheting the database's format major
+// version to FormatSplitUserKeysMarkedCompacted.
+//
+// Note that while this method is called with the DB.mu held, and will not
+// return until all marked files have been compacted, the mutex is dropped while
+// waiting for compactions to complete (or for slots to free up).
+func (d *DB) compactMarkedFilesLocked() error {
+	curr := d.mu.versions.currentVersion()
+	for curr.Stats.MarkedForCompaction > 0 {
+		// Attempt to schedule a compaction to rewrite a file marked for
+		// compaction.
+		d.maybeScheduleCompactionPicker(func(picker compactionPicker, env compactionEnv) *pickedCompaction {
+			return picker.pickRewriteCompaction(env)
+		})
+
+		// The above attempt might succeed and schedule a rewrite compaction. Or
+		// there might not be available compaction concurrency to schedule the
+		// compaction.  Or compaction of the file might have already been in
+		// progress. In any scenario, wait until there's some change in the
+		// state of active compactions.
+
+		// Before waiting, check that the database hasn't been closed. Trying to
+		// schedule the compaction may have dropped d.mu while waiting for a
+		// manifest write to complete. In that dropped interim, the database may
+		// have been closed.
+		if err := d.closed.Load(); err != nil {
+			return err.(error)
+		}
+
+		// Some flush or compaction may have scheduled or completed while we waited
+		// for the manifest lock in maybeScheduleCompactionPicker. Get the latest
+		// Version before waiting on a compaction.
+		curr = d.mu.versions.currentVersion()
+
+		// Only wait on compactions if there are files still marked for compaction.
+		// NB: Waiting on this condition variable drops d.mu while blocked.
+		if curr.Stats.MarkedForCompaction > 0 {
+			if d.mu.compact.compactingCount == 0 {
+				panic("expected a compaction of marked files in progress")
+			}
+			d.mu.compact.cond.Wait()
+			// Refresh the current version again.
+			curr = d.mu.versions.currentVersion()
+		}
+	}
+	return nil
+}
+
+// findFilesFunc scans the LSM for files, returning true if at least one
+// file was found. The returned array contains the matched files, if any, per
+// level.
+type findFilesFunc func(v *version) (found bool, files [numLevels][]*fileMetadata, _ error)
+
+// markFilesWithSplitUserKeys scans the LSM's levels 1 through 6 for adjacent
+// files that contain the same user key. Such arrangements of files were
+// permitted in RocksDB and in Pebble up to SHA a860bbad.
+var markFilesWithSplitUserKeys = func(equal Equal) findFilesFunc {
+	return func(v *version) (found bool, files [numLevels][]*fileMetadata, _ error) {
+		// Files with split user keys are expected to be rare and performing key
+		// comparisons for every file within the LSM is expensive, so drop the
+		// database lock while scanning the file metadata.
+		for l := numLevels - 1; l > 0; l-- {
+			iter := v.Levels[l].Iter()
+			var prevFile *fileMetadata
+			var prevUserKey []byte
+			for f := iter.First(); f != nil; f = iter.Next() {
+				if prevUserKey != nil && equal(prevUserKey, f.Smallest.UserKey) {
+					// NB: We may append a file twice, once as prevFile and once
+					// as f. That's okay, and handled below.
+					files[l] = append(files[l], prevFile, f)
+					found = true
+				}
+				if f.Largest.IsExclusiveSentinel() {
+					prevUserKey = nil
+					prevFile = nil
+				} else {
+					prevUserKey = f.Largest.UserKey
+					prevFile = f
+				}
+			}
+		}
+		return
+	}
+}
+
+// markFilesPrePebblev1 scans the LSM for files that do not support block
+// properties (i.e. a table format version pre-Pebblev1).
+var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc {
+	return func(v *version) (found bool, files [numLevels][]*fileMetadata, err error) {
+		for l := numLevels - 1; l > 0; l-- {
+			iter := v.Levels[l].Iter()
+			for f := iter.First(); f != nil; f = iter.Next() {
+				if f.Virtual {
+					// Any physical sstable which has been virtualized must
+					// have already undergone this migration, and we don't
+					// need to worry about the virtual sstable themselves.
+					panic("pebble: unexpected virtual sstable during migration")
+				}
+				err = tc.withReader(
+					f.PhysicalMeta(), func(r *sstable.Reader) error {
+						tf, err := r.TableFormat()
+						if err != nil {
+							return err
+						}
+						if tf < sstable.TableFormatPebblev1 {
+							found = true
+							files[l] = append(files[l], f)
+						}
+						return nil
+					})
+				if err != nil {
+					return
+				}
+			}
+		}
+		return
+	}
+}
+
+// markFilesLock durably marks the files that match the given findFilesFunc for
+// compaction.
+func (d *DB) markFilesLocked(findFn findFilesFunc) error {
+	jobID := d.mu.nextJobID
+	d.mu.nextJobID++
+
+	// Acquire a read state to have a view of the LSM and a guarantee that none
+	// of the referenced files will be deleted until we've unreferenced the read
+	// state. Some findFilesFuncs may read the files, requiring they not be
+	// deleted.
+	rs := d.loadReadState()
+	var (
+		found bool
+		files [numLevels][]*fileMetadata
+		err   error
+	)
+	func() {
+		defer rs.unrefLocked()
+		// Note the unusual locking: unlock, defer Lock(). The scan of the files in
+		// the version does not need to block other operations that require the
+		// DB.mu. Drop it for the scan, before re-acquiring it.
+		d.mu.Unlock()
+		defer d.mu.Lock()
+		found, files, err = findFn(rs.current)
+	}()
+	if err != nil {
+		return err
+	}
+
+	// The database lock has been acquired again by the defer within the above
+	// anonymous function.
+	if !found {
+		// Nothing to do.
+		return nil
+	}
+
+	// After scanning, if we found files to mark, we fetch the current state of
+	// the LSM (which may have changed) and set MarkedForCompaction on the files,
+	// and update the version's Stats.MarkedForCompaction count, which are both
+	// protected by d.mu.
+
+	// Lock the manifest for a coherent view of the LSM. The database lock has
+	// been re-acquired by the defer within the above anonymous function.
+	d.mu.versions.logLock()
+	vers := d.mu.versions.currentVersion()
+	for l, filesToMark := range files {
+		if len(filesToMark) == 0 {
+			continue
+		}
+		for _, f := range filesToMark {
+			// Ignore files to be marked that have already been compacted or marked.
+			if f.CompactionState == manifest.CompactionStateCompacted ||
+				f.MarkedForCompaction {
+				continue
+			}
+			// Else, mark the file for compaction in this version.
+			vers.Stats.MarkedForCompaction++
+			f.MarkedForCompaction = true
+		}
+		// The compaction picker uses the markedForCompactionAnnotator to
+		// quickly find files marked for compaction, or to quickly determine
+		// that there are no such files marked for compaction within a level.
+		// A b-tree node may be annotated with an annotation recording that
+		// there are no files marked for compaction within the node's subtree,
+		// based on the assumption that it's static.
+		//
+		// Since we're marking files for compaction, these b-tree nodes'
+		// annotations will be out of date. Clear the compaction-picking
+		// annotation, so that it's recomputed the next time the compaction
+		// picker looks for a file marked for compaction.
+		vers.Levels[l].InvalidateAnnotation(markedForCompactionAnnotator{})
+	}
+
+	// The 'marked-for-compaction' bit is persisted in the MANIFEST file
+	// metadata. We've already modified the in-memory file metadata, but the
+	// manifest hasn't been updated. Force rotation to a new MANIFEST file,
+	// which will write every file metadata to the new manifest file and ensure
+	// that the now marked-for-compaction file metadata are persisted as marked.
+	// NB: This call to logAndApply will unlockthe MANIFEST, which we locked up
+	// above before obtaining `vers`.
+	return d.mu.versions.logAndApply(
+		jobID,
+		&manifest.VersionEdit{},
+		map[int]*LevelMetrics{},
+		true, /* forceRotation */
+		func() []compactionInfo { return d.getInProgressCompactionInfoLocked(nil) })
+}

vendor/github.com/cockroachdb/pebble/get_iter.go

@@ -0,0 +1,243 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manifest"
+)
+
+// getIter is an internal iterator used to perform gets. It iterates through
+// the values for a particular key, level by level. It is not a general purpose
+// internalIterator, but specialized for Get operations so that it loads data
+// lazily.
+type getIter struct {
+	logger       Logger
+	comparer     *Comparer
+	newIters     tableNewIters
+	snapshot     uint64
+	key          []byte
+	iter         internalIterator
+	rangeDelIter keyspan.FragmentIterator
+	tombstone    *keyspan.Span
+	levelIter    levelIter
+	level        int
+	batch        *Batch
+	mem          flushableList
+	l0           []manifest.LevelSlice
+	version      *version
+	iterKey      *InternalKey
+	iterValue    base.LazyValue
+	err          error
+}
+
+// TODO(sumeer): CockroachDB code doesn't use getIter, but, for completeness,
+// make this implement InternalIteratorWithStats.
+
+// getIter implements the base.InternalIterator interface.
+var _ base.InternalIterator = (*getIter)(nil)
+
+func (g *getIter) String() string {
+	return fmt.Sprintf("len(l0)=%d, len(mem)=%d, level=%d", len(g.l0), len(g.mem), g.level)
+}
+
+func (g *getIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) {
+	panic("pebble: SeekGE unimplemented")
+}
+
+func (g *getIter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("pebble: SeekPrefixGE unimplemented")
+}
+
+func (g *getIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, base.LazyValue) {
+	panic("pebble: SeekLT unimplemented")
+}
+
+func (g *getIter) First() (*InternalKey, base.LazyValue) {
+	return g.Next()
+}
+
+func (g *getIter) Last() (*InternalKey, base.LazyValue) {
+	panic("pebble: Last unimplemented")
+}
+
+func (g *getIter) Next() (*InternalKey, base.LazyValue) {
+	if g.iter != nil {
+		g.iterKey, g.iterValue = g.iter.Next()
+	}
+
+	for {
+		if g.iter != nil {
+			// We have to check rangeDelIter on each iteration because a single
+			// user-key can be spread across multiple tables in a level. A range
+			// tombstone will appear in the table corresponding to its start
+			// key. Every call to levelIter.Next() potentially switches to a new
+			// table and thus reinitializes rangeDelIter.
+			if g.rangeDelIter != nil {
+				g.tombstone = keyspan.Get(g.comparer.Compare, g.rangeDelIter, g.key)
+				if g.err = g.rangeDelIter.Close(); g.err != nil {
+					return nil, base.LazyValue{}
+				}
+				g.rangeDelIter = nil
+			}
+
+			if g.iterKey != nil {
+				key := g.iterKey
+				if g.tombstone != nil && g.tombstone.CoversAt(g.snapshot, key.SeqNum()) {
+					// We have a range tombstone covering this key. Rather than return a
+					// point or range deletion here, we return false and close our
+					// internal iterator which will make Valid() return false,
+					// effectively stopping iteration.
+					g.err = g.iter.Close()
+					g.iter = nil
+					return nil, base.LazyValue{}
+				}
+				if g.comparer.Equal(g.key, key.UserKey) {
+					if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) {
+						g.iterKey, g.iterValue = g.iter.Next()
+						continue
+					}
+					return g.iterKey, g.iterValue
+				}
+			}
+			// We've advanced the iterator passed the desired key. Move on to the
+			// next memtable / level.
+			g.err = g.iter.Close()
+			g.iter = nil
+			if g.err != nil {
+				return nil, base.LazyValue{}
+			}
+		}
+
+		// Create an iterator from the batch.
+		if g.batch != nil {
+			if g.batch.index == nil {
+				g.err = ErrNotIndexed
+				g.iterKey, g.iterValue = nil, base.LazyValue{}
+				return nil, base.LazyValue{}
+			}
+			g.iter = g.batch.newInternalIter(nil)
+			g.rangeDelIter = g.batch.newRangeDelIter(
+				nil,
+				// Get always reads the entirety of the batch's history, so no
+				// batch keys should be filtered.
+				base.InternalKeySeqNumMax,
+			)
+			g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
+			g.batch = nil
+			continue
+		}
+
+		// If we have a tombstone from a previous level it is guaranteed to delete
+		// keys in lower levels.
+		if g.tombstone != nil && g.tombstone.VisibleAt(g.snapshot) {
+			return nil, base.LazyValue{}
+		}
+
+		// Create iterators from memtables from newest to oldest.
+		if n := len(g.mem); n > 0 {
+			m := g.mem[n-1]
+			g.iter = m.newIter(nil)
+			g.rangeDelIter = m.newRangeDelIter(nil)
+			g.mem = g.mem[:n-1]
+			g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
+			continue
+		}
+
+		if g.level == 0 {
+			// Create iterators from L0 from newest to oldest.
+			if n := len(g.l0); n > 0 {
+				files := g.l0[n-1].Iter()
+				g.l0 = g.l0[:n-1]
+				iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
+				g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
+					files, manifest.L0Sublevel(n), internalIterOpts{})
+				g.levelIter.initRangeDel(&g.rangeDelIter)
+				bc := levelIterBoundaryContext{}
+				g.levelIter.initBoundaryContext(&bc)
+				g.iter = &g.levelIter
+
+				// Compute the key prefix for bloom filtering if split function is
+				// specified, or use the user key as default.
+				prefix := g.key
+				if g.comparer.Split != nil {
+					prefix = g.key[:g.comparer.Split(g.key)]
+				}
+				g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
+				if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
+					g.iterKey = nil
+					g.iterValue = base.LazyValue{}
+				}
+				continue
+			}
+			g.level++
+		}
+
+		if g.level >= numLevels {
+			return nil, base.LazyValue{}
+		}
+		if g.version.Levels[g.level].Empty() {
+			g.level++
+			continue
+		}
+
+		iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
+		g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
+			g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
+		g.levelIter.initRangeDel(&g.rangeDelIter)
+		bc := levelIterBoundaryContext{}
+		g.levelIter.initBoundaryContext(&bc)
+		g.level++
+		g.iter = &g.levelIter
+
+		// Compute the key prefix for bloom filtering if split function is
+		// specified, or use the user key as default.
+		prefix := g.key
+		if g.comparer.Split != nil {
+			prefix = g.key[:g.comparer.Split(g.key)]
+		}
+		g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
+		if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
+			g.iterKey = nil
+			g.iterValue = base.LazyValue{}
+		}
+	}
+}
+
+func (g *getIter) Prev() (*InternalKey, base.LazyValue) {
+	panic("pebble: Prev unimplemented")
+}
+
+func (g *getIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) {
+	panic("pebble: NextPrefix unimplemented")
+}
+
+func (g *getIter) Valid() bool {
+	return g.iterKey != nil && g.err == nil
+}
+
+func (g *getIter) Error() error {
+	return g.err
+}
+
+func (g *getIter) Close() error {
+	if g.iter != nil {
+		if err := g.iter.Close(); err != nil && g.err == nil {
+			g.err = err
+		}
+		g.iter = nil
+	}
+	return g.err
+}
+
+func (g *getIter) SetBounds(lower, upper []byte) {
+	panic("pebble: SetBounds unimplemented")
+}

vendor/github.com/cockroachdb/pebble/internal.go

@@ -0,0 +1,51 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package pebble
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// InternalKeyKind exports the base.InternalKeyKind type.
+type InternalKeyKind = base.InternalKeyKind
+
+// These constants are part of the file format, and should not be changed.
+const (
+	InternalKeyKindDelete          = base.InternalKeyKindDelete
+	InternalKeyKindSet             = base.InternalKeyKindSet
+	InternalKeyKindMerge           = base.InternalKeyKindMerge
+	InternalKeyKindLogData         = base.InternalKeyKindLogData
+	InternalKeyKindSingleDelete    = base.InternalKeyKindSingleDelete
+	InternalKeyKindRangeDelete     = base.InternalKeyKindRangeDelete
+	InternalKeyKindMax             = base.InternalKeyKindMax
+	InternalKeyKindSetWithDelete   = base.InternalKeyKindSetWithDelete
+	InternalKeyKindRangeKeySet     = base.InternalKeyKindRangeKeySet
+	InternalKeyKindRangeKeyUnset   = base.InternalKeyKindRangeKeyUnset
+	InternalKeyKindRangeKeyDelete  = base.InternalKeyKindRangeKeyDelete
+	InternalKeyKindIngestSST       = base.InternalKeyKindIngestSST
+	InternalKeyKindDeleteSized     = base.InternalKeyKindDeleteSized
+	InternalKeyKindInvalid         = base.InternalKeyKindInvalid
+	InternalKeySeqNumBatch         = base.InternalKeySeqNumBatch
+	InternalKeySeqNumMax           = base.InternalKeySeqNumMax
+	InternalKeyRangeDeleteSentinel = base.InternalKeyRangeDeleteSentinel
+)
+
+// InternalKey exports the base.InternalKey type.
+type InternalKey = base.InternalKey
+
+type internalIterator = base.InternalIterator
+
+// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST,
+// sstable) isn't in the expected format.
+var ErrCorruption = base.ErrCorruption
+
+// AttributeAndLen exports the base.AttributeAndLen type.
+type AttributeAndLen = base.AttributeAndLen
+
+// ShortAttribute exports the base.ShortAttribute type.
+type ShortAttribute = base.ShortAttribute
+
+// LazyFetcher exports the base.LazyFetcher type. This export is needed since
+// LazyValue.Clone requires a pointer to a LazyFetcher struct to avoid
+// allocations. No code outside Pebble needs to peer into a LazyFetcher.
+type LazyFetcher = base.LazyFetcher

vendor/github.com/cockroachdb/pebble/internal/arenaskl/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/cockroachdb/pebble/internal/arenaskl/README.md

@@ -0,0 +1,93 @@
+# arenaskl
+
+Fast, lock-free, arena-based Skiplist implementation in Go that supports iteration
+in both directions.
+
+## Advantages
+
+Arenaskl offers several advantages over other skiplist implementations:
+
+* High performance that linearly scales with the number of cores. This is
+  achieved by allocating from a fixed-size arena and by avoiding locks.
+* Iterators that can be allocated on the stack and easily cloned by value.
+* Simple-to-use and low overhead model for detecting and handling race conditions
+  with other threads.
+* Support for iterating in reverse (i.e. previous links). 
+
+## Limitations
+
+The advantages come at a cost that prevents arenaskl from being a general-purpose
+skiplist implementation:
+
+* The size of the arena sets a hard upper bound on the combined size of skiplist
+  nodes, keys, and values. This limit includes even the size of deleted nodes,
+  keys, and values.
+* Deletion is not supported. Instead, higher-level code is expected to
+  add deletion tombstones and needs to process those tombstones
+  appropriately.
+
+## Pedigree
+
+This code is based on Andy Kimball's arenaskl code:
+
+https://github.com/andy-kimball/arenaskl
+
+The arenaskl code is based on the skiplist found in Badger, a Go-based
+KV store:
+
+https://github.com/dgraph-io/badger/tree/master/skl
+
+The skiplist in Badger is itself based on a C++ skiplist built for
+Facebook's RocksDB:
+
+https://github.com/facebook/rocksdb/tree/master/memtable
+
+## Benchmarks
+
+The benchmarks consist of a mix of reads and writes executed in parallel. The
+fraction of reads is indicated in the run name: "frac_X" indicates a run where
+X percent of the operations are reads.
+
+The results are much better than `skiplist` and `slist`.
+
+```
+name                  time/op
+ReadWrite/frac_0-8     470ns ±11%
+ReadWrite/frac_10-8    462ns ± 3%
+ReadWrite/frac_20-8    436ns ± 2%
+ReadWrite/frac_30-8    410ns ± 2%
+ReadWrite/frac_40-8    385ns ± 2%
+ReadWrite/frac_50-8    360ns ± 4%
+ReadWrite/frac_60-8    386ns ± 1%
+ReadWrite/frac_70-8    352ns ± 2%
+ReadWrite/frac_80-8    306ns ± 3%
+ReadWrite/frac_90-8    253ns ± 4%
+ReadWrite/frac_100-8  28.1ns ± 2%
+```
+
+Note that the above numbers are for concurrent operations using 8x
+parallelism. The same benchmarks without concurrency (use these
+numbers when comparing vs batchskl):
+
+```
+name                time/op
+ReadWrite/frac_0    1.53µs ± 1%
+ReadWrite/frac_10   1.46µs ± 2%
+ReadWrite/frac_20   1.39µs ± 3%
+ReadWrite/frac_30   1.28µs ± 3%
+ReadWrite/frac_40   1.21µs ± 2%
+ReadWrite/frac_50   1.11µs ± 3%
+ReadWrite/frac_60   1.23µs ±17%
+ReadWrite/frac_70   1.16µs ± 4%
+ReadWrite/frac_80    959ns ± 3%
+ReadWrite/frac_90    738ns ± 5%
+ReadWrite/frac_100  81.9ns ± 2%
+```
+
+Forward and backward iteration are also fast:
+
+```
+name                time/op
+IterNext            3.97ns ± 5%
+IterPrev            3.88ns ± 3%
+```

vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ * Modifications copyright (C) 2017 Andy Kimball and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package arenaskl
+
+import (
+	"sync/atomic"
+	"unsafe"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/constants"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// Arena is lock-free.
+type Arena struct {
+	n   atomic.Uint64
+	buf []byte
+}
+
+const nodeAlignment = 4
+
+var (
+	// ErrArenaFull indicates that the arena is full and cannot perform any more
+	// allocations.
+	ErrArenaFull = errors.New("allocation failed because arena is full")
+)
+
+// NewArena allocates a new arena using the specified buffer as the backing
+// store.
+func NewArena(buf []byte) *Arena {
+	if len(buf) > constants.MaxUint32OrInt {
+		if invariants.Enabled {
+			panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
+		}
+		buf = buf[:constants.MaxUint32OrInt]
+	}
+	a := &Arena{
+		buf: buf,
+	}
+	// We don't store data at position 0 in order to reserve offset=0 as a kind of
+	// nil pointer.
+	a.n.Store(1)
+	return a
+}
+
+// Size returns the number of bytes allocated by the arena.
+func (a *Arena) Size() uint32 {
+	s := a.n.Load()
+	if s > constants.MaxUint32OrInt {
+		// The last failed allocation can push the size higher than len(a.buf).
+		// Saturate at the maximum representable offset.
+		return constants.MaxUint32OrInt
+	}
+	return uint32(s)
+}
+
+// Capacity returns the capacity of the arena.
+func (a *Arena) Capacity() uint32 {
+	return uint32(len(a.buf))
+}
+
+// alloc allocates a buffer of the given size and with the given alignment
+// (which must be a power of 2).
+//
+// If overflow is not 0, it also ensures that many bytes after the buffer are
+// inside the arena (this is used for structures that are larger than the
+// requested size but don't use those extra bytes).
+func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
+	if invariants.Enabled && (alignment&(alignment-1)) != 0 {
+		panic(errors.AssertionFailedf("invalid alignment %d", alignment))
+	}
+	// Verify that the arena isn't already full.
+	origSize := a.n.Load()
+	if int(origSize) > len(a.buf) {
+		return 0, 0, ErrArenaFull
+	}
+
+	// Pad the allocation with enough bytes to ensure the requested alignment.
+	padded := uint64(size) + uint64(alignment) - 1
+
+	newSize := a.n.Add(padded)
+	if newSize+uint64(overflow) > uint64(len(a.buf)) {
+		return 0, 0, ErrArenaFull
+	}
+
+	// Return the aligned offset.
+	offset := (uint32(newSize) - size) & ^(alignment - 1)
+	return offset, uint32(padded), nil
+}
+
+func (a *Arena) getBytes(offset uint32, size uint32) []byte {
+	if offset == 0 {
+		return nil
+	}
+	return a.buf[offset : offset+size : offset+size]
+}
+
+func (a *Arena) getPointer(offset uint32) unsafe.Pointer {
+	if offset == 0 {
+		return nil
+	}
+	return unsafe.Pointer(&a.buf[offset])
+}
+
+func (a *Arena) getPointerOffset(ptr unsafe.Pointer) uint32 {
+	if ptr == nil {
+		return 0
+	}
+	return uint32(uintptr(ptr) - uintptr(unsafe.Pointer(&a.buf[0])))
+}

vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ * Modifications copyright (C) 2017 Andy Kimball and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package arenaskl
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// flushIterator is an iterator over the skiplist object. Use Skiplist.NewFlushIter
+// to construct an iterator. The current state of the iterator can be cloned by
+// simply value copying the struct.
+type flushIterator struct {
+	Iterator
+	bytesIterated *uint64
+}
+
+// flushIterator implements the base.InternalIterator interface.
+var _ base.InternalIterator = (*flushIterator)(nil)
+
+func (it *flushIterator) String() string {
+	return "memtable"
+}
+
+func (it *flushIterator) SeekGE(
+	key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("pebble: SeekGE unimplemented")
+}
+
+func (it *flushIterator) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("pebble: SeekPrefixGE unimplemented")
+}
+
+func (it *flushIterator) SeekLT(
+	key []byte, flags base.SeekLTFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("pebble: SeekLT unimplemented")
+}
+
+// First seeks position at the first entry in list. Returns the key and value
+// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
+// that First only checks the upper bound. It is up to the caller to ensure
+// that key is greater than or equal to the lower bound.
+func (it *flushIterator) First() (*base.InternalKey, base.LazyValue) {
+	key, val := it.Iterator.First()
+	if key == nil {
+		return nil, base.LazyValue{}
+	}
+	*it.bytesIterated += uint64(it.nd.allocSize)
+	return key, val
+}
+
+// Next advances to the next position. Returns the key and value if the
+// iterator is pointing at a valid entry, and (nil, nil) otherwise.
+// Note: flushIterator.Next mirrors the implementation of Iterator.Next
+// due to performance. Keep the two in sync.
+func (it *flushIterator) Next() (*base.InternalKey, base.LazyValue) {
+	it.nd = it.list.getNext(it.nd, 0)
+	if it.nd == it.list.tail {
+		return nil, base.LazyValue{}
+	}
+	it.decodeKey()
+	*it.bytesIterated += uint64(it.nd.allocSize)
+	return &it.key, base.MakeInPlaceValue(it.value())
+}
+
+func (it *flushIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	panic("pebble: NextPrefix unimplemented")
+}
+
+func (it *flushIterator) Prev() (*base.InternalKey, base.LazyValue) {
+	panic("pebble: Prev unimplemented")
+}

vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go

@@ -0,0 +1,271 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ * Modifications copyright (C) 2017 Andy Kimball and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package arenaskl
+
+import (
+	"sync"
+
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+type splice struct {
+	prev *node
+	next *node
+}
+
+func (s *splice) init(prev, next *node) {
+	s.prev = prev
+	s.next = next
+}
+
+// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
+// to construct an iterator. The current state of the iterator can be cloned by
+// simply value copying the struct. All iterator methods are thread-safe.
+type Iterator struct {
+	list  *Skiplist
+	nd    *node
+	key   base.InternalKey
+	lower []byte
+	upper []byte
+}
+
+// Iterator implements the base.InternalIterator interface.
+var _ base.InternalIterator = (*Iterator)(nil)
+
+var iterPool = sync.Pool{
+	New: func() interface{} {
+		return &Iterator{}
+	},
+}
+
+// Close resets the iterator.
+func (it *Iterator) Close() error {
+	it.list = nil
+	it.nd = nil
+	it.lower = nil
+	it.upper = nil
+	iterPool.Put(it)
+	return nil
+}
+
+func (it *Iterator) String() string {
+	return "memtable"
+}
+
+// Error returns any accumulated error.
+func (it *Iterator) Error() error {
+	return nil
+}
+
+// SeekGE moves the iterator to the first entry whose key is greater than or
+// equal to the given key. Returns the key and value if the iterator is
+// pointing at a valid entry, and (nil, nil) otherwise. Note that SeekGE only
+// checks the upper bound. It is up to the caller to ensure that key is greater
+// than or equal to the lower bound.
+func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	if flags.TrySeekUsingNext() {
+		if it.nd == it.list.tail {
+			// Iterator is done.
+			return nil, base.LazyValue{}
+		}
+		less := it.list.cmp(it.key.UserKey, key) < 0
+		// Arbitrary constant. By measuring the seek cost as a function of the
+		// number of elements in the skip list, and fitting to a model, we
+		// could adjust the number of nexts based on the current size of the
+		// skip list.
+		const numNexts = 5
+		for i := 0; less && i < numNexts; i++ {
+			k, _ := it.Next()
+			if k == nil {
+				// Iterator is done.
+				return nil, base.LazyValue{}
+			}
+			less = it.list.cmp(it.key.UserKey, key) < 0
+		}
+		if !less {
+			return &it.key, base.MakeInPlaceValue(it.value())
+		}
+	}
+	_, it.nd, _ = it.seekForBaseSplice(key)
+	if it.nd == it.list.tail {
+		return nil, base.LazyValue{}
+	}
+	it.decodeKey()
+	if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
+		it.nd = it.list.tail
+		return nil, base.LazyValue{}
+	}
+	return &it.key, base.MakeInPlaceValue(it.value())
+}
+
+// SeekPrefixGE moves the iterator to the first entry whose key is greater than
+// or equal to the given key. This method is equivalent to SeekGE and is
+// provided so that an arenaskl.Iterator implements the
+// internal/base.InternalIterator interface.
+func (it *Iterator) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return it.SeekGE(key, flags)
+}
+
+// SeekLT moves the iterator to the last entry whose key is less than the given
+// key. Returns the key and value if the iterator is pointing at a valid entry,
+// and (nil, nil) otherwise. Note that SeekLT only checks the lower bound. It
+// is up to the caller to ensure that key is less than the upper bound.
+func (it *Iterator) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
+	// NB: the top-level Iterator has already adjusted key based on
+	// the upper-bound.
+	it.nd, _, _ = it.seekForBaseSplice(key)
+	if it.nd == it.list.head {
+		return nil, base.LazyValue{}
+	}
+	it.decodeKey()
+	if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
+		it.nd = it.list.head
+		return nil, base.LazyValue{}
+	}
+	return &it.key, base.MakeInPlaceValue(it.value())
+}
+
+// First seeks position at the first entry in list. Returns the key and value
+// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
+// that First only checks the upper bound. It is up to the caller to ensure
+// that key is greater than or equal to the lower bound (e.g. via a call to SeekGE(lower)).
+func (it *Iterator) First() (*base.InternalKey, base.LazyValue) {
+	it.nd = it.list.getNext(it.list.head, 0)
+	if it.nd == it.list.tail {
+		return nil, base.LazyValue{}
+	}
+	it.decodeKey()
+	if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
+		it.nd = it.list.tail
+		return nil, base.LazyValue{}
+	}
+	return &it.key, base.MakeInPlaceValue(it.value())
+}
+
+// Last seeks position at the last entry in list. Returns the key and value if
+// the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
+// that Last only checks the lower bound. It is up to the caller to ensure that
+// key is less than the upper bound (e.g. via a call to SeekLT(upper)).
+func (it *Iterator) Last() (*base.InternalKey, base.LazyValue) {
+	it.nd = it.list.getPrev(it.list.tail, 0)
+	if it.nd == it.list.head {
+		return nil, base.LazyValue{}
+	}
+	it.decodeKey()
+	if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
+		it.nd = it.list.head
+		return nil, base.LazyValue{}
+	}
+	return &it.key, base.MakeInPlaceValue(it.value())
+}
+
+// Next advances to the next position. Returns the key and value if the
+// iterator is pointing at a valid entry, and (nil, nil) otherwise.
+// Note: flushIterator.Next mirrors the implementation of Iterator.Next
+// due to performance. Keep the two in sync.
+func (it *Iterator) Next() (*base.InternalKey, base.LazyValue) {
+	it.nd = it.list.getNext(it.nd, 0)
+	if it.nd == it.list.tail {
+		return nil, base.LazyValue{}
+	}
+	it.decodeKey()
+	if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
+		it.nd = it.list.tail
+		return nil, base.LazyValue{}
+	}
+	return &it.key, base.MakeInPlaceValue(it.value())
+}
+
+// NextPrefix advances to the next position with a new prefix. Returns the key
+// and value if the iterator is pointing at a valid entry, and (nil, nil)
+// otherwise.
+func (it *Iterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return it.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext())
+}
+
+// Prev moves to the previous position. Returns the key and value if the
+// iterator is pointing at a valid entry, and (nil, nil) otherwise.
+func (it *Iterator) Prev() (*base.InternalKey, base.LazyValue) {
+	it.nd = it.list.getPrev(it.nd, 0)
+	if it.nd == it.list.head {
+		return nil, base.LazyValue{}
+	}
+	it.decodeKey()
+	if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
+		it.nd = it.list.head
+		return nil, base.LazyValue{}
+	}
+	return &it.key, base.MakeInPlaceValue(it.value())
+}
+
+// value returns the value at the current position.
+func (it *Iterator) value() []byte {
+	return it.nd.getValue(it.list.arena)
+}
+
+// Head true iff the iterator is positioned at the sentinel head node.
+func (it *Iterator) Head() bool {
+	return it.nd == it.list.head
+}
+
+// Tail true iff the iterator is positioned at the sentinel tail node.
+func (it *Iterator) Tail() bool {
+	return it.nd == it.list.tail
+}
+
+// SetBounds sets the lower and upper bounds for the iterator. Note that the
+// result of Next and Prev will be undefined until the iterator has been
+// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
+func (it *Iterator) SetBounds(lower, upper []byte) {
+	it.lower = lower
+	it.upper = upper
+}
+
+func (it *Iterator) decodeKey() {
+	it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
+	it.key.Trailer = it.nd.keyTrailer
+}
+
+func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {
+	ikey := base.MakeSearchKey(key)
+	level := int(it.list.Height() - 1)
+
+	prev = it.list.head
+	for {
+		prev, next, found = it.list.findSpliceForLevel(ikey, level, prev)
+
+		if found {
+			if level != 0 {
+				// next is pointing at the target node, but we need to find previous on
+				// the bottom level.
+				prev = it.list.getPrev(next, 0)
+			}
+			break
+		}
+
+		if level == 0 {
+			break
+		}
+
+		level--
+	}
+
+	return
+}

vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ * Modifications copyright (C) 2017 Andy Kimball and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package arenaskl
+
+import (
+	"math"
+	"sync/atomic"
+
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+// MaxNodeSize returns the maximum space needed for a node with the specified
+// key and value sizes. This could overflow a uint32, which is why a uint64
+// is used here. If a key/value overflows a uint32, it should not be added to
+// the skiplist.
+func MaxNodeSize(keySize, valueSize uint32) uint64 {
+	const maxPadding = nodeAlignment - 1
+	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
+}
+
+type links struct {
+	nextOffset atomic.Uint32
+	prevOffset atomic.Uint32
+}
+
+func (l *links) init(prevOffset, nextOffset uint32) {
+	l.nextOffset.Store(nextOffset)
+	l.prevOffset.Store(prevOffset)
+}
+
+type node struct {
+	// Immutable fields, so no need to lock to access key.
+	keyOffset  uint32
+	keySize    uint32
+	keyTrailer uint64
+	valueSize  uint32
+	allocSize  uint32
+
+	// Most nodes do not need to use the full height of the tower, since the
+	// probability of each successive level decreases exponentially. Because
+	// these elements are never accessed, they do not need to be allocated.
+	// Therefore, when a node is allocated in the arena, its memory footprint
+	// is deliberately truncated to not include unneeded tower elements.
+	//
+	// All accesses to elements should use CAS operations, with no need to lock.
+	tower [maxHeight]links
+}
+
+func newNode(
+	arena *Arena, height uint32, key base.InternalKey, value []byte,
+) (nd *node, err error) {
+	if height < 1 || height > maxHeight {
+		panic("height cannot be less than one or greater than the max height")
+	}
+	keySize := len(key.UserKey)
+	if int64(keySize) > math.MaxUint32 {
+		panic("key is too large")
+	}
+	valueSize := len(value)
+	if int64(len(value)) > math.MaxUint32 {
+		panic("value is too large")
+	}
+	if int64(len(value))+int64(keySize)+int64(maxNodeSize) > math.MaxUint32 {
+		panic("combined key and value size is too large")
+	}
+
+	nd, err = newRawNode(arena, height, uint32(keySize), uint32(valueSize))
+	if err != nil {
+		return
+	}
+	nd.keyTrailer = key.Trailer
+	copy(nd.getKeyBytes(arena), key.UserKey)
+	copy(nd.getValue(arena), value)
+	return
+}
+
+func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *node, err error) {
+	// Compute the amount of the tower that will never be used, since the height
+	// is less than maxHeight.
+	unusedSize := uint32((maxHeight - int(height)) * linksSize)
+	nodeSize := uint32(maxNodeSize) - unusedSize
+
+	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
+	if err != nil {
+		return
+	}
+
+	nd = (*node)(arena.getPointer(nodeOffset))
+	nd.keyOffset = nodeOffset + nodeSize
+	nd.keySize = keySize
+	nd.valueSize = valueSize
+	nd.allocSize = allocSize
+	return
+}
+
+func (n *node) getKeyBytes(arena *Arena) []byte {
+	return arena.getBytes(n.keyOffset, n.keySize)
+}
+
+func (n *node) getValue(arena *Arena) []byte {
+	return arena.getBytes(n.keyOffset+n.keySize, uint32(n.valueSize))
+}
+
+func (n *node) nextOffset(h int) uint32 {
+	return n.tower[h].nextOffset.Load()
+}
+
+func (n *node) prevOffset(h int) uint32 {
+	return n.tower[h].prevOffset.Load()
+}
+
+func (n *node) casNextOffset(h int, old, val uint32) bool {
+	return n.tower[h].nextOffset.CompareAndSwap(old, val)
+}
+
+func (n *node) casPrevOffset(h int, old, val uint32) bool {
+	return n.tower[h].prevOffset.CompareAndSwap(old, val)
+}

vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go

@@ -0,0 +1,464 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ * Modifications copyright (C) 2017 Andy Kimball and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+Adapted from RocksDB inline skiplist.
+
+Key differences:
+- No optimization for sequential inserts (no "prev").
+- No custom comparator.
+- Support overwrites. This requires care when we see the same key when inserting.
+  For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
+	there is no need for values. We don't intend to support versioning. In-place updates of values
+	would be more efficient.
+- We discard all non-concurrent code.
+- We do not support Splices. This simplifies the code a lot.
+- No AllocateNode or other pointer arithmetic.
+- We combine the findLessThan, findGreaterOrEqual, etc into one function.
+*/
+
+/*
+Further adapted from Badger: https://github.com/dgraph-io/badger.
+
+Key differences:
+- Support for previous pointers - doubly linked lists. Note that it's up to higher
+  level code to deal with the intermediate state that occurs during insertion,
+  where node A is linked to node B, but node B is not yet linked back to node A.
+- Iterator includes mutator functions.
+*/
+
+package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
+
+import (
+	"math"
+	"runtime"
+	"sync/atomic"
+	"unsafe"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/fastrand"
+)
+
+const (
+	maxHeight   = 20
+	maxNodeSize = int(unsafe.Sizeof(node{}))
+	linksSize   = int(unsafe.Sizeof(links{}))
+	pValue      = 1 / math.E
+)
+
+// ErrRecordExists indicates that an entry with the specified key already
+// exists in the skiplist. Duplicate entries are not directly supported and
+// instead must be handled by the user by appending a unique version suffix to
+// keys.
+var ErrRecordExists = errors.New("record with this key already exists")
+
+// Skiplist is a fast, concurrent skiplist implementation that supports forward
+// and backward iteration. See batchskl.Skiplist for a non-concurrent
+// skiplist. Keys and values are immutable once added to the skiplist and
+// deletion is not supported. Instead, higher-level code is expected to add new
+// entries that shadow existing entries and perform deletion via tombstones. It
+// is up to the user to process these shadow entries and tombstones
+// appropriately during retrieval.
+type Skiplist struct {
+	arena  *Arena
+	cmp    base.Compare
+	head   *node
+	tail   *node
+	height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
+
+	// If set to true by tests, then extra delays are added to make it easier to
+	// detect unusual race conditions.
+	testing bool
+}
+
+// Inserter TODO(peter)
+type Inserter struct {
+	spl    [maxHeight]splice
+	height uint32
+}
+
+// Add TODO(peter)
+func (ins *Inserter) Add(list *Skiplist, key base.InternalKey, value []byte) error {
+	return list.addInternal(key, value, ins)
+}
+
+var (
+	probabilities [maxHeight]uint32
+)
+
+func init() {
+	// Precompute the skiplist probabilities so that only a single random number
+	// needs to be generated and so that the optimal pvalue can be used (inverse
+	// of Euler's number).
+	p := float64(1.0)
+	for i := 0; i < maxHeight; i++ {
+		probabilities[i] = uint32(float64(math.MaxUint32) * p)
+		p *= pValue
+	}
+}
+
+// NewSkiplist constructs and initializes a new, empty skiplist. All nodes, keys,
+// and values in the skiplist will be allocated from the given arena.
+func NewSkiplist(arena *Arena, cmp base.Compare) *Skiplist {
+	skl := &Skiplist{}
+	skl.Reset(arena, cmp)
+	return skl
+}
+
+// Reset the skiplist to empty and re-initialize.
+func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
+	// Allocate head and tail nodes.
+	head, err := newRawNode(arena, maxHeight, 0, 0)
+	if err != nil {
+		panic("arenaSize is not large enough to hold the head node")
+	}
+	head.keyOffset = 0
+
+	tail, err := newRawNode(arena, maxHeight, 0, 0)
+	if err != nil {
+		panic("arenaSize is not large enough to hold the tail node")
+	}
+	tail.keyOffset = 0
+
+	// Link all head/tail levels together.
+	headOffset := arena.getPointerOffset(unsafe.Pointer(head))
+	tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
+	for i := 0; i < maxHeight; i++ {
+		head.tower[i].nextOffset.Store(tailOffset)
+		tail.tower[i].prevOffset.Store(headOffset)
+	}
+
+	*s = Skiplist{
+		arena: arena,
+		cmp:   cmp,
+		head:  head,
+		tail:  tail,
+	}
+	s.height.Store(1)
+}
+
+// Height returns the height of the highest tower within any of the nodes that
+// have ever been allocated as part of this skiplist.
+func (s *Skiplist) Height() uint32 { return s.height.Load() }
+
+// Arena returns the arena backing this skiplist.
+func (s *Skiplist) Arena() *Arena { return s.arena }
+
+// Size returns the number of bytes that have allocated from the arena.
+func (s *Skiplist) Size() uint32 { return s.arena.Size() }
+
+// Add adds a new key if it does not yet exist. If the key already exists, then
+// Add returns ErrRecordExists. If there isn't enough room in the arena, then
+// Add returns ErrArenaFull.
+func (s *Skiplist) Add(key base.InternalKey, value []byte) error {
+	var ins Inserter
+	return s.addInternal(key, value, &ins)
+}
+
+func (s *Skiplist) addInternal(key base.InternalKey, value []byte, ins *Inserter) error {
+	if s.findSplice(key, ins) {
+		// Found a matching node, but handle case where it's been deleted.
+		return ErrRecordExists
+	}
+
+	if s.testing {
+		// Add delay to make it easier to test race between this thread
+		// and another thread that sees the intermediate state between
+		// finding the splice and using it.
+		runtime.Gosched()
+	}
+
+	nd, height, err := s.newNode(key, value)
+	if err != nil {
+		return err
+	}
+
+	ndOffset := s.arena.getPointerOffset(unsafe.Pointer(nd))
+
+	// We always insert from the base level and up. After you add a node in base
+	// level, we cannot create a node in the level above because it would have
+	// discovered the node in the base level.
+	var found bool
+	var invalidateSplice bool
+	for i := 0; i < int(height); i++ {
+		prev := ins.spl[i].prev
+		next := ins.spl[i].next
+
+		if prev == nil {
+			// New node increased the height of the skiplist, so assume that the
+			// new level has not yet been populated.
+			if next != nil {
+				panic("next is expected to be nil, since prev is nil")
+			}
+
+			prev = s.head
+			next = s.tail
+		}
+
+		// +----------------+     +------------+     +----------------+
+		// |      prev      |     |     nd     |     |      next      |
+		// | prevNextOffset |---->|            |     |                |
+		// |                |<----| prevOffset |     |                |
+		// |                |     | nextOffset |---->|                |
+		// |                |     |            |<----| nextPrevOffset |
+		// +----------------+     +------------+     +----------------+
+		//
+		// 1. Initialize prevOffset and nextOffset to point to prev and next.
+		// 2. CAS prevNextOffset to repoint from next to nd.
+		// 3. CAS nextPrevOffset to repoint from prev to nd.
+		for {
+			prevOffset := s.arena.getPointerOffset(unsafe.Pointer(prev))
+			nextOffset := s.arena.getPointerOffset(unsafe.Pointer(next))
+			nd.tower[i].init(prevOffset, nextOffset)
+
+			// Check whether next has an updated link to prev. If it does not,
+			// that can mean one of two things:
+			//   1. The thread that added the next node hasn't yet had a chance
+			//      to add the prev link (but will shortly).
+			//   2. Another thread has added a new node between prev and next.
+			nextPrevOffset := next.prevOffset(i)
+			if nextPrevOffset != prevOffset {
+				// Determine whether #1 or #2 is true by checking whether prev
+				// is still pointing to next. As long as the atomic operations
+				// have at least acquire/release semantics (no need for
+				// sequential consistency), this works, as it is equivalent to
+				// the "publication safety" pattern.
+				prevNextOffset := prev.nextOffset(i)
+				if prevNextOffset == nextOffset {
+					// Ok, case #1 is true, so help the other thread along by
+					// updating the next node's prev link.
+					next.casPrevOffset(i, nextPrevOffset, prevOffset)
+				}
+			}
+
+			if prev.casNextOffset(i, nextOffset, ndOffset) {
+				// Managed to insert nd between prev and next, so update the next
+				// node's prev link and go to the next level.
+				if s.testing {
+					// Add delay to make it easier to test race between this thread
+					// and another thread that sees the intermediate state between
+					// setting next and setting prev.
+					runtime.Gosched()
+				}
+
+				next.casPrevOffset(i, prevOffset, ndOffset)
+				break
+			}
+
+			// CAS failed. We need to recompute prev and next. It is unlikely to
+			// be helpful to try to use a different level as we redo the search,
+			// because it is unlikely that lots of nodes are inserted between prev
+			// and next.
+			prev, next, found = s.findSpliceForLevel(key, i, prev)
+			if found {
+				if i != 0 {
+					panic("how can another thread have inserted a node at a non-base level?")
+				}
+
+				return ErrRecordExists
+			}
+			invalidateSplice = true
+		}
+	}
+
+	// If we had to recompute the splice for a level, invalidate the entire
+	// cached splice.
+	if invalidateSplice {
+		ins.height = 0
+	} else {
+		// The splice was valid. We inserted a node between spl[i].prev and
+		// spl[i].next. Optimistically update spl[i].prev for use in a subsequent
+		// call to add.
+		for i := uint32(0); i < height; i++ {
+			ins.spl[i].prev = nd
+		}
+	}
+
+	return nil
+}
+
+// NewIter returns a new Iterator object. The lower and upper bound parameters
+// control the range of keys the iterator will return. Specifying for nil for
+// lower or upper bound disables the check for that boundary. Note that lower
+// bound is not checked on {SeekGE,First} and upper bound is not check on
+// {SeekLT,Last}. The user is expected to perform that check. Note that it is
+// safe for an iterator to be copied by value.
+func (s *Skiplist) NewIter(lower, upper []byte) *Iterator {
+	it := iterPool.Get().(*Iterator)
+	*it = Iterator{list: s, nd: s.head, lower: lower, upper: upper}
+	return it
+}
+
+// NewFlushIter returns a new flushIterator, which is similar to an Iterator
+// but also sets the current number of the bytes that have been iterated
+// through.
+func (s *Skiplist) NewFlushIter(bytesFlushed *uint64) base.InternalIterator {
+	return &flushIterator{
+		Iterator:      Iterator{list: s, nd: s.head},
+		bytesIterated: bytesFlushed,
+	}
+}
+
+func (s *Skiplist) newNode(
+	key base.InternalKey, value []byte,
+) (nd *node, height uint32, err error) {
+	height = s.randomHeight()
+	nd, err = newNode(s.arena, height, key, value)
+	if err != nil {
+		return
+	}
+
+	// Try to increase s.height via CAS.
+	listHeight := s.Height()
+	for height > listHeight {
+		if s.height.CompareAndSwap(listHeight, height) {
+			// Successfully increased skiplist.height.
+			break
+		}
+
+		listHeight = s.Height()
+	}
+
+	return
+}
+
+func (s *Skiplist) randomHeight() uint32 {
+	rnd := fastrand.Uint32()
+
+	h := uint32(1)
+	for h < maxHeight && rnd <= probabilities[h] {
+		h++
+	}
+
+	return h
+}
+
+func (s *Skiplist) findSplice(key base.InternalKey, ins *Inserter) (found bool) {
+	listHeight := s.Height()
+	var level int
+
+	prev := s.head
+	if ins.height < listHeight {
+		// Our cached height is less than the list height, which means there were
+		// inserts that increased the height of the list. Recompute the splice from
+		// scratch.
+		ins.height = listHeight
+		level = int(ins.height)
+	} else {
+		// Our cached height is equal to the list height.
+		for ; level < int(listHeight); level++ {
+			spl := &ins.spl[level]
+			if s.getNext(spl.prev, level) != spl.next {
+				// One or more nodes have been inserted between the splice at this
+				// level.
+				continue
+			}
+			if spl.prev != s.head && !s.keyIsAfterNode(spl.prev, key) {
+				// Key lies before splice.
+				level = int(listHeight)
+				break
+			}
+			if spl.next != s.tail && s.keyIsAfterNode(spl.next, key) {
+				// Key lies after splice.
+				level = int(listHeight)
+				break
+			}
+			// The splice brackets the key!
+			prev = spl.prev
+			break
+		}
+	}
+
+	for level = level - 1; level >= 0; level-- {
+		var next *node
+		prev, next, found = s.findSpliceForLevel(key, level, prev)
+		if next == nil {
+			next = s.tail
+		}
+		ins.spl[level].init(prev, next)
+	}
+
+	return
+}
+
+func (s *Skiplist) findSpliceForLevel(
+	key base.InternalKey, level int, start *node,
+) (prev, next *node, found bool) {
+	prev = start
+
+	for {
+		// Assume prev.key < key.
+		next = s.getNext(prev, level)
+		if next == s.tail {
+			// Tail node, so done.
+			break
+		}
+
+		offset, size := next.keyOffset, next.keySize
+		nextKey := s.arena.buf[offset : offset+size]
+		cmp := s.cmp(key.UserKey, nextKey)
+		if cmp < 0 {
+			// We are done for this level, since prev.key < key < next.key.
+			break
+		}
+		if cmp == 0 {
+			// User-key equality.
+			if key.Trailer == next.keyTrailer {
+				// Internal key equality.
+				found = true
+				break
+			}
+			if key.Trailer > next.keyTrailer {
+				// We are done for this level, since prev.key < key < next.key.
+				break
+			}
+		}
+
+		// Keep moving right on this level.
+		prev = next
+	}
+
+	return
+}
+
+func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
+	ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
+	cmp := s.cmp(ndKey, key.UserKey)
+	if cmp < 0 {
+		return true
+	}
+	if cmp > 0 {
+		return false
+	}
+	// User-key equality.
+	if key.Trailer == nd.keyTrailer {
+		// Internal key equality.
+		return false
+	}
+	return key.Trailer < nd.keyTrailer
+}
+
+func (s *Skiplist) getNext(nd *node, h int) *node {
+	offset := nd.tower[h].nextOffset.Load()
+	return (*node)(s.arena.getPointer(offset))
+}
+
+func (s *Skiplist) getPrev(nd *node, h int) *node {
+	offset := nd.tower[h].prevOffset.Load()
+	return (*node)(s.arena.getPointer(offset))
+}

vendor/github.com/cockroachdb/pebble/internal/base/cleaner.go

@@ -0,0 +1,60 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import "github.com/cockroachdb/pebble/vfs"
+
+// Cleaner cleans obsolete files.
+type Cleaner interface {
+	Clean(fs vfs.FS, fileType FileType, path string) error
+}
+
+// NeedsFileContents is implemented by a cleaner that needs the contents of the
+// files that it is being asked to clean.
+type NeedsFileContents interface {
+	needsFileContents()
+}
+
+// DeleteCleaner deletes file.
+type DeleteCleaner struct{}
+
+// Clean removes file.
+func (DeleteCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
+	return fs.Remove(path)
+}
+
+func (DeleteCleaner) String() string {
+	return "delete"
+}
+
+// ArchiveCleaner archives file instead delete.
+type ArchiveCleaner struct{}
+
+var _ NeedsFileContents = ArchiveCleaner{}
+
+// Clean archives file.
+func (ArchiveCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
+	switch fileType {
+	case FileTypeLog, FileTypeManifest, FileTypeTable:
+		destDir := fs.PathJoin(fs.PathDir(path), "archive")
+
+		if err := fs.MkdirAll(destDir, 0755); err != nil {
+			return err
+		}
+
+		destPath := fs.PathJoin(destDir, fs.PathBase(path))
+		return fs.Rename(path, destPath)
+
+	default:
+		return fs.Remove(path)
+	}
+}
+
+func (ArchiveCleaner) String() string {
+	return "archive"
+}
+
+func (ArchiveCleaner) needsFileContents() {
+}

vendor/github.com/cockroachdb/pebble/internal/base/comparer.go

@@ -0,0 +1,260 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"strconv"
+	"unicode/utf8"
+)
+
+// Compare returns -1, 0, or +1 depending on whether a is 'less than', 'equal
+// to' or 'greater than' b. The two arguments can only be 'equal' if their
+// contents are exactly equal. Furthermore, the empty slice must be 'less than'
+// any non-empty slice. Compare is used to compare user keys, such as those
+// passed as arguments to the various DB methods, as well as those returned
+// from Separator, Successor, and Split.
+type Compare func(a, b []byte) int
+
+// Equal returns true if a and b are equivalent. For a given Compare,
+// Equal(a,b) must return true iff Compare(a,b) returns zero, that is,
+// Equal is a (potentially faster) specialization of Compare.
+type Equal func(a, b []byte) bool
+
+// AbbreviatedKey returns a fixed length prefix of a user key such that AbbreviatedKey(a)
+// < AbbreviatedKey(b) iff a < b and AbbreviatedKey(a) > AbbreviatedKey(b) iff a > b. If
+// AbbreviatedKey(a) == AbbreviatedKey(b) an additional comparison is required to
+// determine if the two keys are actually equal.
+//
+// This helps optimize indexed batch comparisons for cache locality. If a Split
+// function is specified, AbbreviatedKey usually returns the first eight bytes
+// of the user key prefix in the order that gives the correct ordering.
+type AbbreviatedKey func(key []byte) uint64
+
+// FormatKey returns a formatter for the user key.
+type FormatKey func(key []byte) fmt.Formatter
+
+// FormatValue returns a formatter for the user value. The key is also
+// specified for the value formatter in order to support value formatting that
+// is dependent on the key.
+type FormatValue func(key, value []byte) fmt.Formatter
+
+// Separator is used to construct SSTable index blocks. A trivial implementation
+// is `return a`, but appending fewer bytes leads to smaller SSTables.
+//
+// Given keys a, b for which Compare(a, b) < 0, Separator returns a key k such
+// that:
+//
+// 1. Compare(a, k) <= 0, and
+// 2. Compare(k, b) < 0.
+//
+// As a special case, b may be nil in which case the second condition is dropped.
+//
+// For example, if dst, a and b are the []byte equivalents of the strings
+// "aqua", "black" and "blue", then the result may be "aquablb".
+// Similarly, if the arguments were "aqua", "green" and "", then the result
+// may be "aquah".
+type Separator func(dst, a, b []byte) []byte
+
+// Successor returns a shortened key given a key a, such that Compare(k, a) >=
+// 0. A simple implementation may return a unchanged. The dst parameter may be
+// used to store the returned key, though it is valid to pass nil. The returned
+// key must be valid to pass to Compare.
+type Successor func(dst, a []byte) []byte
+
+// ImmediateSuccessor is invoked with a prefix key ([Split(a) == len(a)]) and
+// returns the smallest key that is larger than the given prefix a.
+// ImmediateSuccessor must return a prefix key k such that:
+//
+//	Split(k) == len(k) and Compare(k, a) > 0
+//
+// and there exists no representable k2 such that:
+//
+//	Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
+//
+// As an example, an implementation built on the natural byte ordering using
+// bytes.Compare could append a `\0` to `a`.
+//
+// The dst parameter may be used to store the returned key, though it is valid
+// to pass nil. The returned key must be valid to pass to Compare.
+type ImmediateSuccessor func(dst, a []byte) []byte
+
+// Split returns the length of the prefix of the user key that corresponds to
+// the key portion of an MVCC encoding scheme to enable the use of prefix bloom
+// filters.
+//
+// The method will only ever be called with valid MVCC keys, that is, keys that
+// the user could potentially store in the database. Pebble does not know which
+// keys are MVCC keys and which are not, and may call Split on both MVCC keys
+// and non-MVCC keys.
+//
+// A trivial MVCC scheme is one in which Split() returns len(a). This
+// corresponds to assigning a constant version to each key in the database. For
+// performance reasons, it is preferable to use a `nil` split in this case.
+//
+// The returned prefix must have the following properties:
+//
+//  1. The prefix must be a byte prefix:
+//
+//     bytes.HasPrefix(a, prefix(a))
+//
+//  2. A key consisting of just a prefix must sort before all other keys with
+//     that prefix:
+//
+//     Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
+//
+//  3. Prefixes must be used to order keys before suffixes:
+//
+//     If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
+//
+//  4. Suffixes themselves must be valid keys and comparable, respecting the same
+//     ordering as within a key.
+//
+//     If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
+type Split func(a []byte) int
+
+// Comparer defines a total ordering over the space of []byte keys: a 'less
+// than' relationship.
+type Comparer struct {
+	Compare            Compare
+	Equal              Equal
+	AbbreviatedKey     AbbreviatedKey
+	FormatKey          FormatKey
+	FormatValue        FormatValue
+	Separator          Separator
+	Split              Split
+	Successor          Successor
+	ImmediateSuccessor ImmediateSuccessor
+
+	// Name is the name of the comparer.
+	//
+	// The Level-DB on-disk format stores the comparer name, and opening a
+	// database with a different comparer from the one it was created with
+	// will result in an error.
+	Name string
+}
+
+// DefaultFormatter is the default implementation of user key formatting:
+// non-ASCII data is formatted as escaped hexadecimal values.
+var DefaultFormatter = func(key []byte) fmt.Formatter {
+	return FormatBytes(key)
+}
+
+// DefaultComparer is the default implementation of the Comparer interface.
+// It uses the natural ordering, consistent with bytes.Compare.
+var DefaultComparer = &Comparer{
+	Compare: bytes.Compare,
+	Equal:   bytes.Equal,
+
+	AbbreviatedKey: func(key []byte) uint64 {
+		if len(key) >= 8 {
+			return binary.BigEndian.Uint64(key)
+		}
+		var v uint64
+		for _, b := range key {
+			v <<= 8
+			v |= uint64(b)
+		}
+		return v << uint(8*(8-len(key)))
+	},
+
+	FormatKey: DefaultFormatter,
+
+	Separator: func(dst, a, b []byte) []byte {
+		i, n := SharedPrefixLen(a, b), len(dst)
+		dst = append(dst, a...)
+
+		min := len(a)
+		if min > len(b) {
+			min = len(b)
+		}
+		if i >= min {
+			// Do not shorten if one string is a prefix of the other.
+			return dst
+		}
+
+		if a[i] >= b[i] {
+			// b is smaller than a or a is already the shortest possible.
+			return dst
+		}
+
+		if i < len(b)-1 || a[i]+1 < b[i] {
+			i += n
+			dst[i]++
+			return dst[:i+1]
+		}
+
+		i += n + 1
+		for ; i < len(dst); i++ {
+			if dst[i] != 0xff {
+				dst[i]++
+				return dst[:i+1]
+			}
+		}
+		return dst
+	},
+
+	Successor: func(dst, a []byte) (ret []byte) {
+		for i := 0; i < len(a); i++ {
+			if a[i] != 0xff {
+				dst = append(dst, a[:i+1]...)
+				dst[len(dst)-1]++
+				return dst
+			}
+		}
+		// a is a run of 0xffs, leave it alone.
+		return append(dst, a...)
+	},
+
+	ImmediateSuccessor: func(dst, a []byte) (ret []byte) {
+		return append(append(dst, a...), 0x00)
+	},
+
+	// This name is part of the C++ Level-DB implementation's default file
+	// format, and should not be changed.
+	Name: "leveldb.BytewiseComparator",
+}
+
+// SharedPrefixLen returns the largest i such that a[:i] equals b[:i].
+// This function can be useful in implementing the Comparer interface.
+func SharedPrefixLen(a, b []byte) int {
+	i, n := 0, len(a)
+	if n > len(b) {
+		n = len(b)
+	}
+	asUint64 := func(c []byte, i int) uint64 {
+		return binary.LittleEndian.Uint64(c[i:])
+	}
+	for i < n-7 && asUint64(a, i) == asUint64(b, i) {
+		i += 8
+	}
+	for i < n && a[i] == b[i] {
+		i++
+	}
+	return i
+}
+
+// FormatBytes formats a byte slice using hexadecimal escapes for non-ASCII
+// data.
+type FormatBytes []byte
+
+const lowerhex = "0123456789abcdef"
+
+// Format implements the fmt.Formatter interface.
+func (p FormatBytes) Format(s fmt.State, c rune) {
+	buf := make([]byte, 0, len(p))
+	for _, b := range p {
+		if b < utf8.RuneSelf && strconv.IsPrint(rune(b)) {
+			buf = append(buf, b)
+			continue
+		}
+		buf = append(buf, `\x`...)
+		buf = append(buf, lowerhex[b>>4])
+		buf = append(buf, lowerhex[b&0xF])
+	}
+	s.Write(buf)
+}

vendor/github.com/cockroachdb/pebble/internal/base/error.go

@@ -0,0 +1,28 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import "github.com/cockroachdb/errors"
+
+// ErrNotFound means that a get or delete call did not find the requested key.
+var ErrNotFound = errors.New("pebble: not found")
+
+// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST,
+// sstable) isn't in the expected format.
+var ErrCorruption = errors.New("pebble: corruption")
+
+// MarkCorruptionError marks given error as a corruption error.
+func MarkCorruptionError(err error) error {
+	if errors.Is(err, ErrCorruption) {
+		return err
+	}
+	return errors.Mark(err, ErrCorruption)
+}
+
+// CorruptionErrorf formats according to a format specifier and returns
+// the string as an error value that is marked as a corruption error.
+func CorruptionErrorf(format string, args ...interface{}) error {
+	return errors.Mark(errors.Newf(format, args...), ErrCorruption)
+}

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -0,0 +1,204 @@
+// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/redact"
+)
+
+// FileNum is an internal DB identifier for a file.
+type FileNum uint64
+
+// String returns a string representation of the file number.
+func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
+
+// SafeFormat implements redact.SafeFormatter.
+func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%06d", redact.SafeUint(fn))
+}
+
+// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
+// called if the caller can ensure that the FileNum belongs to a physical file
+// on disk. These could be manifests, log files, physical sstables on disk, the
+// options file, but not virtual sstables.
+func (fn FileNum) DiskFileNum() DiskFileNum {
+	return DiskFileNum{fn}
+}
+
+// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
+// Note that a FileNum is an internal DB identifier and it could belong to files
+// which don't exist on disk. An example would be virtual sstable FileNums.
+// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
+// FileNum to DiskFileNum may not be valid and care should be taken to prove
+// that the FileNum actually exists on disk.
+type DiskFileNum struct {
+	fn FileNum
+}
+
+func (dfn DiskFileNum) String() string { return dfn.fn.String() }
+
+// SafeFormat implements redact.SafeFormatter.
+func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
+	dfn.fn.SafeFormat(w, verb)
+}
+
+// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
+func (dfn DiskFileNum) FileNum() FileNum {
+	return dfn.fn
+}
+
+// FileType enumerates the types of files found in a DB.
+type FileType int
+
+// The FileType enumeration.
+const (
+	FileTypeLog FileType = iota
+	FileTypeLock
+	FileTypeTable
+	FileTypeManifest
+	FileTypeCurrent
+	FileTypeOptions
+	FileTypeOldTemp
+	FileTypeTemp
+)
+
+// MakeFilename builds a filename from components.
+func MakeFilename(fileType FileType, dfn DiskFileNum) string {
+	switch fileType {
+	case FileTypeLog:
+		return fmt.Sprintf("%s.log", dfn)
+	case FileTypeLock:
+		return "LOCK"
+	case FileTypeTable:
+		return fmt.Sprintf("%s.sst", dfn)
+	case FileTypeManifest:
+		return fmt.Sprintf("MANIFEST-%s", dfn)
+	case FileTypeCurrent:
+		return "CURRENT"
+	case FileTypeOptions:
+		return fmt.Sprintf("OPTIONS-%s", dfn)
+	case FileTypeOldTemp:
+		return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
+	case FileTypeTemp:
+		return fmt.Sprintf("temporary.%s.dbtmp", dfn)
+	}
+	panic("unreachable")
+}
+
+// MakeFilepath builds a filepath from components.
+func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
+	return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
+}
+
+// ParseFilename parses the components from a filename.
+func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
+	filename = fs.PathBase(filename)
+	switch {
+	case filename == "CURRENT":
+		return FileTypeCurrent, DiskFileNum{0}, true
+	case filename == "LOCK":
+		return FileTypeLock, DiskFileNum{0}, true
+	case strings.HasPrefix(filename, "MANIFEST-"):
+		dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
+		if !ok {
+			break
+		}
+		return FileTypeManifest, dfn, true
+	case strings.HasPrefix(filename, "OPTIONS-"):
+		dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
+		if !ok {
+			break
+		}
+		return FileTypeOptions, dfn, ok
+	case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
+		s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
+		dfn, ok = parseDiskFileNum(s)
+		if !ok {
+			break
+		}
+		return FileTypeOldTemp, dfn, ok
+	case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
+		s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
+		dfn, ok = parseDiskFileNum(s)
+		if !ok {
+			break
+		}
+		return FileTypeTemp, dfn, ok
+	default:
+		i := strings.IndexByte(filename, '.')
+		if i < 0 {
+			break
+		}
+		dfn, ok = parseDiskFileNum(filename[:i])
+		if !ok {
+			break
+		}
+		switch filename[i+1:] {
+		case "sst":
+			return FileTypeTable, dfn, true
+		case "log":
+			return FileTypeLog, dfn, true
+		}
+	}
+	return 0, dfn, false
+}
+
+func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
+	u, err := strconv.ParseUint(s, 10, 64)
+	if err != nil {
+		return dfn, false
+	}
+	return DiskFileNum{FileNum(u)}, true
+}
+
+// A Fataler fatals a process with a message when called.
+type Fataler interface {
+	Fatalf(format string, args ...interface{})
+}
+
+// MustExist checks if err is an error indicating a file does not exist.
+// If it is, it lists the containing directory's files to annotate the error
+// with counts of the various types of files and invokes the provided fataler.
+// See cockroachdb/cockroach#56490.
+func MustExist(fs vfs.FS, filename string, fataler Fataler, err error) {
+	if err == nil || !oserror.IsNotExist(err) {
+		return
+	}
+
+	ls, lsErr := fs.List(fs.PathDir(filename))
+	if lsErr != nil {
+		// TODO(jackson): if oserror.IsNotExist(lsErr), the the data directory
+		// doesn't exist anymore. Another process likely deleted it before
+		// killing the process. We want to fatal the process, but without
+		// triggering error reporting like Sentry.
+		fataler.Fatalf("%s:\norig err: %s\nlist err: %s", redact.Safe(fs.PathBase(filename)), err, lsErr)
+	}
+	var total, unknown, tables, logs, manifests int
+	total = len(ls)
+	for _, f := range ls {
+		typ, _, ok := ParseFilename(fs, f)
+		if !ok {
+			unknown++
+			continue
+		}
+		switch typ {
+		case FileTypeTable:
+			tables++
+		case FileTypeLog:
+			logs++
+		case FileTypeManifest:
+			manifests++
+		}
+	}
+
+	fataler.Fatalf("%s:\n%s\ndirectory contains %d files, %d unknown, %d tables, %d logs, %d manifests",
+		fs.PathBase(filename), err, total, unknown, tables, logs, manifests)
+}

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -0,0 +1,506 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base // import "github.com/cockroachdb/pebble/internal/base"
+
+import (
+	"encoding/binary"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/cockroachdb/redact"
+)
+
+const (
+	// SeqNumZero is the zero sequence number, set by compactions if they can
+	// guarantee there are no keys underneath an internal key.
+	SeqNumZero = uint64(0)
+	// SeqNumStart is the first sequence number assigned to a key. Sequence
+	// numbers 1-9 are reserved for potential future use.
+	SeqNumStart = uint64(10)
+)
+
+// InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
+// value, a merged value, etc.
+type InternalKeyKind uint8
+
+// These constants are part of the file format, and should not be changed.
+const (
+	InternalKeyKindDelete  InternalKeyKind = 0
+	InternalKeyKindSet     InternalKeyKind = 1
+	InternalKeyKindMerge   InternalKeyKind = 2
+	InternalKeyKindLogData InternalKeyKind = 3
+	//InternalKeyKindColumnFamilyDeletion     InternalKeyKind = 4
+	//InternalKeyKindColumnFamilyValue        InternalKeyKind = 5
+	//InternalKeyKindColumnFamilyMerge        InternalKeyKind = 6
+
+	// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
+	// solely for compactions (to reduce write amp and space amp). Readers other
+	// than compactions should treat SINGLEDEL as equivalent to a DEL.
+	// Historically, it was simpler for readers other than compactions to treat
+	// SINGLEDEL as equivalent to DEL, but as of the introduction of
+	// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
+	// correctness.
+	InternalKeyKindSingleDelete InternalKeyKind = 7
+	//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
+	//InternalKeyKindBeginPrepareXID          InternalKeyKind = 9
+	//InternalKeyKindEndPrepareXID            InternalKeyKind = 10
+	//InternalKeyKindCommitXID                InternalKeyKind = 11
+	//InternalKeyKindRollbackXID              InternalKeyKind = 12
+	//InternalKeyKindNoop                     InternalKeyKind = 13
+	//InternalKeyKindColumnFamilyRangeDelete  InternalKeyKind = 14
+	InternalKeyKindRangeDelete InternalKeyKind = 15
+	//InternalKeyKindColumnFamilyBlobIndex    InternalKeyKind = 16
+	//InternalKeyKindBlobIndex                InternalKeyKind = 17
+
+	// InternalKeyKindSeparator is a key used for separator / successor keys
+	// written to sstable block indexes.
+	//
+	// NOTE: the RocksDB value has been repurposed. This was done to ensure that
+	// keys written to block indexes with value "17" (when 17 happened to be the
+	// max value, and InternalKeyKindMax was therefore set to 17), remain stable
+	// when new key kinds are supported in Pebble.
+	InternalKeyKindSeparator InternalKeyKind = 17
+
+	// InternalKeyKindSetWithDelete keys are SET keys that have met with a
+	// DELETE or SINGLEDEL key in a prior compaction. This key kind is
+	// specific to Pebble. See
+	// https://github.com/cockroachdb/pebble/issues/1255.
+	InternalKeyKindSetWithDelete InternalKeyKind = 18
+
+	// InternalKeyKindRangeKeyDelete removes all range keys within a key range.
+	// See the internal/rangekey package for more details.
+	InternalKeyKindRangeKeyDelete InternalKeyKind = 19
+	// InternalKeyKindRangeKeySet and InternalKeyKindRangeUnset represent
+	// keys that set and unset values associated with ranges of key
+	// space. See the internal/rangekey package for more details.
+	InternalKeyKindRangeKeyUnset InternalKeyKind = 20
+	InternalKeyKindRangeKeySet   InternalKeyKind = 21
+
+	// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
+	// the WAL entry for ingested sstables that are added to the flushable
+	// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
+	// batch, or in an sstable.
+	InternalKeyKindIngestSST InternalKeyKind = 22
+
+	// InternalKeyKindDeleteSized keys behave identically to
+	// InternalKeyKindDelete keys, except that they hold an associated uint64
+	// value indicating the (len(key)+len(value)) of the shadowed entry the
+	// tombstone is expected to delete. This value is used to inform compaction
+	// heuristics, but is not required to be accurate for correctness.
+	InternalKeyKindDeleteSized InternalKeyKind = 23
+
+	// This maximum value isn't part of the file format. Future extensions may
+	// increase this value.
+	//
+	// When constructing an internal key to pass to DB.Seek{GE,LE},
+	// internalKeyComparer sorts decreasing by kind (after sorting increasing by
+	// user key and decreasing by sequence number). Thus, use InternalKeyKindMax,
+	// which sorts 'less than or equal to' any other valid internalKeyKind, when
+	// searching for any kind of internal key formed by a certain user key and
+	// seqNum.
+	InternalKeyKindMax InternalKeyKind = 23
+
+	// Internal to the sstable format. Not exposed by any sstable iterator.
+	// Declared here to prevent definition of valid key kinds that set this bit.
+	InternalKeyKindSSTableInternalObsoleteBit  InternalKeyKind = 64
+	InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
+
+	// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
+	// zero sequence number.
+	InternalKeyZeroSeqnumMaxTrailer = uint64(255)
+
+	// A marker for an invalid key.
+	InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
+
+	// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
+	// which prevents those entries from being excluded from iteration.
+	InternalKeySeqNumBatch = uint64(1 << 55)
+
+	// InternalKeySeqNumMax is the largest valid sequence number.
+	InternalKeySeqNumMax = uint64(1<<56 - 1)
+
+	// InternalKeyRangeDeleteSentinel is the marker for a range delete sentinel
+	// key. This sequence number and kind are used for the upper stable boundary
+	// when a range deletion tombstone is the largest key in an sstable. This is
+	// necessary because sstable boundaries are inclusive, while the end key of a
+	// range deletion tombstone is exclusive.
+	InternalKeyRangeDeleteSentinel = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeDelete)
+
+	// InternalKeyBoundaryRangeKey is the marker for a range key boundary. This
+	// sequence number and kind are used during interleaved range key and point
+	// iteration to allow an iterator to stop at range key start keys where
+	// there exists no point key.
+	InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
+)
+
+// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
+const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
+
+var internalKeyKindNames = []string{
+	InternalKeyKindDelete:         "DEL",
+	InternalKeyKindSet:            "SET",
+	InternalKeyKindMerge:          "MERGE",
+	InternalKeyKindLogData:        "LOGDATA",
+	InternalKeyKindSingleDelete:   "SINGLEDEL",
+	InternalKeyKindRangeDelete:    "RANGEDEL",
+	InternalKeyKindSeparator:      "SEPARATOR",
+	InternalKeyKindSetWithDelete:  "SETWITHDEL",
+	InternalKeyKindRangeKeySet:    "RANGEKEYSET",
+	InternalKeyKindRangeKeyUnset:  "RANGEKEYUNSET",
+	InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
+	InternalKeyKindIngestSST:      "INGESTSST",
+	InternalKeyKindDeleteSized:    "DELSIZED",
+	InternalKeyKindInvalid:        "INVALID",
+}
+
+func (k InternalKeyKind) String() string {
+	if int(k) < len(internalKeyKindNames) {
+		return internalKeyKindNames[k]
+	}
+	return fmt.Sprintf("UNKNOWN:%d", k)
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Print(redact.SafeString(k.String()))
+}
+
+// InternalKey is a key used for the in-memory and on-disk partial DBs that
+// make up a pebble DB.
+//
+// It consists of the user key (as given by the code that uses package pebble)
+// followed by 8-bytes of metadata:
+//   - 1 byte for the type of internal key: delete or set,
+//   - 7 bytes for a uint56 sequence number, in little-endian format.
+type InternalKey struct {
+	UserKey []byte
+	Trailer uint64
+}
+
+// InvalidInternalKey is an invalid internal key for which Valid() will return
+// false.
+var InvalidInternalKey = MakeInternalKey(nil, 0, InternalKeyKindInvalid)
+
+// MakeInternalKey constructs an internal key from a specified user key,
+// sequence number and kind.
+func MakeInternalKey(userKey []byte, seqNum uint64, kind InternalKeyKind) InternalKey {
+	return InternalKey{
+		UserKey: userKey,
+		Trailer: (seqNum << 8) | uint64(kind),
+	}
+}
+
+// MakeTrailer constructs an internal key trailer from the specified sequence
+// number and kind.
+func MakeTrailer(seqNum uint64, kind InternalKeyKind) uint64 {
+	return (seqNum << 8) | uint64(kind)
+}
+
+// MakeSearchKey constructs an internal key that is appropriate for searching
+// for a the specified user key. The search key contain the maximal sequence
+// number and kind ensuring that it sorts before any other internal keys for
+// the same user key.
+func MakeSearchKey(userKey []byte) InternalKey {
+	return InternalKey{
+		UserKey: userKey,
+		Trailer: (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindMax),
+	}
+}
+
+// MakeRangeDeleteSentinelKey constructs an internal key that is a range
+// deletion sentinel key, used as the upper boundary for an sstable when a
+// range deletion is the largest key in an sstable.
+func MakeRangeDeleteSentinelKey(userKey []byte) InternalKey {
+	return InternalKey{
+		UserKey: userKey,
+		Trailer: InternalKeyRangeDeleteSentinel,
+	}
+}
+
+// MakeExclusiveSentinelKey constructs an internal key that is an
+// exclusive sentinel key, used as the upper boundary for an sstable
+// when a ranged key is the largest key in an sstable.
+func MakeExclusiveSentinelKey(kind InternalKeyKind, userKey []byte) InternalKey {
+	return InternalKey{
+		UserKey: userKey,
+		Trailer: (InternalKeySeqNumMax << 8) | uint64(kind),
+	}
+}
+
+var kindsMap = map[string]InternalKeyKind{
+	"DEL":           InternalKeyKindDelete,
+	"SINGLEDEL":     InternalKeyKindSingleDelete,
+	"RANGEDEL":      InternalKeyKindRangeDelete,
+	"LOGDATA":       InternalKeyKindLogData,
+	"SET":           InternalKeyKindSet,
+	"MERGE":         InternalKeyKindMerge,
+	"INVALID":       InternalKeyKindInvalid,
+	"SEPARATOR":     InternalKeyKindSeparator,
+	"SETWITHDEL":    InternalKeyKindSetWithDelete,
+	"RANGEKEYSET":   InternalKeyKindRangeKeySet,
+	"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
+	"RANGEKEYDEL":   InternalKeyKindRangeKeyDelete,
+	"INGESTSST":     InternalKeyKindIngestSST,
+	"DELSIZED":      InternalKeyKindDeleteSized,
+}
+
+// ParseInternalKey parses the string representation of an internal key. The
+// format is <user-key>.<kind>.<seq-num>. If the seq-num starts with a "b" it
+// is marked as a batch-seq-num (i.e. the InternalKeySeqNumBatch bit is set).
+func ParseInternalKey(s string) InternalKey {
+	x := strings.Split(s, ".")
+	ukey := x[0]
+	kind, ok := kindsMap[x[1]]
+	if !ok {
+		panic(fmt.Sprintf("unknown kind: %q", x[1]))
+	}
+	j := 0
+	if x[2][0] == 'b' {
+		j = 1
+	}
+	seqNum, _ := strconv.ParseUint(x[2][j:], 10, 64)
+	if x[2][0] == 'b' {
+		seqNum |= InternalKeySeqNumBatch
+	}
+	return MakeInternalKey([]byte(ukey), seqNum, kind)
+}
+
+// ParseKind parses the string representation of an internal key kind.
+func ParseKind(s string) InternalKeyKind {
+	kind, ok := kindsMap[s]
+	if !ok {
+		panic(fmt.Sprintf("unknown kind: %q", s))
+	}
+	return kind
+}
+
+// InternalTrailerLen is the number of bytes used to encode InternalKey.Trailer.
+const InternalTrailerLen = 8
+
+// DecodeInternalKey decodes an encoded internal key. See InternalKey.Encode().
+func DecodeInternalKey(encodedKey []byte) InternalKey {
+	n := len(encodedKey) - InternalTrailerLen
+	var trailer uint64
+	if n >= 0 {
+		trailer = binary.LittleEndian.Uint64(encodedKey[n:])
+		encodedKey = encodedKey[:n:n]
+	} else {
+		trailer = uint64(InternalKeyKindInvalid)
+		encodedKey = nil
+	}
+	return InternalKey{
+		UserKey: encodedKey,
+		Trailer: trailer,
+	}
+}
+
+// InternalCompare compares two internal keys using the specified comparison
+// function. For equal user keys, internal keys compare in descending sequence
+// number order. For equal user keys and sequence numbers, internal keys
+// compare in descending kind order (this may happen in practice among range
+// keys).
+func InternalCompare(userCmp Compare, a, b InternalKey) int {
+	if x := userCmp(a.UserKey, b.UserKey); x != 0 {
+		return x
+	}
+	if a.Trailer > b.Trailer {
+		return -1
+	}
+	if a.Trailer < b.Trailer {
+		return 1
+	}
+	return 0
+}
+
+// Encode encodes the receiver into the buffer. The buffer must be large enough
+// to hold the encoded data. See InternalKey.Size().
+func (k InternalKey) Encode(buf []byte) {
+	i := copy(buf, k.UserKey)
+	binary.LittleEndian.PutUint64(buf[i:], k.Trailer)
+}
+
+// EncodeTrailer returns the trailer encoded to an 8-byte array.
+func (k InternalKey) EncodeTrailer() [8]byte {
+	var buf [8]byte
+	binary.LittleEndian.PutUint64(buf[:], k.Trailer)
+	return buf
+}
+
+// Separator returns a separator key such that k <= x && x < other, where less
+// than is consistent with the Compare function. The buf parameter may be used
+// to store the returned InternalKey.UserKey, though it is valid to pass a
+// nil. See the Separator type for details on separator keys.
+func (k InternalKey) Separator(
+	cmp Compare, sep Separator, buf []byte, other InternalKey,
+) InternalKey {
+	buf = sep(buf, k.UserKey, other.UserKey)
+	if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
+		// The separator user key is physically shorter than k.UserKey (if it is
+		// longer, we'll continue to use "k"), but logically after. Tack on the max
+		// sequence number to the shortened user key. Note that we could tack on
+		// any sequence number and kind here to create a valid separator key. We
+		// use the max sequence number to match the behavior of LevelDB and
+		// RocksDB.
+		return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
+	}
+	return k
+}
+
+// Successor returns a successor key such that k <= x. A simple implementation
+// may return k unchanged. The buf parameter may be used to store the returned
+// InternalKey.UserKey, though it is valid to pass a nil.
+func (k InternalKey) Successor(cmp Compare, succ Successor, buf []byte) InternalKey {
+	buf = succ(buf, k.UserKey)
+	if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
+		// The successor user key is physically shorter that k.UserKey (if it is
+		// longer, we'll continue to use "k"), but logically after. Tack on the max
+		// sequence number to the shortened user key. Note that we could tack on
+		// any sequence number and kind here to create a valid separator key. We
+		// use the max sequence number to match the behavior of LevelDB and
+		// RocksDB.
+		return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
+	}
+	return k
+}
+
+// Size returns the encoded size of the key.
+func (k InternalKey) Size() int {
+	return len(k.UserKey) + 8
+}
+
+// SetSeqNum sets the sequence number component of the key.
+func (k *InternalKey) SetSeqNum(seqNum uint64) {
+	k.Trailer = (seqNum << 8) | (k.Trailer & 0xff)
+}
+
+// SeqNum returns the sequence number component of the key.
+func (k InternalKey) SeqNum() uint64 {
+	return k.Trailer >> 8
+}
+
+// SeqNumFromTrailer returns the sequence number component of a trailer.
+func SeqNumFromTrailer(t uint64) uint64 {
+	return t >> 8
+}
+
+// Visible returns true if the key is visible at the specified snapshot
+// sequence number.
+func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
+	return Visible(k.SeqNum(), snapshot, batchSnapshot)
+}
+
+// Visible returns true if a key with the provided sequence number is visible at
+// the specified snapshot sequence numbers.
+func Visible(seqNum uint64, snapshot, batchSnapshot uint64) bool {
+	// There are two snapshot sequence numbers, one for committed keys and one
+	// for batch keys. If a seqNum is less than `snapshot`, then seqNum
+	// corresponds to a committed key that is visible. If seqNum has its batch
+	// bit set, then seqNum corresponds to an uncommitted batch key. Its
+	// visible if its snapshot is less than batchSnapshot.
+	//
+	// There's one complication. The maximal sequence number
+	// (`InternalKeySeqNumMax`) is used across Pebble for exclusive sentinel
+	// keys and other purposes. The maximal sequence number has its batch bit
+	// set, but it can never be < `batchSnapshot`, since there is no expressible
+	// larger snapshot. We dictate that the maximal sequence number is always
+	// visible.
+	return seqNum < snapshot ||
+		((seqNum&InternalKeySeqNumBatch) != 0 && seqNum < batchSnapshot) ||
+		seqNum == InternalKeySeqNumMax
+}
+
+// SetKind sets the kind component of the key.
+func (k *InternalKey) SetKind(kind InternalKeyKind) {
+	k.Trailer = (k.Trailer &^ 0xff) | uint64(kind)
+}
+
+// Kind returns the kind component of the key.
+func (k InternalKey) Kind() InternalKeyKind {
+	return TrailerKind(k.Trailer)
+}
+
+// TrailerKind returns the key kind of the key trailer.
+func TrailerKind(trailer uint64) InternalKeyKind {
+	return InternalKeyKind(trailer & 0xff)
+}
+
+// Valid returns true if the key has a valid kind.
+func (k InternalKey) Valid() bool {
+	return k.Kind() <= InternalKeyKindMax
+}
+
+// Clone clones the storage for the UserKey component of the key.
+func (k InternalKey) Clone() InternalKey {
+	if len(k.UserKey) == 0 {
+		return k
+	}
+	return InternalKey{
+		UserKey: append([]byte(nil), k.UserKey...),
+		Trailer: k.Trailer,
+	}
+}
+
+// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
+// reusing any space already used for the current UserKey.
+func (k *InternalKey) CopyFrom(k2 InternalKey) {
+	k.UserKey = append(k.UserKey[:0], k2.UserKey...)
+	k.Trailer = k2.Trailer
+}
+
+// String returns a string representation of the key.
+func (k InternalKey) String() string {
+	return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())
+}
+
+// Pretty returns a formatter for the key.
+func (k InternalKey) Pretty(f FormatKey) fmt.Formatter {
+	return prettyInternalKey{k, f}
+}
+
+// IsExclusiveSentinel returns whether this internal key excludes point keys
+// with the same user key if used as an end boundary. See the comment on
+// InternalKeyRangeDeletionSentinel.
+func (k InternalKey) IsExclusiveSentinel() bool {
+	switch kind := k.Kind(); kind {
+	case InternalKeyKindRangeDelete:
+		return k.Trailer == InternalKeyRangeDeleteSentinel
+	case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
+		return (k.Trailer >> 8) == InternalKeySeqNumMax
+	default:
+		return false
+	}
+}
+
+type prettyInternalKey struct {
+	InternalKey
+	formatKey FormatKey
+}
+
+func (k prettyInternalKey) Format(s fmt.State, c rune) {
+	if seqNum := k.SeqNum(); seqNum == InternalKeySeqNumMax {
+		fmt.Fprintf(s, "%s#inf,%s", k.formatKey(k.UserKey), k.Kind())
+	} else {
+		fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
+	}
+}
+
+// ParsePrettyInternalKey parses the pretty string representation of an
+// internal key. The format is <user-key>#<seq-num>,<kind>.
+func ParsePrettyInternalKey(s string) InternalKey {
+	x := strings.FieldsFunc(s, func(c rune) bool { return c == '#' || c == ',' })
+	ukey := x[0]
+	kind, ok := kindsMap[x[2]]
+	if !ok {
+		panic(fmt.Sprintf("unknown kind: %q", x[2]))
+	}
+	var seqNum uint64
+	if x[1] == "max" || x[1] == "inf" {
+		seqNum = InternalKeySeqNumMax
+	} else {
+		seqNum, _ = strconv.ParseUint(x[1], 10, 64)
+	}
+	return MakeInternalKey([]byte(ukey), seqNum, kind)
+}

vendor/github.com/cockroachdb/pebble/internal/base/iterator.go

@@ -0,0 +1,444 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/redact"
+)
+
+// InternalIterator iterates over a DB's key/value pairs in key order. Unlike
+// the Iterator interface, the returned keys are InternalKeys composed of the
+// user-key, a sequence number and a key kind. In forward iteration, key/value
+// pairs for identical user-keys are returned in descending sequence order. In
+// reverse iteration, key/value pairs for identical user-keys are returned in
+// ascending sequence order.
+//
+// InternalIterators provide 5 absolute positioning methods and 2 relative
+// positioning methods. The absolute positioning methods are:
+//
+// - SeekGE
+// - SeekPrefixGE
+// - SeekLT
+// - First
+// - Last
+//
+// The relative positioning methods are:
+//
+// - Next
+// - Prev
+//
+// The relative positioning methods can be used in conjunction with any of the
+// absolute positioning methods with one exception: SeekPrefixGE does not
+// support reverse iteration via Prev. It is undefined to call relative
+// positioning methods without ever calling an absolute positioning method.
+//
+// InternalIterators can optionally implement a prefix iteration mode. This
+// mode is entered by calling SeekPrefixGE and exited by any other absolute
+// positioning method (SeekGE, SeekLT, First, Last). When in prefix iteration
+// mode, a call to Next will advance to the next key which has the same
+// "prefix" as the one supplied to SeekPrefixGE. Note that "prefix" in this
+// context is not a strict byte prefix, but defined by byte equality for the
+// result of the Comparer.Split method. An InternalIterator is not required to
+// support prefix iteration mode, and can implement SeekPrefixGE by forwarding
+// to SeekGE. When the iteration prefix is exhausted, it is not valid to call
+// Next on an internal iterator that's already returned (nil,nilv) or a key
+// beyond the prefix.
+//
+// Bounds, [lower, upper), can be set on iterators, either using the SetBounds()
+// function in the interface, or in implementation specific ways during iterator
+// creation. The forward positioning routines (SeekGE, First, and Next) only
+// check the upper bound. The reverse positioning routines (SeekLT, Last, and
+// Prev) only check the lower bound. It is up to the caller to ensure that the
+// forward positioning routines respect the lower bound and the reverse
+// positioning routines respect the upper bound (i.e. calling SeekGE instead of
+// First if there is a lower bound, and SeekLT instead of Last if there is an
+// upper bound). This imposition is done in order to elevate that enforcement to
+// the caller (generally pebble.Iterator or pebble.mergingIter) rather than
+// having it duplicated in every InternalIterator implementation.
+//
+// Additionally, the caller needs to ensure that SeekGE/SeekPrefixGE are not
+// called with a key > the upper bound, and SeekLT is not called with a key <
+// the lower bound. InternalIterator implementations are required to respect
+// the iterator bounds, never returning records outside of the bounds with one
+// exception: an iterator may generate synthetic RANGEDEL marker records. See
+// levelIter.syntheticBoundary for the sole existing example of this behavior.
+// Specifically, levelIter can return synthetic keys whose user key is equal to
+// the lower/upper bound.
+//
+// The bounds provided to an internal iterator must remain valid until a
+// subsequent call to SetBounds has returned. This requirement exists so that
+// iterator implementations may compare old and new bounds to apply low-level
+// optimizations. The pebble.Iterator satisfies this requirement by maintaining
+// two bound buffers and switching between them.
+//
+// An iterator must be closed after use, but it is not necessary to read an
+// iterator until exhaustion.
+//
+// An iterator is not goroutine-safe, but it is safe to use multiple iterators
+// concurrently, either in separate goroutines or switching between the
+// iterators in a single goroutine.
+//
+// It is also safe to use an iterator concurrently with modifying its
+// underlying DB, if that DB permits modification. However, the resultant
+// key/value pairs are not guaranteed to be a consistent snapshot of that DB
+// at a particular point in time.
+//
+// InternalIterators accumulate errors encountered during operation, exposing
+// them through the Error method. All of the absolute positioning methods
+// reset any accumulated error before positioning. Relative positioning
+// methods return without advancing if the iterator has accumulated an error.
+//
+// nilv == shorthand for LazyValue{}, which represents a nil value.
+type InternalIterator interface {
+	// SeekGE moves the iterator to the first key/value pair whose key is greater
+	// than or equal to the given key. Returns the key and value if the iterator
+	// is pointing at a valid entry, and (nil, nilv) otherwise. Note that SeekGE
+	// only checks the upper bound. It is up to the caller to ensure that key
+	// is greater than or equal to the lower bound.
+	SeekGE(key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
+
+	// SeekPrefixGE moves the iterator to the first key/value pair whose key is
+	// greater than or equal to the given key. Returns the key and value if the
+	// iterator is pointing at a valid entry, and (nil, nilv) otherwise. Note that
+	// SeekPrefixGE only checks the upper bound. It is up to the caller to ensure
+	// that key is greater than or equal to the lower bound.
+	//
+	// The prefix argument is used by some InternalIterator implementations (e.g.
+	// sstable.Reader) to avoid expensive operations. A user-defined Split
+	// function must be supplied to the Comparer for the DB. The supplied prefix
+	// will be the prefix of the given key returned by that Split function. If
+	// the iterator is able to determine that no key with the prefix exists, it
+	// can return (nil,nilv). Unlike SeekGE, this is not an indication that
+	// iteration is exhausted.
+	//
+	// Note that the iterator may return keys not matching the prefix. It is up
+	// to the caller to check if the prefix matches.
+	//
+	// Calling SeekPrefixGE places the receiver into prefix iteration mode. Once
+	// in this mode, reverse iteration may not be supported and will return an
+	// error. Note that pebble/Iterator.SeekPrefixGE has this same restriction on
+	// not supporting reverse iteration in prefix iteration mode until a
+	// different positioning routine (SeekGE, SeekLT, First or Last) switches the
+	// iterator out of prefix iteration.
+	SeekPrefixGE(prefix, key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
+
+	// SeekLT moves the iterator to the last key/value pair whose key is less
+	// than the given key. Returns the key and value if the iterator is pointing
+	// at a valid entry, and (nil, nilv) otherwise. Note that SeekLT only checks
+	// the lower bound. It is up to the caller to ensure that key is less than
+	// the upper bound.
+	SeekLT(key []byte, flags SeekLTFlags) (*InternalKey, LazyValue)
+
+	// First moves the iterator the the first key/value pair. Returns the key and
+	// value if the iterator is pointing at a valid entry, and (nil, nilv)
+	// otherwise. Note that First only checks the upper bound. It is up to the
+	// caller to ensure that First() is not called when there is a lower bound,
+	// and instead call SeekGE(lower).
+	First() (*InternalKey, LazyValue)
+
+	// Last moves the iterator the the last key/value pair. Returns the key and
+	// value if the iterator is pointing at a valid entry, and (nil, nilv)
+	// otherwise. Note that Last only checks the lower bound. It is up to the
+	// caller to ensure that Last() is not called when there is an upper bound,
+	// and instead call SeekLT(upper).
+	Last() (*InternalKey, LazyValue)
+
+	// Next moves the iterator to the next key/value pair. Returns the key and
+	// value if the iterator is pointing at a valid entry, and (nil, nilv)
+	// otherwise. Note that Next only checks the upper bound. It is up to the
+	// caller to ensure that key is greater than or equal to the lower bound.
+	//
+	// It is valid to call Next when the iterator is positioned before the first
+	// key/value pair due to either a prior call to SeekLT or Prev which returned
+	// (nil, nilv). It is not allowed to call Next when the previous call to SeekGE,
+	// SeekPrefixGE or Next returned (nil, nilv).
+	Next() (*InternalKey, LazyValue)
+
+	// NextPrefix moves the iterator to the next key/value pair with a different
+	// prefix than the key at the current iterator position. Returns the key and
+	// value if the iterator is pointing at a valid entry, and (nil, nil)
+	// otherwise. Note that NextPrefix only checks the upper bound. It is up to
+	// the caller to ensure that key is greater than or equal to the lower
+	// bound.
+	//
+	// NextPrefix is passed the immediate successor to the current prefix key. A
+	// valid implementation of NextPrefix is to call SeekGE with succKey.
+	//
+	// It is not allowed to call NextPrefix when the previous call was a reverse
+	// positioning operation or a call to a forward positioning method that
+	// returned (nil, nilv). It is also not allowed to call NextPrefix when the
+	// iterator is in prefix iteration mode.
+	NextPrefix(succKey []byte) (*InternalKey, LazyValue)
+
+	// Prev moves the iterator to the previous key/value pair. Returns the key
+	// and value if the iterator is pointing at a valid entry, and (nil, nilv)
+	// otherwise. Note that Prev only checks the lower bound. It is up to the
+	// caller to ensure that key is less than the upper bound.
+	//
+	// It is valid to call Prev when the iterator is positioned after the last
+	// key/value pair due to either a prior call to SeekGE or Next which returned
+	// (nil, nilv). It is not allowed to call Prev when the previous call to SeekLT
+	// or Prev returned (nil, nilv).
+	Prev() (*InternalKey, LazyValue)
+
+	// Error returns any accumulated error. It may not include errors returned
+	// to the client when calling LazyValue.Value().
+	Error() error
+
+	// Close closes the iterator and returns any accumulated error. Exhausting
+	// all the key/value pairs in a table is not considered to be an error.
+	// It is valid to call Close multiple times. Other methods should not be
+	// called after the iterator has been closed.
+	Close() error
+
+	// SetBounds sets the lower and upper bounds for the iterator. Note that the
+	// result of Next and Prev will be undefined until the iterator has been
+	// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
+	//
+	// The bounds provided must remain valid until a subsequent call to
+	// SetBounds has returned. This requirement exists so that iterator
+	// implementations may compare old and new bounds to apply low-level
+	// optimizations.
+	SetBounds(lower, upper []byte)
+
+	fmt.Stringer
+}
+
+// SeekGEFlags holds flags that may configure the behavior of a forward seek.
+// Not all flags are relevant to all iterators.
+type SeekGEFlags uint8
+
+const (
+	seekGEFlagTrySeekUsingNext uint8 = iota
+	seekGEFlagRelativeSeek
+	seekGEFlagBatchJustRefreshed
+)
+
+// SeekGEFlagsNone is the default value of SeekGEFlags, with all flags disabled.
+const SeekGEFlagsNone = SeekGEFlags(0)
+
+// TrySeekUsingNext indicates whether a performance optimization was enabled
+// by a caller, indicating the caller has not done any action to move this
+// iterator beyond the first key that would be found if this iterator were to
+// honestly do the intended seek. For example, say the caller did a
+// SeekGE(k1...), followed by SeekGE(k2...) where k1 <= k2, without any
+// intermediate positioning calls. The caller can safely specify true for this
+// parameter in the second call. As another example, say the caller did do one
+// call to Next between the two Seek calls, and k1 < k2. Again, the caller can
+// safely specify a true value for this parameter. Note that a false value is
+// always safe. The callee is free to ignore the true value if its
+// implementation does not permit this optimization.
+//
+// We make the caller do this determination since a string comparison of k1, k2
+// is not necessarily cheap, and there may be many iterators in the iterator
+// stack. Doing it once at the root of the iterator stack is cheaper.
+//
+// This optimization could also be applied to SeekLT (where it would be
+// trySeekUsingPrev). We currently only do it for SeekPrefixGE and SeekGE
+// because this is where this optimization helps the performance of CockroachDB.
+// The SeekLT cases in CockroachDB are typically accompanied with bounds that
+// change between seek calls, and is optimized inside certain iterator
+// implementations, like singleLevelIterator, without any extra parameter
+// passing (though the same amortization of string comparisons could be done to
+// improve that optimization, by making the root of the iterator stack do it).
+func (s SeekGEFlags) TrySeekUsingNext() bool { return (s & (1 << seekGEFlagTrySeekUsingNext)) != 0 }
+
+// RelativeSeek is set when in the course of a forward positioning operation, a
+// higher-level iterator seeks a lower-level iterator to a larger key than the
+// one at the current iterator position.
+//
+// Concretely, this occurs when the merging iterator observes a range deletion
+// covering the key at a level's current position, and the merging iterator
+// seeks the level to the range deletion's end key. During lazy-combined
+// iteration, this flag signals to the level iterator that the seek is NOT an
+// absolute-positioning operation from the perspective of the pebble.Iterator,
+// and the level iterator must look for range keys in tables between the current
+// iterator position and the new seeked position.
+func (s SeekGEFlags) RelativeSeek() bool { return (s & (1 << seekGEFlagRelativeSeek)) != 0 }
+
+// BatchJustRefreshed is set by Seek[Prefix]GE when an iterator's view of an
+// indexed batch was just refreshed. It serves as a signal to the batch iterator
+// to ignore the TrySeekUsingNext optimization, because the external knowledge
+// imparted by the TrySeekUsingNext flag does not apply to the batch iterator's
+// position. See (pebble.Iterator).batchJustRefreshed.
+func (s SeekGEFlags) BatchJustRefreshed() bool { return (s & (1 << seekGEFlagBatchJustRefreshed)) != 0 }
+
+// EnableTrySeekUsingNext returns the provided flags with the
+// try-seek-using-next optimization enabled. See TrySeekUsingNext for an
+// explanation of this optimization.
+func (s SeekGEFlags) EnableTrySeekUsingNext() SeekGEFlags {
+	return s | (1 << seekGEFlagTrySeekUsingNext)
+}
+
+// DisableTrySeekUsingNext returns the provided flags with the
+// try-seek-using-next optimization disabled.
+func (s SeekGEFlags) DisableTrySeekUsingNext() SeekGEFlags {
+	return s &^ (1 << seekGEFlagTrySeekUsingNext)
+}
+
+// EnableRelativeSeek returns the provided flags with the relative-seek flag
+// enabled. See RelativeSeek for an explanation of this flag's use.
+func (s SeekGEFlags) EnableRelativeSeek() SeekGEFlags {
+	return s | (1 << seekGEFlagRelativeSeek)
+}
+
+// DisableRelativeSeek returns the provided flags with the relative-seek flag
+// disabled.
+func (s SeekGEFlags) DisableRelativeSeek() SeekGEFlags {
+	return s &^ (1 << seekGEFlagRelativeSeek)
+}
+
+// EnableBatchJustRefreshed returns the provided flags with the
+// batch-just-refreshed bit set. See BatchJustRefreshed for an explanation of
+// this flag.
+func (s SeekGEFlags) EnableBatchJustRefreshed() SeekGEFlags {
+	return s | (1 << seekGEFlagBatchJustRefreshed)
+}
+
+// DisableBatchJustRefreshed returns the provided flags with the
+// batch-just-refreshed bit unset.
+func (s SeekGEFlags) DisableBatchJustRefreshed() SeekGEFlags {
+	return s &^ (1 << seekGEFlagBatchJustRefreshed)
+}
+
+// SeekLTFlags holds flags that may configure the behavior of a reverse seek.
+// Not all flags are relevant to all iterators.
+type SeekLTFlags uint8
+
+const (
+	seekLTFlagRelativeSeek uint8 = iota
+)
+
+// SeekLTFlagsNone is the default value of SeekLTFlags, with all flags disabled.
+const SeekLTFlagsNone = SeekLTFlags(0)
+
+// RelativeSeek is set when in the course of a reverse positioning operation, a
+// higher-level iterator seeks a lower-level iterator to a smaller key than the
+// one at the current iterator position.
+//
+// Concretely, this occurs when the merging iterator observes a range deletion
+// covering the key at a level's current position, and the merging iterator
+// seeks the level to the range deletion's start key. During lazy-combined
+// iteration, this flag signals to the level iterator that the seek is NOT an
+// absolute-positioning operation from the perspective of the pebble.Iterator,
+// and the level iterator must look for range keys in tables between the current
+// iterator position and the new seeked position.
+func (s SeekLTFlags) RelativeSeek() bool { return s&(1<<seekLTFlagRelativeSeek) != 0 }
+
+// EnableRelativeSeek returns the provided flags with the relative-seek flag
+// enabled. See RelativeSeek for an explanation of this flag's use.
+func (s SeekLTFlags) EnableRelativeSeek() SeekLTFlags {
+	return s | (1 << seekLTFlagRelativeSeek)
+}
+
+// DisableRelativeSeek returns the provided flags with the relative-seek flag
+// disabled.
+func (s SeekLTFlags) DisableRelativeSeek() SeekLTFlags {
+	return s &^ (1 << seekLTFlagRelativeSeek)
+}
+
+// InternalIteratorStats contains miscellaneous stats produced by
+// InternalIterators that are part of the InternalIterator tree. Not every
+// field is relevant for an InternalIterator implementation. The field values
+// are aggregated as one goes up the InternalIterator tree.
+type InternalIteratorStats struct {
+	// Bytes in the loaded blocks. If the block was compressed, this is the
+	// compressed bytes. Currently, only the index blocks, data blocks
+	// containing points, and filter blocks are included.
+	BlockBytes uint64
+	// Subset of BlockBytes that were in the block cache.
+	BlockBytesInCache uint64
+	// BlockReadDuration accumulates the duration spent fetching blocks
+	// due to block cache misses.
+	// TODO(sumeer): this currently excludes the time spent in Reader creation,
+	// and in reading the rangedel and rangekey blocks. Fix that.
+	BlockReadDuration time.Duration
+	// The following can repeatedly count the same points if they are iterated
+	// over multiple times. Additionally, they may count a point twice when
+	// switching directions. The latter could be improved if needed.
+
+	// Bytes in keys that were iterated over. Currently, only point keys are
+	// included.
+	KeyBytes uint64
+	// Bytes in values that were iterated over. Currently, only point values are
+	// included. For separated values, this is the size of the handle.
+	ValueBytes uint64
+	// The count of points iterated over.
+	PointCount uint64
+	// Points that were iterated over that were covered by range tombstones. It
+	// can be useful for discovering instances of
+	// https://github.com/cockroachdb/pebble/issues/1070.
+	PointsCoveredByRangeTombstones uint64
+
+	// Stats related to points in value blocks encountered during iteration.
+	// These are useful to understand outliers, since typical user facing
+	// iteration should tend to only look at the latest point, and hence have
+	// the following stats close to 0.
+	SeparatedPointValue struct {
+		// Count is a count of points that were in value blocks. This is not a
+		// subset of PointCount: PointCount is produced by mergingIter and if
+		// positioned once, and successful in returning a point, will have a
+		// PointCount of 1, regardless of how many sstables (and memtables etc.)
+		// in the heap got positioned. The count here includes every sstable
+		// iterator that got positioned in the heap.
+		Count uint64
+		// ValueBytes represent the total byte length of the values (in value
+		// blocks) of the points corresponding to Count.
+		ValueBytes uint64
+		// ValueBytesFetched is the total byte length of the values (in value
+		// blocks) that were retrieved.
+		ValueBytesFetched uint64
+	}
+}
+
+// Merge merges the stats in from into the given stats.
+func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
+	s.BlockBytes += from.BlockBytes
+	s.BlockBytesInCache += from.BlockBytesInCache
+	s.BlockReadDuration += from.BlockReadDuration
+	s.KeyBytes += from.KeyBytes
+	s.ValueBytes += from.ValueBytes
+	s.PointCount += from.PointCount
+	s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones
+	s.SeparatedPointValue.Count += from.SeparatedPointValue.Count
+	s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
+	s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
+}
+
+func (s *InternalIteratorStats) String() string {
+	return redact.StringWithoutMarkers(s)
+}
+
+// SafeFormat implements the redact.SafeFormatter interface.
+func (s *InternalIteratorStats) SafeFormat(p redact.SafePrinter, verb rune) {
+	p.Printf("blocks: %s cached",
+		humanize.Bytes.Uint64(s.BlockBytesInCache),
+	)
+	if s.BlockBytes != s.BlockBytesInCache || s.BlockReadDuration != 0 {
+		p.Printf(", %s not cached (read time: %s)",
+			humanize.Bytes.Uint64(s.BlockBytes-s.BlockBytesInCache),
+			humanize.FormattedString(s.BlockReadDuration.String()),
+		)
+	}
+	p.Printf("; points: %s", humanize.Count.Uint64(s.PointCount))
+
+	if s.PointsCoveredByRangeTombstones != 0 {
+		p.Printf("(%s tombstoned)", humanize.Count.Uint64(s.PointsCoveredByRangeTombstones))
+	}
+	p.Printf(" (%s keys, %s values)",
+		humanize.Bytes.Uint64(s.KeyBytes),
+		humanize.Bytes.Uint64(s.ValueBytes),
+	)
+	if s.SeparatedPointValue.Count != 0 {
+		p.Printf("; separated: %s (%s, %s fetched)",
+			humanize.Count.Uint64(s.SeparatedPointValue.Count),
+			humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytes),
+			humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytesFetched))
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go

@@ -0,0 +1,287 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import "github.com/cockroachdb/pebble/internal/invariants"
+
+// A value can have user-defined attributes that are a function of the value
+// byte slice. For now, we only support "short attributes", which can be
+// encoded in 3 bits. We will likely extend this to "long attributes" later
+// for values that are even more expensive to access than those in value
+// blocks in the same sstable.
+//
+// When a sstable writer chooses not to store a value together with the key,
+// it can call the ShortAttributeExtractor to extract the attribute and store
+// it together with the key. This allows for cheap retrieval of
+// AttributeAndLen on the read-path, without doing a more expensive retrieval
+// of the value. In general, the extraction code may want to also look at the
+// key to decide how to treat the value, hence the key* parameters.
+//
+// Write path performance: The ShortAttributeExtractor func cannot be inlined,
+// so we will pay the cost of this function call. However, we will only pay
+// this when (a) the value is not being stored together with the key, and (b)
+// the key-value pair is being initially written to the DB, or a compaction is
+// transitioning the key-value pair from being stored together to being stored
+// separately.
+
+// ShortAttribute encodes a user-specified attribute of the value.
+type ShortAttribute uint8
+
+// MaxShortAttribute is the maximum value of the short attribute (3 bits).
+const MaxShortAttribute = 7
+
+// ShortAttributeExtractor is an extractor that given the value, will return
+// the ShortAttribute.
+type ShortAttributeExtractor func(
+	key []byte, keyPrefixLen int, value []byte) (ShortAttribute, error)
+
+// AttributeAndLen represents the pair of value length and the short
+// attribute.
+type AttributeAndLen struct {
+	ValueLen       int32
+	ShortAttribute ShortAttribute
+}
+
+// LazyValue represents a value that may not already have been extracted.
+// Currently, it can represent either an in-place value (stored with the key)
+// or a value stored in the value section. However, the interface is general
+// enough to support values that are stored in separate files.
+//
+// LazyValue is used in the InternalIterator interface, such that all
+// positioning calls return (*InternalKey, LazyValue). It is also exposed via
+// the public Iterator for callers that need to remember a recent but not
+// necessarily latest LazyValue, in case they need the actual value in the
+// future. An example is a caller that is iterating in reverse and looking for
+// the latest MVCC version for a key -- it cannot identify the latest MVCC
+// version without stepping to the previous key-value pair e.g.
+// storage.pebbleMVCCScanner in CockroachDB.
+//
+// Performance note: It is important for this struct to not exceed a sizeof 32
+// bytes, for optimizing the common case of the in-place value. Prior to
+// introducing LazyValue, we were passing around a []byte which is 24 bytes.
+// Passing a 40 byte or larger struct causes performance to drop by 75% on
+// some benchmarks that do tight iteration loops.
+//
+// Memory management:
+// This is subtle, but important for performance.
+//
+// A LazyValue returned by an InternalIterator or Iterator is unstable in that
+// repositioning the iterator will invalidate the memory inside it. A caller
+// wishing to maintain that LazyValue needs to call LazyValue.Clone(). Note
+// that this does not fetch the value if it is not in-place. Clone() should
+// ideally not be called if LazyValue.Value() has been called, since the
+// cloned LazyValue will forget the extracted/fetched value, and calling
+// Value() on this clone will cause the value to be extracted again. That is,
+// Clone() does not make any promise about the memory stability of the
+// underlying value.
+//
+// A user of an iterator that calls LazyValue.Value() wants as much as
+// possible for the returned value []byte to point to iterator owned memory.
+//
+//  1. [P1] The underlying iterator that owns that memory also needs a promise
+//     from that user that at any time there is at most one value []byte slice
+//     that the caller is expecting it to maintain. Otherwise, the underlying
+//     iterator has to maintain multiple such []byte slices which results in
+//     more complicated and inefficient code.
+//
+//  2. [P2] The underlying iterator, in order to make the promise that it is
+//     maintaining the one value []byte slice, also needs a way to know when
+//     it is relieved of that promise. One way it is relieved of that promise
+//     is by being told that it is being repositioned. Typically, the owner of
+//     the value []byte slice is a sstable iterator, and it will know that it
+//     is relieved of the promise when it is repositioned. However, consider
+//     the case where the caller has used LazyValue.Clone() and repositioned
+//     the iterator (which is actually a tree of iterators). In this case the
+//     underlying sstable iterator may not even be open. LazyValue.Value()
+//     will still work (at a higher cost), but since the sstable iterator is
+//     not open, it does not have a mechanism to know when the retrieved value
+//     is no longer in use. We refer to this situation as "not satisfying P2".
+//     To handle this situation, the LazyValue.Value() method accepts a caller
+//     owned buffer, that the callee will use if needed. The callee explicitly
+//     tells the caller whether the []byte slice for the value is now owned by
+//     the caller. This will be true if the callee attempted to use buf and
+//     either successfully used it or allocated a new []byte slice.
+//
+// To ground the above in reality, we consider three examples of callers of
+// LazyValue.Value():
+//
+//   - Iterator: it calls LazyValue.Value for its own use when merging values.
+//     When merging during reverse iteration, it may have cloned the LazyValue.
+//     In this case it calls LazyValue.Value() on the cloned value, merges it,
+//     and then calls LazyValue.Value() on the current iterator position and
+//     merges it. So it is honoring P1.
+//
+//   - Iterator on behalf of Iterator clients: The Iterator.Value() method
+//     needs to call LazyValue.Value(). The client of Iterator is satisfying P1
+//     because of the inherent Iterator interface constraint, i.e., it is calling
+//     Iterator.Value() on the current Iterator position. It is possible that
+//     the Iterator has cloned this LazyValue (for the reverse iteration case),
+//     which the client is unaware of, so the underlying sstable iterator may
+//     not be able to satisfy P2. This is ok because Iterator will call
+//     LazyValue.Value with its (reusable) owned buffer.
+//
+//   - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
+//     since during reverse iteration in order to find the highest version that
+//     satisfies a read it needs to clone the LazyValue, step back the iterator
+//     and then decide whether it needs the value from the previously cloned
+//     LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
+//     similar to the previous case in that it will call LazyValue.Value with
+//     its (reusable) owned buffer.
+//
+// Corollary: callers that directly use InternalIterator can know that they
+// have done nothing to interfere with promise P2 can pass in a nil buf and be
+// sure that it will not trigger an allocation.
+//
+// Repeated calling of LazyValue.Value:
+// This is ok as long as the caller continues to satisfy P1. The previously
+// fetched value will be remembered inside LazyValue to avoid fetching again.
+// So if the caller's buffer is used the first time the value was fetched, it
+// is still in use.
+//
+// LazyValue fields are visible outside the package for use in
+// InternalIterator implementations and in Iterator, but not meant for direct
+// use by users of Pebble.
+type LazyValue struct {
+	// ValueOrHandle represents a value, or a handle to be passed to ValueFetcher.
+	// - Fetcher == nil: ValueOrHandle is a value.
+	// - Fetcher != nil: ValueOrHandle is a handle and Fetcher.Attribute is
+	//   initialized.
+	// The ValueOrHandle exposed by InternalIterator or Iterator may not be stable
+	// if the iterator is stepped. To make it stable, make a copy using Clone.
+	ValueOrHandle []byte
+	// Fetcher provides support for fetching an actually lazy value.
+	Fetcher *LazyFetcher
+}
+
+// LazyFetcher supports fetching a lazy value.
+//
+// Fetcher and Attribute are to be initialized at creation time. The fields
+// are arranged to reduce the sizeof this struct.
+type LazyFetcher struct {
+	// Fetcher, given a handle, returns the value.
+	Fetcher ValueFetcher
+	err     error
+	value   []byte
+	// Attribute includes the short attribute and value length.
+	Attribute   AttributeAndLen
+	fetched     bool
+	callerOwned bool
+}
+
+// ValueFetcher is an interface for fetching a value.
+type ValueFetcher interface {
+	// Fetch returns the value, given the handle. It is acceptable to call the
+	// ValueFetcher.Fetch as long as the DB is open. However, one should assume
+	// there is a fast-path when the iterator tree has not moved off the sstable
+	// iterator that initially provided this LazyValue. Hence, to utilize this
+	// fast-path the caller should try to decide whether it needs the value or
+	// not as soon as possible, with minimal possible stepping of the iterator.
+	//
+	// buf will be used if the fetcher cannot satisfy P2 (see earlier comment).
+	// If the fetcher attempted to use buf *and* len(buf) was insufficient, it
+	// will allocate a new slice for the value. In either case it will set
+	// callerOwned to true.
+	Fetch(
+		handle []byte, valLen int32, buf []byte) (val []byte, callerOwned bool, err error)
+}
+
+// Value returns the underlying value.
+func (lv *LazyValue) Value(buf []byte) (val []byte, callerOwned bool, err error) {
+	if lv.Fetcher == nil {
+		return lv.ValueOrHandle, false, nil
+	}
+	// Do the rest of the work in a separate method to attempt mid-stack
+	// inlining of Value(). Unfortunately, this still does not inline since the
+	// cost of 85 exceeds the budget of 80.
+	//
+	// TODO(sumeer): Packing the return values into a struct{[]byte error bool}
+	// causes it to be below the budget. Consider this if we need to recover
+	// more performance. I suspect that inlining this only matters in
+	// micro-benchmarks, and in actual use cases in CockroachDB it will not
+	// matter because there is substantial work done with a fetched value.
+	return lv.fetchValue(buf)
+}
+
+// INVARIANT: lv.Fetcher != nil
+func (lv *LazyValue) fetchValue(buf []byte) (val []byte, callerOwned bool, err error) {
+	f := lv.Fetcher
+	if !f.fetched {
+		f.fetched = true
+		f.value, f.callerOwned, f.err = f.Fetcher.Fetch(
+			lv.ValueOrHandle, lv.Fetcher.Attribute.ValueLen, buf)
+	}
+	return f.value, f.callerOwned, f.err
+}
+
+// InPlaceValue returns the value under the assumption that it is in-place.
+// This is for Pebble-internal code.
+func (lv *LazyValue) InPlaceValue() []byte {
+	if invariants.Enabled && lv.Fetcher != nil {
+		panic("value must be in-place")
+	}
+	return lv.ValueOrHandle
+}
+
+// Len returns the length of the value.
+func (lv *LazyValue) Len() int {
+	if lv.Fetcher == nil {
+		return len(lv.ValueOrHandle)
+	}
+	return int(lv.Fetcher.Attribute.ValueLen)
+}
+
+// TryGetShortAttribute returns the ShortAttribute and a bool indicating
+// whether the ShortAttribute was populated.
+func (lv *LazyValue) TryGetShortAttribute() (ShortAttribute, bool) {
+	if lv.Fetcher == nil {
+		return 0, false
+	}
+	return lv.Fetcher.Attribute.ShortAttribute, true
+}
+
+// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
+// The fetcher parameter must be non-nil and may be over-written and used
+// inside the returned LazyValue -- this is needed to avoid an allocation.
+// Most callers have at most K cloned LazyValues, where K is hard-coded, so
+// they can have a pool of exactly K LazyFetcher structs they can reuse in
+// these calls. The alternative of allocating LazyFetchers from a sync.Pool is
+// not viable since we have no code trigger for returning to the pool
+// (LazyValues are simply GC'd).
+//
+// NB: It is highly preferable that LazyValue.Value() has not been called,
+// since the Clone will forget any previously extracted value, and a future
+// call to Value will cause it to be fetched again. We do this since we don't
+// want to reason about whether or not to clone an already extracted value
+// inside the Fetcher (we don't). Property P1 applies here too: if lv1.Value()
+// has been called, and then lv2 is created as a clone of lv1, then calling
+// lv2.Value() can invalidate any backing memory maintained inside the fetcher
+// for lv1 (even though these are the same values). We initially prohibited
+// calling LazyValue.Clone() if LazyValue.Value() has been called, but there
+// is at least one complex caller (pebbleMVCCScanner inside CockroachDB) where
+// it is not easy to prove this invariant.
+func (lv *LazyValue) Clone(buf []byte, fetcher *LazyFetcher) (LazyValue, []byte) {
+	var lvCopy LazyValue
+	if lv.Fetcher != nil {
+		*fetcher = LazyFetcher{
+			Fetcher:   lv.Fetcher.Fetcher,
+			Attribute: lv.Fetcher.Attribute,
+			// Not copying anything that has been extracted.
+		}
+		lvCopy.Fetcher = fetcher
+	}
+	vLen := len(lv.ValueOrHandle)
+	if vLen == 0 {
+		return lvCopy, buf
+	}
+	bufLen := len(buf)
+	buf = append(buf, lv.ValueOrHandle...)
+	lvCopy.ValueOrHandle = buf[bufLen : bufLen+vLen]
+	return lvCopy, buf
+}
+
+// MakeInPlaceValue constructs an in-place value.
+func MakeInPlaceValue(val []byte) LazyValue {
+	return LazyValue{ValueOrHandle: val}
+}

vendor/github.com/cockroachdb/pebble/internal/base/logger.go

@@ -0,0 +1,144 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"sync"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// Logger defines an interface for writing log messages.
+type Logger interface {
+	Infof(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+}
+type defaultLogger struct{}
+
+// DefaultLogger logs to the Go stdlib logs.
+var DefaultLogger defaultLogger
+
+var _ Logger = DefaultLogger
+
+// Infof implements the Logger.Infof interface.
+func (defaultLogger) Infof(format string, args ...interface{}) {
+	_ = log.Output(2, fmt.Sprintf(format, args...))
+}
+
+// Fatalf implements the Logger.Fatalf interface.
+func (defaultLogger) Fatalf(format string, args ...interface{}) {
+	_ = log.Output(2, fmt.Sprintf(format, args...))
+	os.Exit(1)
+}
+
+// InMemLogger implements Logger using an in-memory buffer (used for testing).
+// The buffer can be read via String() and cleared via Reset().
+type InMemLogger struct {
+	mu struct {
+		sync.Mutex
+		buf bytes.Buffer
+	}
+}
+
+var _ Logger = (*InMemLogger)(nil)
+
+// Reset clears the internal buffer.
+func (b *InMemLogger) Reset() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.mu.buf.Reset()
+}
+
+// String returns the current internal buffer.
+func (b *InMemLogger) String() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.mu.buf.String()
+}
+
+// Infof is part of the Logger interface.
+func (b *InMemLogger) Infof(format string, args ...interface{}) {
+	s := fmt.Sprintf(format, args...)
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.mu.buf.Write([]byte(s))
+	if n := len(s); n == 0 || s[n-1] != '\n' {
+		b.mu.buf.Write([]byte("\n"))
+	}
+}
+
+// Fatalf is part of the Logger interface.
+func (b *InMemLogger) Fatalf(format string, args ...interface{}) {
+	b.Infof(format, args...)
+	runtime.Goexit()
+}
+
+// LoggerAndTracer defines an interface for logging and tracing.
+type LoggerAndTracer interface {
+	Logger
+	// Eventf formats and emits a tracing log, if tracing is enabled in the
+	// current context.
+	Eventf(ctx context.Context, format string, args ...interface{})
+	// IsTracingEnabled returns true if tracing is enabled. It can be used as an
+	// optimization to avoid calling Eventf (which will be a noop when tracing
+	// is not enabled) to avoid the overhead of boxing the args.
+	IsTracingEnabled(ctx context.Context) bool
+}
+
+// LoggerWithNoopTracer wraps a logger and does no tracing.
+type LoggerWithNoopTracer struct {
+	Logger
+}
+
+var _ LoggerAndTracer = &LoggerWithNoopTracer{}
+
+// Eventf implements LoggerAndTracer.
+func (*LoggerWithNoopTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
+	if invariants.Enabled && ctx == nil {
+		panic("Eventf context is nil")
+	}
+}
+
+// IsTracingEnabled implements LoggerAndTracer.
+func (*LoggerWithNoopTracer) IsTracingEnabled(ctx context.Context) bool {
+	if invariants.Enabled && ctx == nil {
+		panic("IsTracingEnabled ctx is nil")
+	}
+	return false
+}
+
+// NoopLoggerAndTracer does no logging and tracing. Remember that struct{} is
+// special cased in Go and does not incur an allocation when it backs the
+// interface LoggerAndTracer.
+type NoopLoggerAndTracer struct{}
+
+var _ LoggerAndTracer = NoopLoggerAndTracer{}
+
+// Infof implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) Infof(format string, args ...interface{}) {}
+
+// Fatalf implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) Fatalf(format string, args ...interface{}) {}
+
+// Eventf implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
+	if invariants.Enabled && ctx == nil {
+		panic("Eventf context is nil")
+	}
+}
+
+// IsTracingEnabled implements LoggerAndTracer.
+func (l NoopLoggerAndTracer) IsTracingEnabled(ctx context.Context) bool {
+	if invariants.Enabled && ctx == nil {
+		panic("IsTracingEnabled ctx is nil")
+	}
+	return false
+}

vendor/github.com/cockroachdb/pebble/internal/base/merger.go

@@ -0,0 +1,133 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import "io"
+
+// Merge creates a ValueMerger for the specified key initialized with the value
+// of one merge operand.
+type Merge func(key, value []byte) (ValueMerger, error)
+
+// ValueMerger receives merge operands one by one. The operand received is either
+// newer or older than all operands received so far as indicated by the function
+// names, `MergeNewer()` and `MergeOlder()`. Once all operands have been received,
+// the client will invoke `Finish()` to obtain the final result. The order of
+// a merge is not changed after the first call to `MergeNewer()` or
+// `MergeOlder()`, i.e. the same method is used to submit all operands.
+//
+// The implementation may choose to merge values into the result immediately upon
+// receiving each operand, or buffer operands until Finish() is called. For example,
+// buffering may be useful to avoid (de)serializing partial merge results.
+//
+// The merge operation must be associative. That is, for the values A, B, C:
+//
+//	Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
+//
+// Examples of merge operators are integer addition, list append, and string
+// concatenation.
+type ValueMerger interface {
+	// MergeNewer adds an operand that is newer than all existing operands.
+	// The caller retains ownership of value.
+	//
+	// If an error is returned the merge is aborted and no other methods must
+	// be called.
+	MergeNewer(value []byte) error
+
+	// MergeOlder adds an operand that is older than all existing operands.
+	// The caller retains ownership of value.
+	//
+	// If an error is returned the merge is aborted and no other methods must
+	// be called.
+	MergeOlder(value []byte) error
+
+	// Finish does any final processing of the added operands and returns a
+	// result. The caller can assume the returned byte slice will not be mutated.
+	//
+	// Finish must be the last function called on the ValueMerger. The caller
+	// must not call any other ValueMerger functions after calling Finish.
+	//
+	// If `includesBase` is true, the oldest merge operand was part of the
+	// merge. This will always be the true during normal iteration, but may be
+	// false during compaction when only a subset of operands may be
+	// available. Note that `includesBase` is set to true conservatively: a false
+	// value means that we could not definitely determine that the base merge
+	// operand was included.
+	//
+	// If a Closer is returned, the returned slice will remain valid until it is
+	// closed. The caller must arrange for the closer to be eventually closed.
+	Finish(includesBase bool) ([]byte, io.Closer, error)
+}
+
+// DeletableValueMerger is an extension to ValueMerger which allows indicating that the
+// result of a merge operation is non-existent. Such non-existent entries will eventually
+// be deleted during compaction. Note that during compaction, non-existence of the result
+// of a merge means that the merge operands will not result in any record being output.
+// This is not the same as transforming the merge operands into a deletion tombstone, as
+// older merge operands will still be visible during iteration. Deletion of the merge operands
+// in this way is akin to the way a SingleDelete+Set combine into non-existence while leaving
+// older records for the same key unaffected.
+type DeletableValueMerger interface {
+	ValueMerger
+
+	// DeletableFinish enables a value merger to indicate that the result of a merge operation
+	// is non-existent. See Finish for a description of includesBase.
+	DeletableFinish(includesBase bool) (value []byte, delete bool, closer io.Closer, err error)
+}
+
+// Merger defines an associative merge operation. The merge operation merges
+// two or more values for a single key. A merge operation is requested by
+// writing a value using {Batch,DB}.Merge(). The value at that key is merged
+// with any existing value. It is valid to Set a value at a key and then Merge
+// a new value. Similar to non-merged values, a merged value can be deleted by
+// either Delete or DeleteRange.
+//
+// The merge operation is invoked when a merge value is encountered during a
+// read, either during a compaction or during iteration.
+type Merger struct {
+	Merge Merge
+
+	// Name is the name of the merger.
+	//
+	// Pebble stores the merger name on disk, and opening a database with a
+	// different merger from the one it was created with will result in an error.
+	Name string
+}
+
+// AppendValueMerger concatenates merge operands in order from oldest to newest.
+type AppendValueMerger struct {
+	buf []byte
+}
+
+// MergeNewer appends value to the result.
+func (a *AppendValueMerger) MergeNewer(value []byte) error {
+	a.buf = append(a.buf, value...)
+	return nil
+}
+
+// MergeOlder prepends value to the result, which involves allocating a new buffer.
+func (a *AppendValueMerger) MergeOlder(value []byte) error {
+	buf := make([]byte, len(a.buf)+len(value))
+	copy(buf, value)
+	copy(buf[len(value):], a.buf)
+	a.buf = buf
+	return nil
+}
+
+// Finish returns the buffer that was constructed on-demand in `Merge{OlderNewer}()` calls.
+func (a *AppendValueMerger) Finish(includesBase bool) ([]byte, io.Closer, error) {
+	return a.buf, nil, nil
+}
+
+// DefaultMerger is the default implementation of the Merger interface. It
+// concatenates the two values to merge.
+var DefaultMerger = &Merger{
+	Merge: func(key, value []byte) (ValueMerger, error) {
+		res := &AppendValueMerger{}
+		res.buf = append(res.buf, value...)
+		return res, nil
+	},
+
+	Name: "pebble.concatenate",
+}

vendor/github.com/cockroachdb/pebble/internal/base/metrics.go

@@ -0,0 +1,98 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+import "time"
+
+// ThroughputMetric is used to measure the byte throughput of some component
+// that performs work in a single-threaded manner. The throughput can be
+// approximated by Bytes/(WorkDuration+IdleTime). The idle time is represented
+// separately, so that the user of this metric could approximate the peak
+// throughput as Bytes/WorkTime. The metric is designed to be cumulative (see
+// Merge).
+type ThroughputMetric struct {
+	// Bytes is the processes bytes by the component.
+	Bytes int64
+	// WorkDuration is the duration that the component spent doing work.
+	WorkDuration time.Duration
+	// IdleDuration is the duration that the component was idling, waiting for
+	// work.
+	IdleDuration time.Duration
+}
+
+// Merge accumulates the information from another throughput metric.
+func (tm *ThroughputMetric) Merge(x ThroughputMetric) {
+	tm.Bytes += x.Bytes
+	tm.WorkDuration += x.WorkDuration
+	tm.IdleDuration += x.IdleDuration
+}
+
+// Subtract subtracts the information from another ThroughputMetric
+func (tm *ThroughputMetric) Subtract(x ThroughputMetric) {
+	tm.Bytes -= x.Bytes
+	tm.WorkDuration -= x.WorkDuration
+	tm.IdleDuration -= x.IdleDuration
+}
+
+// PeakRate returns the approximate peak rate if there was no idling.
+func (tm *ThroughputMetric) PeakRate() int64 {
+	if tm.Bytes == 0 {
+		return 0
+	}
+	return int64((float64(tm.Bytes) / float64(tm.WorkDuration)) * float64(time.Second))
+}
+
+// Rate returns the observed rate.
+func (tm *ThroughputMetric) Rate() int64 {
+	if tm.Bytes == 0 {
+		return 0
+	}
+	return int64((float64(tm.Bytes) / float64(tm.WorkDuration+tm.IdleDuration)) *
+		float64(time.Second))
+}
+
+// Utilization returns a percent [0, 1.0] indicating the percent of time
+// work was performed.
+func (tm *ThroughputMetric) Utilization() float64 {
+	if tm.WorkDuration == 0 {
+		return 0
+	}
+	return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
+}
+
+// GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
+// accumulating samples of that gauge.
+type GaugeSampleMetric struct {
+	// The sum of all the samples.
+	sampleSum int64
+	// The number of samples.
+	count int64
+}
+
+// AddSample adds the given sample.
+func (gsm *GaugeSampleMetric) AddSample(sample int64) {
+	gsm.sampleSum += sample
+	gsm.count++
+}
+
+// Merge accumulates the information from another gauge metric.
+func (gsm *GaugeSampleMetric) Merge(x GaugeSampleMetric) {
+	gsm.sampleSum += x.sampleSum
+	gsm.count += x.count
+}
+
+// Subtract subtracts the information from another gauge metric.
+func (gsm *GaugeSampleMetric) Subtract(x GaugeSampleMetric) {
+	gsm.sampleSum -= x.sampleSum
+	gsm.count -= x.count
+}
+
+// Mean returns the mean value.
+func (gsm *GaugeSampleMetric) Mean() float64 {
+	if gsm.count == 0 {
+		return 0
+	}
+	return float64(gsm.sampleSum) / float64(gsm.count)
+}

vendor/github.com/cockroachdb/pebble/internal/base/options.go

@@ -0,0 +1,76 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package base
+
+// SSTable block defaults.
+const (
+	DefaultBlockRestartInterval = 16
+	DefaultBlockSize            = 4096
+	DefaultBlockSizeThreshold   = 90
+)
+
+// FilterType is the level at which to apply a filter: block or table.
+type FilterType int
+
+// The available filter types.
+const (
+	TableFilter FilterType = iota
+)
+
+func (t FilterType) String() string {
+	switch t {
+	case TableFilter:
+		return "table"
+	}
+	return "unknown"
+}
+
+// FilterWriter provides an interface for creating filter blocks. See
+// FilterPolicy for more details about filters.
+type FilterWriter interface {
+	// AddKey adds a key to the current filter block.
+	AddKey(key []byte)
+
+	// Finish appends to dst an encoded filter tha holds the current set of
+	// keys. The writer state is reset after the call to Finish allowing the
+	// writer to be reused for the creation of additional filters.
+	Finish(dst []byte) []byte
+}
+
+// FilterPolicy is an algorithm for probabilistically encoding a set of keys.
+// The canonical implementation is a Bloom filter.
+//
+// Every FilterPolicy has a name. This names the algorithm itself, not any one
+// particular instance. Aspects specific to a particular instance, such as the
+// set of keys or any other parameters, will be encoded in the []byte filter
+// returned by NewWriter.
+//
+// The name may be written to files on disk, along with the filter data. To use
+// these filters, the FilterPolicy name at the time of writing must equal the
+// name at the time of reading. If they do not match, the filters will be
+// ignored, which will not affect correctness but may affect performance.
+type FilterPolicy interface {
+	// Name names the filter policy.
+	Name() string
+
+	// MayContain returns whether the encoded filter may contain given key.
+	// False positives are possible, where it returns true for keys not in the
+	// original set.
+	MayContain(ftype FilterType, filter, key []byte) bool
+
+	// NewWriter creates a new FilterWriter.
+	NewWriter(ftype FilterType) FilterWriter
+}
+
+// BlockPropertyFilter is used in an Iterator to filter sstables and blocks
+// within the sstable. It should not maintain any per-sstable state, and must
+// be thread-safe.
+type BlockPropertyFilter interface {
+	// Name returns the name of the block property collector.
+	Name() string
+	// Intersects returns true if the set represented by prop intersects with
+	// the set in the filter.
+	Intersects(prop []byte) (bool, error)
+}

vendor/github.com/cockroachdb/pebble/internal/batchskl/README.md

@@ -0,0 +1,56 @@
+# batchskl
+
+Fast, non-concurrent skiplist implementation in Go that supports
+forward and backward iteration.
+
+## Limitations
+
+* The interface is tailored for use in indexing pebble batches. Keys
+  and values are stored outside of the skiplist making the skiplist
+  awkward for general purpose use.
+* Deletion is not supported. Instead, higher-level code is expected to
+  add deletion tombstones and needs to process those tombstones
+  appropriately.
+
+## Pedigree
+
+This code is based on Andy Kimball's arenaskl code.
+
+The arenaskl code is based on the skiplist found in Badger, a Go-based
+KV store:
+
+https://github.com/dgraph-io/badger/tree/master/skl
+
+The skiplist in Badger is itself based on a C++ skiplist built for
+Facebook's RocksDB:
+
+https://github.com/facebook/rocksdb/tree/master/memtable
+
+## Benchmarks
+
+The benchmarks consist of a mix of reads and writes executed in parallel. The
+fraction of reads is indicated in the run name: "frac_X" indicates a run where
+X percent of the operations are reads.
+
+```
+name                  time/op
+ReadWrite/frac_0      1.03µs ± 2%
+ReadWrite/frac_10     1.32µs ± 1%
+ReadWrite/frac_20     1.26µs ± 1%
+ReadWrite/frac_30     1.18µs ± 1%
+ReadWrite/frac_40     1.09µs ± 1%
+ReadWrite/frac_50      987ns ± 2%
+ReadWrite/frac_60     1.07µs ± 1%
+ReadWrite/frac_70      909ns ± 1%
+ReadWrite/frac_80      693ns ± 2%
+ReadWrite/frac_90      599ns ± 2%
+ReadWrite/frac_100    45.3ns ± 3%
+```
+
+Forward and backward iteration are also fast:
+
+```
+name                  time/op
+IterNext              4.49ns ± 3%
+IterPrev              4.48ns ± 3%
+```

vendor/github.com/cockroachdb/pebble/internal/batchskl/iterator.go

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ * Modifications copyright (C) 2017 Andy Kimball and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package batchskl
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+type splice struct {
+	prev uint32
+	next uint32
+}
+
+// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
+// to construct an iterator. The current state of the iterator can be cloned
+// by simply value copying the struct.
+type Iterator struct {
+	list  *Skiplist
+	nd    uint32
+	key   base.InternalKey
+	lower []byte
+	upper []byte
+}
+
+// Close resets the iterator.
+func (it *Iterator) Close() error {
+	it.list = nil
+	it.nd = 0
+	return nil
+}
+
+// SeekGE moves the iterator to the first entry whose key is greater than or
+// equal to the given key. Returns true if the iterator is pointing at a valid
+// entry and false otherwise. Note that SeekGE only checks the upper bound. It
+// is up to the caller to ensure that key is greater than or equal to the lower
+// bound.
+func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) *base.InternalKey {
+	if flags.TrySeekUsingNext() {
+		if it.nd == it.list.tail {
+			// Iterator is done.
+			return nil
+		}
+		less := it.list.cmp(it.key.UserKey, key) < 0
+		// Arbitrary constant. By measuring the seek cost as a function of the
+		// number of elements in the skip list, and fitting to a model, we
+		// could adjust the number of nexts based on the current size of the
+		// skip list.
+		const numNexts = 5
+		for i := 0; less && i < numNexts; i++ {
+			k := it.Next()
+			if k == nil {
+				// Iterator is done.
+				return nil
+			}
+			less = it.list.cmp(k.UserKey, key) < 0
+		}
+		if !less {
+			return &it.key
+		}
+	}
+
+	_, it.nd = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
+	if it.nd == it.list.tail {
+		return nil
+	}
+	nodeKey := it.list.getKey(it.nd)
+	if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
+		it.nd = it.list.tail
+		return nil
+	}
+	it.key = nodeKey
+	return &it.key
+}
+
+// SeekLT moves the iterator to the last entry whose key is less the given
+// key. Returns true if the iterator is pointing at a valid entry and false
+// otherwise. Note that SeekLT only checks the lower bound. It is up to the
+// caller to ensure that key is less than the upper bound.
+func (it *Iterator) SeekLT(key []byte) *base.InternalKey {
+	it.nd, _ = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
+	if it.nd == it.list.head {
+		return nil
+	}
+	nodeKey := it.list.getKey(it.nd)
+	if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
+		it.nd = it.list.head
+		return nil
+	}
+	it.key = nodeKey
+	return &it.key
+}
+
+// First seeks position at the first entry in list. Final state of iterator is
+// Valid() iff list is not empty. Note that First only checks the upper
+// bound. It is up to the caller to ensure that key is greater than or equal to
+// the lower bound (e.g. via a call to SeekGE(lower)).
+func (it *Iterator) First() *base.InternalKey {
+	it.nd = it.list.getNext(it.list.head, 0)
+	if it.nd == it.list.tail {
+		return nil
+	}
+	nodeKey := it.list.getKey(it.nd)
+	if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
+		it.nd = it.list.tail
+		return nil
+	}
+	it.key = nodeKey
+	return &it.key
+}
+
+// Last seeks position at the last entry in list. Final state of iterator is
+// Valid() iff list is not empty. Note that Last only checks the lower
+// bound. It is up to the caller to ensure that key is less than the upper
+// bound (e.g. via a call to SeekLT(upper)).
+func (it *Iterator) Last() *base.InternalKey {
+	it.nd = it.list.getPrev(it.list.tail, 0)
+	if it.nd == it.list.head {
+		return nil
+	}
+	nodeKey := it.list.getKey(it.nd)
+	if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
+		it.nd = it.list.head
+		return nil
+	}
+	it.key = nodeKey
+	return &it.key
+}
+
+// Next advances to the next position. If there are no following nodes, then
+// Valid() will be false after this call.
+func (it *Iterator) Next() *base.InternalKey {
+	it.nd = it.list.getNext(it.nd, 0)
+	if it.nd == it.list.tail {
+		return nil
+	}
+	nodeKey := it.list.getKey(it.nd)
+	if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
+		it.nd = it.list.tail
+		return nil
+	}
+	it.key = nodeKey
+	return &it.key
+}
+
+// Prev moves to the previous position. If there are no previous nodes, then
+// Valid() will be false after this call.
+func (it *Iterator) Prev() *base.InternalKey {
+	it.nd = it.list.getPrev(it.nd, 0)
+	if it.nd == it.list.head {
+		return nil
+	}
+	nodeKey := it.list.getKey(it.nd)
+	if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
+		it.nd = it.list.head
+		return nil
+	}
+	it.key = nodeKey
+	return &it.key
+}
+
+// Key returns the key at the current position.
+func (it *Iterator) Key() *base.InternalKey {
+	return &it.key
+}
+
+// KeyInfo returns the offset of the start of the record, the start of the key,
+// and the end of the key.
+func (it *Iterator) KeyInfo() (offset, keyStart, keyEnd uint32) {
+	n := it.list.node(it.nd)
+	return n.offset, n.keyStart, n.keyEnd
+}
+
+// Head true iff the iterator is positioned at the sentinel head node.
+func (it *Iterator) Head() bool {
+	return it.nd == it.list.head
+}
+
+// Tail true iff the iterator is positioned at the sentinel tail node.
+func (it *Iterator) Tail() bool {
+	return it.nd == it.list.tail
+}
+
+// Valid returns nil iff the iterator is positioned at a valid node.
+func (it *Iterator) Valid() bool {
+	return it.list != nil && it.nd != it.list.head && it.nd != it.list.tail
+}
+
+func (it *Iterator) String() string {
+	return "batch"
+}
+
+// SetBounds sets the lower and upper bounds for the iterator. Note that the
+// result of Next and Prev will be undefined until the iterator has been
+// repositioned with SeekGE, SeekLT, First, or Last.
+func (it *Iterator) SetBounds(lower, upper []byte) {
+	it.lower = lower
+	it.upper = upper
+}
+
+func (it *Iterator) seekForBaseSplice(key []byte, abbreviatedKey uint64) (prev, next uint32) {
+	prev = it.list.head
+	for level := it.list.height - 1; ; level-- {
+		prev, next = it.list.findSpliceForLevel(key, abbreviatedKey, level, prev)
+		if level == 0 {
+			break
+		}
+	}
+
+	return
+}

vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go

@@ -0,0 +1,442 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ * Modifications copyright (C) 2017 Andy Kimball and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+Adapted from RocksDB inline skiplist.
+
+Key differences:
+- No optimization for sequential inserts (no "prev").
+- No custom comparator.
+- Support overwrites. This requires care when we see the same key when inserting.
+  For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
+	there is no need for values. We don't intend to support versioning. In-place updates of values
+	would be more efficient.
+- We discard all non-concurrent code.
+- We do not support Splices. This simplifies the code a lot.
+- No AllocateNode or other pointer arithmetic.
+- We combine the findLessThan, findGreaterOrEqual, etc into one function.
+*/
+
+/*
+Further adapted from Badger: https://github.com/dgraph-io/badger.
+
+Key differences:
+- Support for previous pointers - doubly linked lists. Note that it's up to higher
+  level code to deal with the intermediate state that occurs during insertion,
+  where node A is linked to node B, but node B is not yet linked back to node A.
+- Iterator includes mutator functions.
+*/
+
+/*
+Further adapted from arenaskl: https://github.com/andy-kimball/arenaskl
+
+Key differences:
+- Removed support for deletion.
+- Removed support for concurrency.
+- External storage of keys.
+- Node storage grows to an arbitrary size.
+*/
+
+package batchskl // import "github.com/cockroachdb/pebble/internal/batchskl"
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"time"
+	"unsafe"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/constants"
+	"golang.org/x/exp/rand"
+)
+
+const (
+	maxHeight    = 20
+	maxNodeSize  = uint64(unsafe.Sizeof(node{}))
+	linksSize    = uint64(unsafe.Sizeof(links{}))
+	maxNodesSize = constants.MaxUint32OrInt
+)
+
+var (
+	// ErrExists indicates that a duplicate record was inserted. This should never
+	// happen for normal usage of batchskl as every key should have a unique
+	// sequence number.
+	ErrExists = errors.New("record with this key already exists")
+
+	// ErrTooManyRecords is a sentinel error returned when the size of the raw
+	// nodes slice exceeds the maximum allowed size (currently 1 << 32 - 1). This
+	// corresponds to ~117 M skiplist entries.
+	ErrTooManyRecords = errors.New("too many records")
+)
+
+type links struct {
+	next uint32
+	prev uint32
+}
+
+type node struct {
+	// The offset of the start of the record in the storage.
+	offset uint32
+	// The offset of the start and end of the key in storage.
+	keyStart uint32
+	keyEnd   uint32
+	// A fixed 8-byte abbreviation of the key, used to avoid retrieval of the key
+	// during seek operations. The key retrieval can be expensive purely due to
+	// cache misses while the abbreviatedKey stored here will be in the same
+	// cache line as the key and the links making accessing and comparing against
+	// it almost free.
+	abbreviatedKey uint64
+	// Most nodes do not need to use the full height of the link tower, since the
+	// probability of each successive level decreases exponentially. Because
+	// these elements are never accessed, they do not need to be allocated.
+	// Therefore, when a node is allocated, its memory footprint is deliberately
+	// truncated to not include unneeded link elements.
+	links [maxHeight]links
+}
+
+// Skiplist is a fast, non-cocnurrent skiplist implementation that supports
+// forward and backward iteration. See arenaskl.Skiplist for a concurrent
+// skiplist. Keys and values are stored externally from the skiplist via the
+// Storage interface. Deletion is not supported. Instead, higher-level code is
+// expected to perform deletion via tombstones and needs to process those
+// tombstones appropriately during retrieval operations.
+type Skiplist struct {
+	storage        *[]byte
+	cmp            base.Compare
+	abbreviatedKey base.AbbreviatedKey
+	nodes          []byte
+	head           uint32
+	tail           uint32
+	height         uint32 // Current height: 1 <= height <= maxHeight
+	rand           rand.PCGSource
+}
+
+var (
+	probabilities [maxHeight]uint32
+)
+
+func init() {
+	const pValue = 1 / math.E
+
+	// Precompute the skiplist probabilities so that only a single random number
+	// needs to be generated and so that the optimal pvalue can be used (inverse
+	// of Euler's number).
+	p := float64(1.0)
+	for i := 0; i < maxHeight; i++ {
+		probabilities[i] = uint32(float64(math.MaxUint32) * p)
+		p *= pValue
+	}
+}
+
+// NewSkiplist constructs and initializes a new, empty skiplist.
+func NewSkiplist(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) *Skiplist {
+	s := &Skiplist{}
+	s.Init(storage, cmp, abbreviatedKey)
+	return s
+}
+
+// Reset the fields in the skiplist for reuse.
+func (s *Skiplist) Reset() {
+	*s = Skiplist{
+		nodes:  s.nodes[:0],
+		height: 1,
+	}
+	const batchMaxRetainedSize = 1 << 20 // 1 MB
+	if cap(s.nodes) > batchMaxRetainedSize {
+		s.nodes = nil
+	}
+}
+
+// Init the skiplist to empty and re-initialize.
+func (s *Skiplist) Init(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) {
+	*s = Skiplist{
+		storage:        storage,
+		cmp:            cmp,
+		abbreviatedKey: abbreviatedKey,
+		nodes:          s.nodes[:0],
+		height:         1,
+	}
+	s.rand.Seed(uint64(time.Now().UnixNano()))
+
+	const initBufSize = 256
+	if cap(s.nodes) < initBufSize {
+		s.nodes = make([]byte, 0, initBufSize)
+	}
+
+	// Allocate head and tail nodes. While allocating a new node can fail, in the
+	// context of initializing the skiplist we consider it unrecoverable.
+	var err error
+	s.head, err = s.newNode(maxHeight, 0, 0, 0, 0)
+	if err != nil {
+		panic(err)
+	}
+	s.tail, err = s.newNode(maxHeight, 0, 0, 0, 0)
+	if err != nil {
+		panic(err)
+	}
+
+	// Link all head/tail levels together.
+	headNode := s.node(s.head)
+	tailNode := s.node(s.tail)
+	for i := uint32(0); i < maxHeight; i++ {
+		headNode.links[i].next = s.tail
+		tailNode.links[i].prev = s.head
+	}
+}
+
+// Add adds a new key to the skiplist if it does not yet exist. If the record
+// already exists, then Add returns ErrRecordExists.
+func (s *Skiplist) Add(keyOffset uint32) error {
+	data := (*s.storage)[keyOffset+1:]
+	v, n := binary.Uvarint(data)
+	if n <= 0 {
+		return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
+	}
+	data = data[n:]
+	if v > uint64(len(data)) {
+		return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
+	}
+	keyStart := 1 + keyOffset + uint32(n)
+	keyEnd := keyStart + uint32(v)
+	key := data[:v]
+	abbreviatedKey := s.abbreviatedKey(key)
+
+	// spl holds the list of next and previous links for each level in the
+	// skiplist indicating where the new node will be inserted.
+	var spl [maxHeight]splice
+
+	// Fast-path for in-order insertion of keys: compare the new key against the
+	// last key.
+	prev := s.getPrev(s.tail, 0)
+	if prevNode := s.node(prev); prev == s.head ||
+		abbreviatedKey > prevNode.abbreviatedKey ||
+		(abbreviatedKey == prevNode.abbreviatedKey &&
+			s.cmp(key, (*s.storage)[prevNode.keyStart:prevNode.keyEnd]) > 0) {
+		for level := uint32(0); level < s.height; level++ {
+			spl[level].prev = s.getPrev(s.tail, level)
+			spl[level].next = s.tail
+		}
+	} else {
+		s.findSplice(key, abbreviatedKey, &spl)
+	}
+
+	height := s.randomHeight()
+	// Increase s.height as necessary.
+	for ; s.height < height; s.height++ {
+		spl[s.height].next = s.tail
+		spl[s.height].prev = s.head
+	}
+
+	// We always insert from the base level and up. After you add a node in base
+	// level, we cannot create a node in the level above because it would have
+	// discovered the node in the base level.
+	nd, err := s.newNode(height, keyOffset, keyStart, keyEnd, abbreviatedKey)
+	if err != nil {
+		return err
+	}
+	newNode := s.node(nd)
+	for level := uint32(0); level < height; level++ {
+		next := spl[level].next
+		prev := spl[level].prev
+		newNode.links[level].next = next
+		newNode.links[level].prev = prev
+		s.node(next).links[level].prev = nd
+		s.node(prev).links[level].next = nd
+	}
+
+	return nil
+}
+
+// NewIter returns a new Iterator object. The lower and upper bound parameters
+// control the range of keys the iterator will return. Specifying for nil for
+// lower or upper bound disables the check for that boundary. Note that lower
+// bound is not checked on {SeekGE,First} and upper bound is not check on
+// {SeekLT,Last}. The user is expected to perform that check. Note that it is
+// safe for an iterator to be copied by value.
+func (s *Skiplist) NewIter(lower, upper []byte) Iterator {
+	return Iterator{list: s, lower: lower, upper: upper}
+}
+
+func (s *Skiplist) newNode(
+	height,
+	offset, keyStart, keyEnd uint32, abbreviatedKey uint64,
+) (uint32, error) {
+	if height < 1 || height > maxHeight {
+		panic("height cannot be less than one or greater than the max height")
+	}
+
+	unusedSize := uint64(maxHeight-int(height)) * linksSize
+	nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
+	if err != nil {
+		return 0, err
+	}
+	nd := s.node(nodeOffset)
+
+	nd.offset = offset
+	nd.keyStart = keyStart
+	nd.keyEnd = keyEnd
+	nd.abbreviatedKey = abbreviatedKey
+	return nodeOffset, nil
+}
+
+func (s *Skiplist) alloc(size uint32) (uint32, error) {
+	offset := uint64(len(s.nodes))
+
+	// We only have a need for memory up to offset + size, but we never want
+	// to allocate a node whose tail points into unallocated memory.
+	minAllocSize := offset + maxNodeSize
+	if uint64(cap(s.nodes)) < minAllocSize {
+		allocSize := uint64(cap(s.nodes)) * 2
+		if allocSize < minAllocSize {
+			allocSize = minAllocSize
+		}
+		// Cap the allocation at the max allowed size to avoid wasted capacity.
+		if allocSize > maxNodesSize {
+			// The new record may still not fit within the allocation, in which case
+			// we return early with an error. This avoids the panic below when we
+			// resize the slice. It also avoids the allocation and copy.
+			if uint64(offset)+uint64(size) > maxNodesSize {
+				return 0, errors.Wrapf(ErrTooManyRecords,
+					"alloc of new record (size=%d) would overflow uint32 (current size=%d)",
+					uint64(offset)+uint64(size), offset,
+				)
+			}
+			allocSize = maxNodesSize
+		}
+		tmp := make([]byte, len(s.nodes), allocSize)
+		copy(tmp, s.nodes)
+		s.nodes = tmp
+	}
+
+	newSize := uint32(offset) + size
+	s.nodes = s.nodes[:newSize]
+	return uint32(offset), nil
+}
+
+func (s *Skiplist) node(offset uint32) *node {
+	return (*node)(unsafe.Pointer(&s.nodes[offset]))
+}
+
+func (s *Skiplist) randomHeight() uint32 {
+	rnd := uint32(s.rand.Uint64())
+	h := uint32(1)
+	for h < maxHeight && rnd <= probabilities[h] {
+		h++
+	}
+	return h
+}
+
+func (s *Skiplist) findSplice(key []byte, abbreviatedKey uint64, spl *[maxHeight]splice) {
+	prev := s.head
+
+	for level := s.height - 1; ; level-- {
+		// The code in this loop is the same as findSpliceForLevel(). For some
+		// reason, calling findSpliceForLevel() here is much much slower than the
+		// inlined code below. The excess time is also caught up in the final
+		// return statement which makes little sense. Revisit when in go1.14 or
+		// later if inlining improves.
+
+		next := s.getNext(prev, level)
+		for next != s.tail {
+			// Assume prev.key < key.
+			nextNode := s.node(next)
+			nextAbbreviatedKey := nextNode.abbreviatedKey
+			if abbreviatedKey < nextAbbreviatedKey {
+				// We are done for this level, since prev.key < key < next.key.
+				break
+			}
+			if abbreviatedKey == nextAbbreviatedKey {
+				if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
+					// We are done for this level, since prev.key < key <= next.key.
+					break
+				}
+			}
+
+			// Keep moving right on this level.
+			prev = next
+			next = nextNode.links[level].next
+		}
+
+		spl[level].prev = prev
+		spl[level].next = next
+		if level == 0 {
+			break
+		}
+	}
+}
+
+func (s *Skiplist) findSpliceForLevel(
+	key []byte, abbreviatedKey uint64, level, start uint32,
+) (prev, next uint32) {
+	prev = start
+	next = s.getNext(prev, level)
+
+	for next != s.tail {
+		// Assume prev.key < key.
+		nextNode := s.node(next)
+		nextAbbreviatedKey := nextNode.abbreviatedKey
+		if abbreviatedKey < nextAbbreviatedKey {
+			// We are done for this level, since prev.key < key < next.key.
+			break
+		}
+		if abbreviatedKey == nextAbbreviatedKey {
+			if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
+				// We are done for this level, since prev.key < key < next.key.
+				break
+			}
+		}
+
+		// Keep moving right on this level.
+		prev = next
+		next = nextNode.links[level].next
+	}
+
+	return
+}
+
+func (s *Skiplist) getKey(nd uint32) base.InternalKey {
+	n := s.node(nd)
+	kind := base.InternalKeyKind((*s.storage)[n.offset])
+	key := (*s.storage)[n.keyStart:n.keyEnd]
+	return base.MakeInternalKey(key, uint64(n.offset)|base.InternalKeySeqNumBatch, kind)
+}
+
+func (s *Skiplist) getNext(nd, h uint32) uint32 {
+	return s.node(nd).links[h].next
+}
+
+func (s *Skiplist) getPrev(nd, h uint32) uint32 {
+	return s.node(nd).links[h].prev
+}
+
+func (s *Skiplist) debug() string {
+	var buf bytes.Buffer
+	for level := uint32(0); level < s.height; level++ {
+		var count int
+		for nd := s.head; nd != s.tail; nd = s.getNext(nd, level) {
+			count++
+		}
+		fmt.Fprintf(&buf, "%d: %d\n", level, count)
+	}
+	return buf.String()
+}
+
+// Silence unused warning.
+var _ = (*Skiplist).debug

vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go

@@ -0,0 +1,69 @@
+// Copyright 2016 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package bytealloc
+
+import "github.com/cockroachdb/pebble/internal/rawalloc"
+
+// An A provides chunk allocation of []byte, amortizing the overhead of each
+// allocation. Because the underlying storage for the slices is shared, they
+// should share a similar lifetime in order to avoid pinning large amounts of
+// memory unnecessarily. The allocator itself is a []byte where cap() indicates
+// the total amount of memory and len() is the amount already allocated. The
+// size of the buffer to allocate from is grown exponentially when it runs out
+// of room up to a maximum size (chunkAllocMaxSize).
+type A []byte
+
+const chunkAllocMinSize = 512
+const chunkAllocMaxSize = 512 << 10 // 512 KB
+
+func (a A) reserve(n int) A {
+	allocSize := cap(a) * 2
+	if allocSize < chunkAllocMinSize {
+		allocSize = chunkAllocMinSize
+	} else if allocSize > chunkAllocMaxSize {
+		allocSize = chunkAllocMaxSize
+	}
+	if allocSize < n {
+		allocSize = n
+	}
+	return rawalloc.New(0, allocSize)
+}
+
+// Alloc allocates a new chunk of memory with the specified length.
+func (a A) Alloc(n int) (A, []byte) {
+	if cap(a)-len(a) < n {
+		a = a.reserve(n)
+	}
+	p := len(a)
+	r := a[p : p+n : p+n]
+	a = a[:p+n]
+	return a, r
+}
+
+// Copy allocates a new chunk of memory, initializing it from src.
+func (a A) Copy(src []byte) (A, []byte) {
+	var alloc []byte
+	a, alloc = a.Alloc(len(src))
+	copy(alloc, src)
+	return a, alloc
+}
+
+// Reset returns the current chunk, resetting allocated memory back to none.
+// Future allocations will use memory previously allocated by previous calls to
+// Alloc or Copy, so the caller must know know that none of the previously
+// allocated byte slices are still in use.
+func (a A) Reset() A {
+	return a[:0]
+}

vendor/github.com/cockroachdb/pebble/internal/cache/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2018 Damian Gryski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/cockroachdb/pebble/internal/cache/cgo_disabled.go

@@ -0,0 +1,10 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !cgo
+// +build !cgo
+
+package cache
+
+const cgoEnabled = false

vendor/github.com/cockroachdb/pebble/internal/cache/cgo_enabled.go

@@ -0,0 +1,10 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build cgo
+// +build cgo
+
+package cache
+
+const cgoEnabled = true

vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go

@@ -0,0 +1,909 @@
+// Copyright 2018. All rights reserved. Use of this source code is governed by
+// an MIT-style license that can be found in the LICENSE file.
+
+// Package cache implements the CLOCK-Pro caching algorithm.
+//
+// CLOCK-Pro is a patent-free alternative to the Adaptive Replacement Cache,
+// https://en.wikipedia.org/wiki/Adaptive_replacement_cache.
+// It is an approximation of LIRS ( https://en.wikipedia.org/wiki/LIRS_caching_algorithm ),
+// much like the CLOCK page replacement algorithm is an approximation of LRU.
+//
+// This implementation is based on the python code from https://bitbucket.org/SamiLehtinen/pyclockpro .
+//
+// Slides describing the algorithm: http://fr.slideshare.net/huliang64/clockpro
+//
+// The original paper: http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html
+//
+// It is MIT licensed, like the original.
+package cache // import "github.com/cockroachdb/pebble/internal/cache"
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+type fileKey struct {
+	// id is the namespace for fileNums.
+	id      uint64
+	fileNum base.DiskFileNum
+}
+
+type key struct {
+	fileKey
+	offset uint64
+}
+
+// file returns the "file key" for the receiver. This is the key used for the
+// shard.files map.
+func (k key) file() key {
+	k.offset = 0
+	return k
+}
+
+func (k key) String() string {
+	return fmt.Sprintf("%d/%d/%d", k.id, k.fileNum, k.offset)
+}
+
+// Handle provides a strong reference to a value in the cache. The reference
+// does not pin the value in the cache, but it does prevent the underlying byte
+// slice from being reused.
+type Handle struct {
+	value *Value
+}
+
+// Get returns the value stored in handle.
+func (h Handle) Get() []byte {
+	if h.value != nil {
+		// NB: We don't increment shard.hits in this code path because we only want
+		// to record a hit when the handle is retrieved from the cache.
+		return h.value.buf
+	}
+	return nil
+}
+
+// Release releases the reference to the cache entry.
+func (h Handle) Release() {
+	h.value.release()
+}
+
+type shard struct {
+	hits   atomic.Int64
+	misses atomic.Int64
+
+	mu sync.RWMutex
+
+	reservedSize int64
+	maxSize      int64
+	coldTarget   int64
+	blocks       robinHoodMap // fileNum+offset -> block
+	files        robinHoodMap // fileNum -> list of blocks
+
+	// The blocks and files maps store values in manually managed memory that is
+	// invisible to the Go GC. This is fine for Value and entry objects that are
+	// stored in manually managed memory, but when the "invariants" build tag is
+	// set, all Value and entry objects are Go allocated and the entries map will
+	// contain a reference to every entry.
+	entries map[*entry]struct{}
+
+	handHot  *entry
+	handCold *entry
+	handTest *entry
+
+	sizeHot  int64
+	sizeCold int64
+	sizeTest int64
+
+	// The count fields are used exclusively for asserting expectations.
+	// We've seen infinite looping (cockroachdb/cockroach#70154) that
+	// could be explained by a corrupted sizeCold. Through asserting on
+	// these fields, we hope to gain more insight from any future
+	// reproductions.
+	countHot  int64
+	countCold int64
+	countTest int64
+}
+
+func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
+	c.mu.RLock()
+	var value *Value
+	if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
+		value = e.acquireValue()
+		if value != nil {
+			e.referenced.Store(true)
+		}
+	}
+	c.mu.RUnlock()
+	if value == nil {
+		c.misses.Add(1)
+		return Handle{}
+	}
+	c.hits.Add(1)
+	return Handle{value: value}
+}
+
+func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
+	if n := value.refs(); n != 1 {
+		panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	k := key{fileKey{id, fileNum}, offset}
+	e := c.blocks.Get(k)
+
+	switch {
+	case e == nil:
+		// no cache entry? add it
+		e = newEntry(c, k, int64(len(value.buf)))
+		e.setValue(value)
+		if c.metaAdd(k, e) {
+			value.ref.trace("add-cold")
+			c.sizeCold += e.size
+			c.countCold++
+		} else {
+			value.ref.trace("skip-cold")
+			e.free()
+			e = nil
+		}
+
+	case e.peekValue() != nil:
+		// cache entry was a hot or cold page
+		e.setValue(value)
+		e.referenced.Store(true)
+		delta := int64(len(value.buf)) - e.size
+		e.size = int64(len(value.buf))
+		if e.ptype == etHot {
+			value.ref.trace("add-hot")
+			c.sizeHot += delta
+		} else {
+			value.ref.trace("add-cold")
+			c.sizeCold += delta
+		}
+		c.evict()
+
+	default:
+		// cache entry was a test page
+		c.sizeTest -= e.size
+		c.countTest--
+		c.metaDel(e).release()
+		c.metaCheck(e)
+
+		e.size = int64(len(value.buf))
+		c.coldTarget += e.size
+		if c.coldTarget > c.targetSize() {
+			c.coldTarget = c.targetSize()
+		}
+
+		e.referenced.Store(false)
+		e.setValue(value)
+		e.ptype = etHot
+		if c.metaAdd(k, e) {
+			value.ref.trace("add-hot")
+			c.sizeHot += e.size
+			c.countHot++
+		} else {
+			value.ref.trace("skip-hot")
+			e.free()
+			e = nil
+		}
+	}
+
+	c.checkConsistency()
+
+	// Values are initialized with a reference count of 1. That reference count
+	// is being transferred to the returned Handle.
+	return Handle{value: value}
+}
+
+func (c *shard) checkConsistency() {
+	// See the comment above the count{Hot,Cold,Test} fields.
+	switch {
+	case c.sizeHot < 0 || c.sizeCold < 0 || c.sizeTest < 0 || c.countHot < 0 || c.countCold < 0 || c.countTest < 0:
+		panic(fmt.Sprintf("pebble: unexpected negative: %d (%d bytes) hot, %d (%d bytes) cold, %d (%d bytes) test",
+			c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest))
+	case c.sizeHot > 0 && c.countHot == 0:
+		panic(fmt.Sprintf("pebble: mismatch %d hot size, %d hot count", c.sizeHot, c.countHot))
+	case c.sizeCold > 0 && c.countCold == 0:
+		panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
+	case c.sizeTest > 0 && c.countTest == 0:
+		panic(fmt.Sprintf("pebble: mismatch %d test size, %d test count", c.sizeTest, c.countTest))
+	}
+}
+
+// Delete deletes the cached value for the specified file and offset.
+func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
+	// The common case is there is nothing to delete, so do a quick check with
+	// shared lock.
+	k := key{fileKey{id, fileNum}, offset}
+	c.mu.RLock()
+	exists := c.blocks.Get(k) != nil
+	c.mu.RUnlock()
+	if !exists {
+		return
+	}
+
+	var deletedValue *Value
+	func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+
+		e := c.blocks.Get(k)
+		if e == nil {
+			return
+		}
+		deletedValue = c.metaEvict(e)
+		c.checkConsistency()
+	}()
+	// Now that the mutex has been dropped, release the reference which will
+	// potentially free the memory associated with the previous cached value.
+	deletedValue.release()
+}
+
+// EvictFile evicts all of the cache values for the specified file.
+func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
+	fkey := key{fileKey{id, fileNum}, 0}
+	for c.evictFileRun(fkey) {
+		// Sched switch to give another goroutine an opportunity to acquire the
+		// shard mutex.
+		runtime.Gosched()
+	}
+}
+
+func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
+	// If most of the file's blocks are held in the block cache, evicting all
+	// the blocks may take a while. We don't want to block the entire cache
+	// shard, forcing concurrent readers to wait until we're finished. We drop
+	// the mutex every [blocksPerMutexAcquisition] blocks to give other
+	// goroutines an opportunity to make progress.
+	const blocksPerMutexAcquisition = 5
+	c.mu.Lock()
+
+	// Releasing a value may result in free-ing it back to the memory allocator.
+	// This can have a nontrivial cost that we'd prefer to not pay while holding
+	// the shard mutex, so we collect the evicted values in a local slice and
+	// only release them in a defer after dropping the cache mutex.
+	var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
+	obsoleteValues := obsoleteValuesAlloc[:0]
+	defer func() {
+		c.mu.Unlock()
+		for _, v := range obsoleteValues {
+			v.release()
+		}
+	}()
+
+	blocks := c.files.Get(fkey)
+	if blocks == nil {
+		// No blocks for this file.
+		return false
+	}
+
+	// b is the current head of the doubly linked list, and n is the entry after b.
+	for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
+		n = b.fileLink.next
+		obsoleteValues = append(obsoleteValues, c.metaEvict(b))
+		if b == n {
+			// b == n represents the case where b was the last entry remaining
+			// in the doubly linked list, which is why it pointed at itself. So
+			// no more entries left.
+			c.checkConsistency()
+			return false
+		}
+	}
+	// Exhausted blocksPerMutexAcquisition.
+	return true
+}
+
+func (c *shard) Free() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// NB: we use metaDel rather than metaEvict in order to avoid the expensive
+	// metaCheck call when the "invariants" build tag is specified.
+	for c.handHot != nil {
+		e := c.handHot
+		c.metaDel(c.handHot).release()
+		e.free()
+	}
+
+	c.blocks.free()
+	c.files.free()
+}
+
+func (c *shard) Reserve(n int) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.reservedSize += int64(n)
+
+	// Changing c.reservedSize will either increase or decrease
+	// the targetSize. But we want coldTarget to be in the range
+	// [0, targetSize]. So, if c.targetSize decreases, make sure
+	// that the coldTarget fits within the limits.
+	targetSize := c.targetSize()
+	if c.coldTarget > targetSize {
+		c.coldTarget = targetSize
+	}
+
+	c.evict()
+	c.checkConsistency()
+}
+
+// Size returns the current space used by the cache.
+func (c *shard) Size() int64 {
+	c.mu.RLock()
+	size := c.sizeHot + c.sizeCold
+	c.mu.RUnlock()
+	return size
+}
+
+func (c *shard) targetSize() int64 {
+	target := c.maxSize - c.reservedSize
+	// Always return a positive integer for targetSize. This is so that we don't
+	// end up in an infinite loop in evict(), in cases where reservedSize is
+	// greater than or equal to maxSize.
+	if target < 1 {
+		return 1
+	}
+	return target
+}
+
+// Add the entry to the cache, returning true if the entry was added and false
+// if it would not fit in the cache.
+func (c *shard) metaAdd(key key, e *entry) bool {
+	c.evict()
+	if e.size > c.targetSize() {
+		// The entry is larger than the target cache size.
+		return false
+	}
+
+	c.blocks.Put(key, e)
+	if entriesGoAllocated {
+		// Go allocated entries need to be referenced from Go memory. The entries
+		// map provides that reference.
+		c.entries[e] = struct{}{}
+	}
+
+	if c.handHot == nil {
+		// first element
+		c.handHot = e
+		c.handCold = e
+		c.handTest = e
+	} else {
+		c.handHot.link(e)
+	}
+
+	if c.handCold == c.handHot {
+		c.handCold = c.handCold.prev()
+	}
+
+	fkey := key.file()
+	if fileBlocks := c.files.Get(fkey); fileBlocks == nil {
+		c.files.Put(fkey, e)
+	} else {
+		fileBlocks.linkFile(e)
+	}
+	return true
+}
+
+// Remove the entry from the cache. This removes the entry from the blocks map,
+// the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
+// entry. Returns the deleted value that must be released, if any.
+func (c *shard) metaDel(e *entry) (deletedValue *Value) {
+	if value := e.peekValue(); value != nil {
+		value.ref.trace("metaDel")
+	}
+	// Remove the pointer to the value.
+	deletedValue = e.val
+	e.val = nil
+
+	c.blocks.Delete(e.key)
+	if entriesGoAllocated {
+		// Go allocated entries need to be referenced from Go memory. The entries
+		// map provides that reference.
+		delete(c.entries, e)
+	}
+
+	if e == c.handHot {
+		c.handHot = c.handHot.prev()
+	}
+	if e == c.handCold {
+		c.handCold = c.handCold.prev()
+	}
+	if e == c.handTest {
+		c.handTest = c.handTest.prev()
+	}
+
+	if e.unlink() == e {
+		// This was the last entry in the cache.
+		c.handHot = nil
+		c.handCold = nil
+		c.handTest = nil
+	}
+
+	fkey := e.key.file()
+	if next := e.unlinkFile(); e == next {
+		c.files.Delete(fkey)
+	} else {
+		c.files.Put(fkey, next)
+	}
+	return deletedValue
+}
+
+// Check that the specified entry is not referenced by the cache.
+func (c *shard) metaCheck(e *entry) {
+	if invariants.Enabled {
+		if _, ok := c.entries[e]; ok {
+			fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in entries map\n%s",
+				e, e.key, debug.Stack())
+			os.Exit(1)
+		}
+		if c.blocks.findByValue(e) != nil {
+			fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks map\n%s\n%s",
+				e, e.key, &c.blocks, debug.Stack())
+			os.Exit(1)
+		}
+		if c.files.findByValue(e) != nil {
+			fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in files map\n%s\n%s",
+				e, e.key, &c.files, debug.Stack())
+			os.Exit(1)
+		}
+		// NB: c.hand{Hot,Cold,Test} are pointers into a single linked list. We
+		// only have to traverse one of them to check all of them.
+		var countHot, countCold, countTest int64
+		var sizeHot, sizeCold, sizeTest int64
+		for t := c.handHot.next(); t != nil; t = t.next() {
+			// Recompute count{Hot,Cold,Test} and size{Hot,Cold,Test}.
+			switch t.ptype {
+			case etHot:
+				countHot++
+				sizeHot += t.size
+			case etCold:
+				countCold++
+				sizeCold += t.size
+			case etTest:
+				countTest++
+				sizeTest += t.size
+			}
+			if e == t {
+				fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks list\n%s",
+					e, e.key, debug.Stack())
+				os.Exit(1)
+			}
+			if t == c.handHot {
+				break
+			}
+		}
+		if countHot != c.countHot || countCold != c.countCold || countTest != c.countTest ||
+			sizeHot != c.sizeHot || sizeCold != c.sizeCold || sizeTest != c.sizeTest {
+			fmt.Fprintf(os.Stderr, `divergence of Hot,Cold,Test statistics
+				cache's statistics: hot %d, %d, cold %d, %d, test %d, %d
+				recalculated statistics: hot %d, %d, cold %d, %d, test %d, %d\n%s`,
+				c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest,
+				countHot, sizeHot, countCold, sizeCold, countTest, sizeTest,
+				debug.Stack())
+			os.Exit(1)
+		}
+	}
+}
+
+func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
+	switch e.ptype {
+	case etHot:
+		c.sizeHot -= e.size
+		c.countHot--
+	case etCold:
+		c.sizeCold -= e.size
+		c.countCold--
+	case etTest:
+		c.sizeTest -= e.size
+		c.countTest--
+	}
+	evictedValue = c.metaDel(e)
+	c.metaCheck(e)
+	e.free()
+	return evictedValue
+}
+
+func (c *shard) evict() {
+	for c.targetSize() <= c.sizeHot+c.sizeCold && c.handCold != nil {
+		c.runHandCold(c.countCold, c.sizeCold)
+	}
+}
+
+func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
+	// countColdDebug and sizeColdDebug should equal c.countCold and
+	// c.sizeCold. They're parameters only to aid in debugging of
+	// cockroachdb/cockroach#70154. Since they're parameters, their
+	// arguments will appear within stack traces should we encounter
+	// a reproduction.
+	if c.countCold != countColdDebug || c.sizeCold != sizeColdDebug {
+		panic(fmt.Sprintf("runHandCold: cold count and size are %d, %d, arguments are %d and %d",
+			c.countCold, c.sizeCold, countColdDebug, sizeColdDebug))
+	}
+
+	e := c.handCold
+	if e.ptype == etCold {
+		if e.referenced.Load() {
+			e.referenced.Store(false)
+			e.ptype = etHot
+			c.sizeCold -= e.size
+			c.countCold--
+			c.sizeHot += e.size
+			c.countHot++
+		} else {
+			e.setValue(nil)
+			e.ptype = etTest
+			c.sizeCold -= e.size
+			c.countCold--
+			c.sizeTest += e.size
+			c.countTest++
+			for c.targetSize() < c.sizeTest && c.handTest != nil {
+				c.runHandTest()
+			}
+		}
+	}
+
+	c.handCold = c.handCold.next()
+
+	for c.targetSize()-c.coldTarget <= c.sizeHot && c.handHot != nil {
+		c.runHandHot()
+	}
+}
+
+func (c *shard) runHandHot() {
+	if c.handHot == c.handTest && c.handTest != nil {
+		c.runHandTest()
+		if c.handHot == nil {
+			return
+		}
+	}
+
+	e := c.handHot
+	if e.ptype == etHot {
+		if e.referenced.Load() {
+			e.referenced.Store(false)
+		} else {
+			e.ptype = etCold
+			c.sizeHot -= e.size
+			c.countHot--
+			c.sizeCold += e.size
+			c.countCold++
+		}
+	}
+
+	c.handHot = c.handHot.next()
+}
+
+func (c *shard) runHandTest() {
+	if c.sizeCold > 0 && c.handTest == c.handCold && c.handCold != nil {
+		// sizeCold is > 0, so assert that countCold == 0. See the
+		// comment above count{Hot,Cold,Test}.
+		if c.countCold == 0 {
+			panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
+		}
+
+		c.runHandCold(c.countCold, c.sizeCold)
+		if c.handTest == nil {
+			return
+		}
+	}
+
+	e := c.handTest
+	if e.ptype == etTest {
+		c.sizeTest -= e.size
+		c.countTest--
+		c.coldTarget -= e.size
+		if c.coldTarget < 0 {
+			c.coldTarget = 0
+		}
+		c.metaDel(e).release()
+		c.metaCheck(e)
+		e.free()
+	}
+
+	c.handTest = c.handTest.next()
+}
+
+// Metrics holds metrics for the cache.
+type Metrics struct {
+	// The number of bytes inuse by the cache.
+	Size int64
+	// The count of objects (blocks or tables) in the cache.
+	Count int64
+	// The number of cache hits.
+	Hits int64
+	// The number of cache misses.
+	Misses int64
+}
+
+// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
+// used for page replacement
+// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
+// order to provide better concurrency, 4 x NumCPUs shards are created, with
+// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
+// is run independently on each shard.
+//
+// Blocks are keyed by an (id, fileNum, offset) triple. The ID is a namespace
+// for file numbers and allows a single Cache to be shared between multiple
+// Pebble instances. The fileNum and offset refer to an sstable file number and
+// the offset of the block within the file. Because sstables are immutable and
+// file numbers are never reused, (fileNum,offset) are unique for the lifetime
+// of a Pebble instance.
+//
+// In addition to maintaining a map from (fileNum,offset) to data, each shard
+// maintains a map of the cached blocks for a particular fileNum. This allows
+// efficient eviction of all of the blocks for a file which is used when an
+// sstable is deleted from disk.
+//
+// # Memory Management
+//
+// In order to reduce pressure on the Go GC, manual memory management is
+// performed for the data stored in the cache. Manual memory management is
+// performed by calling into C.{malloc,free} to allocate memory. Cache.Values
+// are reference counted and the memory backing a manual value is freed when
+// the reference count drops to 0.
+//
+// Manual memory management brings the possibility of memory leaks. It is
+// imperative that every Handle returned by Cache.{Get,Set} is eventually
+// released. The "invariants" build tag enables a leak detection facility that
+// places a GC finalizer on cache.Value. When the cache.Value finalizer is run,
+// if the underlying buffer is still present a leak has occurred. The "tracing"
+// build tag enables tracing of cache.Value reference count manipulation and
+// eases finding where a leak has occurred. These two facilities are usually
+// used in combination by specifying `-tags invariants,tracing`. Note that
+// "tracing" produces a significant slowdown, while "invariants" does not.
+type Cache struct {
+	refs    atomic.Int64
+	maxSize int64
+	idAlloc atomic.Uint64
+	shards  []shard
+
+	// Traces recorded by Cache.trace. Used for debugging.
+	tr struct {
+		sync.Mutex
+		msgs []string
+	}
+}
+
+// New creates a new cache of the specified size. Memory for the cache is
+// allocated on demand, not during initialization. The cache is created with a
+// reference count of 1. Each DB it is associated with adds a reference, so the
+// creator of the cache should usually release their reference after the DB is
+// created.
+//
+//	c := cache.New(...)
+//	defer c.Unref()
+//	d, err := pebble.Open(pebble.Options{Cache: c})
+func New(size int64) *Cache {
+	// How many cache shards should we create?
+	//
+	// Note that the probability two processors will try to access the same
+	// shard at the same time increases superlinearly with the number of
+	// processors (Eg, consider the brithday problem where each CPU is a person,
+	// and each shard is a possible birthday).
+	//
+	// We could consider growing the number of shards superlinearly, but
+	// increasing the shard count may reduce the effectiveness of the caching
+	// algorithm if frequently-accessed blocks are insufficiently distributed
+	// across shards. If a shard's size is smaller than a single frequently
+	// scanned sstable, then the shard will be unable to hold the entire
+	// frequently-scanned table in memory despite other shards still holding
+	// infrequently accessed blocks.
+	//
+	// Experimentally, we've observed contention contributing to tail latencies
+	// at 2 shards per processor. For now we use 4 shards per processor,
+	// recognizing this may not be final word.
+	m := 4 * runtime.GOMAXPROCS(0)
+
+	// In tests we can use large CPU machines with small cache sizes and have
+	// many caches in existence at a time. If sharding into m shards would
+	// produce too small shards, constrain the number of shards to 4.
+	const minimumShardSize = 4 << 20 // 4 MiB
+	if m > 4 && int(size)/m < minimumShardSize {
+		m = 4
+	}
+	return newShards(size, m)
+}
+
+func newShards(size int64, shards int) *Cache {
+	c := &Cache{
+		maxSize: size,
+		shards:  make([]shard, shards),
+	}
+	c.refs.Store(1)
+	c.idAlloc.Store(1)
+	c.trace("alloc", c.refs.Load())
+	for i := range c.shards {
+		c.shards[i] = shard{
+			maxSize:    size / int64(len(c.shards)),
+			coldTarget: size / int64(len(c.shards)),
+		}
+		if entriesGoAllocated {
+			c.shards[i].entries = make(map[*entry]struct{})
+		}
+		c.shards[i].blocks.init(16)
+		c.shards[i].files.init(16)
+	}
+
+	// Note: this is a no-op if invariants are disabled or race is enabled.
+	invariants.SetFinalizer(c, func(obj interface{}) {
+		c := obj.(*Cache)
+		if v := c.refs.Load(); v != 0 {
+			c.tr.Lock()
+			fmt.Fprintf(os.Stderr,
+				"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
+			if len(c.tr.msgs) > 0 {
+				fmt.Fprintf(os.Stderr, "%s\n", strings.Join(c.tr.msgs, "\n"))
+			}
+			c.tr.Unlock()
+			os.Exit(1)
+		}
+	})
+	return c
+}
+
+func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
+	if id == 0 {
+		panic("pebble: 0 cache ID is invalid")
+	}
+
+	// Inlined version of fnv.New64 + Write.
+	const offset64 = 14695981039346656037
+	const prime64 = 1099511628211
+
+	h := uint64(offset64)
+	for i := 0; i < 8; i++ {
+		h *= prime64
+		h ^= uint64(id & 0xff)
+		id >>= 8
+	}
+	fileNumVal := uint64(fileNum.FileNum())
+	for i := 0; i < 8; i++ {
+		h *= prime64
+		h ^= uint64(fileNumVal) & 0xff
+		fileNumVal >>= 8
+	}
+	for i := 0; i < 8; i++ {
+		h *= prime64
+		h ^= uint64(offset & 0xff)
+		offset >>= 8
+	}
+
+	return &c.shards[h%uint64(len(c.shards))]
+}
+
+// Ref adds a reference to the cache. The cache only remains valid as long a
+// reference is maintained to it.
+func (c *Cache) Ref() {
+	v := c.refs.Add(1)
+	if v <= 1 {
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+	}
+	c.trace("ref", v)
+}
+
+// Unref releases a reference on the cache.
+func (c *Cache) Unref() {
+	v := c.refs.Add(-1)
+	c.trace("unref", v)
+	switch {
+	case v < 0:
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+	case v == 0:
+		for i := range c.shards {
+			c.shards[i].Free()
+		}
+	}
+}
+
+// Get retrieves the cache value for the specified file and offset, returning
+// nil if no value is present.
+func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
+	return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
+}
+
+// Set sets the cache value for the specified file and offset, overwriting an
+// existing value if present. A Handle is returned which provides faster
+// retrieval of the cached value than Get (lock-free and avoidance of the map
+// lookup). The value must have been allocated by Cache.Alloc.
+func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
+	return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
+}
+
+// Delete deletes the cached value for the specified file and offset.
+func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
+	c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
+}
+
+// EvictFile evicts all of the cache values for the specified file.
+func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
+	if id == 0 {
+		panic("pebble: 0 cache ID is invalid")
+	}
+	for i := range c.shards {
+		c.shards[i].EvictFile(id, fileNum)
+	}
+}
+
+// MaxSize returns the max size of the cache.
+func (c *Cache) MaxSize() int64 {
+	return c.maxSize
+}
+
+// Size returns the current space used by the cache.
+func (c *Cache) Size() int64 {
+	var size int64
+	for i := range c.shards {
+		size += c.shards[i].Size()
+	}
+	return size
+}
+
+// Alloc allocates a byte slice of the specified size, possibly reusing
+// previously allocated but unused memory. The memory backing the value is
+// manually managed. The caller MUST either add the value to the cache (via
+// Cache.Set), or release the value (via Cache.Free). Failure to do so will
+// result in a memory leak.
+func Alloc(n int) *Value {
+	return newValue(n)
+}
+
+// Free frees the specified value. The buffer associated with the value will
+// possibly be reused, making it invalid to use the buffer after calling
+// Free. Do not call Free on a value that has been added to the cache.
+func Free(v *Value) {
+	if n := v.refs(); n > 1 {
+		panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
+	}
+	v.release()
+}
+
+// Reserve N bytes in the cache. This effectively shrinks the size of the cache
+// by N bytes, without actually consuming any memory. The returned closure
+// should be invoked to release the reservation.
+func (c *Cache) Reserve(n int) func() {
+	// Round-up the per-shard reservation. Most reservations should be large, so
+	// this probably doesn't matter in practice.
+	shardN := (n + len(c.shards) - 1) / len(c.shards)
+	for i := range c.shards {
+		c.shards[i].Reserve(shardN)
+	}
+	return func() {
+		if shardN == -1 {
+			panic("pebble: cache reservation already released")
+		}
+		for i := range c.shards {
+			c.shards[i].Reserve(-shardN)
+		}
+		shardN = -1
+	}
+}
+
+// Metrics returns the metrics for the cache.
+func (c *Cache) Metrics() Metrics {
+	var m Metrics
+	for i := range c.shards {
+		s := &c.shards[i]
+		s.mu.RLock()
+		m.Count += int64(s.blocks.Count())
+		m.Size += s.sizeHot + s.sizeCold
+		s.mu.RUnlock()
+		m.Hits += s.hits.Load()
+		m.Misses += s.misses.Load()
+	}
+	return m
+}
+
+// NewID returns a new ID to be used as a namespace for cached file
+// blocks.
+func (c *Cache) NewID() uint64 {
+	return c.idAlloc.Add(1)
+}

vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_normal.go

@@ -0,0 +1,10 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !tracing
+// +build !tracing
+
+package cache
+
+func (c *Cache) trace(_ string, _ int64) {}

vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_tracing.go

@@ -0,0 +1,20 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build tracing
+// +build tracing
+
+package cache
+
+import (
+	"fmt"
+	"runtime/debug"
+)
+
+func (c *Cache) trace(msg string, refs int64) {
+	s := fmt.Sprintf("%s: refs=%d\n%s", msg, refs, debug.Stack())
+	c.tr.Lock()
+	c.tr.msgs = append(c.tr.msgs, s)
+	c.tr.Unlock()
+}

vendor/github.com/cockroachdb/pebble/internal/cache/entry.go

@@ -0,0 +1,155 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package cache
+
+import "sync/atomic"
+
+type entryType int8
+
+const (
+	etTest entryType = iota
+	etCold
+	etHot
+)
+
+func (p entryType) String() string {
+	switch p {
+	case etTest:
+		return "test"
+	case etCold:
+		return "cold"
+	case etHot:
+		return "hot"
+	}
+	return "unknown"
+}
+
+// entry holds the metadata for a cache entry. The memory for an entry is
+// allocated from manually managed memory.
+//
+// Using manual memory management for entries is technically a volation of the
+// Cgo pointer rules:
+//
+//	https://golang.org/cmd/cgo/#hdr-Passing_pointers
+//
+// Specifically, Go pointers should not be stored in C allocated memory. The
+// reason for this rule is that the Go GC will not look at C allocated memory
+// to find pointers to Go objects. If the only reference to a Go object is
+// stored in C allocated memory, the object will be reclaimed. The shard field
+// of the entry struct points to a Go allocated object, thus the
+// violation. What makes this "safe" is that the Cache guarantees that there
+// are other pointers to the shard which will keep it alive.
+type entry struct {
+	key key
+	// The value associated with the entry. The entry holds a reference on the
+	// value which is maintained by entry.setValue().
+	val       *Value
+	blockLink struct {
+		next *entry
+		prev *entry
+	}
+	fileLink struct {
+		next *entry
+		prev *entry
+	}
+	size  int64
+	ptype entryType
+	// referenced is atomically set to indicate that this entry has been accessed
+	// since the last time one of the clock hands swept it.
+	referenced atomic.Bool
+	shard      *shard
+	// Reference count for the entry. The entry is freed when the reference count
+	// drops to zero.
+	ref refcnt
+}
+
+func newEntry(s *shard, key key, size int64) *entry {
+	e := entryAllocNew()
+	*e = entry{
+		key:   key,
+		size:  size,
+		ptype: etCold,
+		shard: s,
+	}
+	e.blockLink.next = e
+	e.blockLink.prev = e
+	e.fileLink.next = e
+	e.fileLink.prev = e
+	e.ref.init(1)
+	return e
+}
+
+func (e *entry) free() {
+	e.setValue(nil)
+	*e = entry{}
+	entryAllocFree(e)
+}
+
+func (e *entry) next() *entry {
+	if e == nil {
+		return nil
+	}
+	return e.blockLink.next
+}
+
+func (e *entry) prev() *entry {
+	if e == nil {
+		return nil
+	}
+	return e.blockLink.prev
+}
+
+func (e *entry) link(s *entry) {
+	s.blockLink.prev = e.blockLink.prev
+	s.blockLink.prev.blockLink.next = s
+	s.blockLink.next = e
+	s.blockLink.next.blockLink.prev = s
+}
+
+func (e *entry) unlink() *entry {
+	next := e.blockLink.next
+	e.blockLink.prev.blockLink.next = e.blockLink.next
+	e.blockLink.next.blockLink.prev = e.blockLink.prev
+	e.blockLink.prev = e
+	e.blockLink.next = e
+	return next
+}
+
+func (e *entry) linkFile(s *entry) {
+	s.fileLink.prev = e.fileLink.prev
+	s.fileLink.prev.fileLink.next = s
+	s.fileLink.next = e
+	s.fileLink.next.fileLink.prev = s
+}
+
+func (e *entry) unlinkFile() *entry {
+	next := e.fileLink.next
+	e.fileLink.prev.fileLink.next = e.fileLink.next
+	e.fileLink.next.fileLink.prev = e.fileLink.prev
+	e.fileLink.prev = e
+	e.fileLink.next = e
+	return next
+}
+
+func (e *entry) setValue(v *Value) {
+	if v != nil {
+		v.acquire()
+	}
+	old := e.val
+	e.val = v
+	old.release()
+}
+
+func (e *entry) peekValue() *Value {
+	return e.val
+}
+
+func (e *entry) acquireValue() *Value {
+	v := e.val
+	if v != nil {
+		v.acquire()
+	}
+	return v
+}

vendor/github.com/cockroachdb/pebble/internal/cache/entry_invariants.go

@@ -0,0 +1,38 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+//
+//go:build (invariants && !race) || (tracing && !race)
+// +build invariants,!race tracing,!race
+
+package cache
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// When the "invariants" or "tracing" build tags are enabled, we need to
+// allocate entries using the Go allocator so entry.val properly maintains a
+// reference to the Value.
+const entriesGoAllocated = true
+
+func entryAllocNew() *entry {
+	e := &entry{}
+	// Note: this is a no-op if invariants and tracing are disabled or race is
+	// enabled.
+	invariants.SetFinalizer(e, func(obj interface{}) {
+		e := obj.(*entry)
+		if v := e.ref.refs(); v != 0 {
+			fmt.Fprintf(os.Stderr, "%p: cache entry has non-zero reference count: %d\n%s",
+				e, v, e.ref.traces())
+			os.Exit(1)
+		}
+	})
+	return e
+}
+
+func entryAllocFree(e *entry) {
+}

vendor/github.com/cockroachdb/pebble/internal/cache/entry_normal.go

@@ -0,0 +1,103 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+//
+//go:build (!invariants && !tracing) || race
+// +build !invariants,!tracing race
+
+package cache
+
+import (
+	"runtime"
+	"sync"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/manual"
+)
+
+const (
+	entrySize            = int(unsafe.Sizeof(entry{}))
+	entryAllocCacheLimit = 128
+	// Avoid using runtime.SetFinalizer in race builds as finalizers tickle a bug
+	// in the Go race detector in go1.15 and earlier versions. This requires that
+	// entries are Go allocated rather than manually allocated.
+	//
+	// If cgo is disabled we need to allocate the entries using the Go allocator
+	// and is violates the Go GC rules to put Go pointers (such as the entry
+	// pointer fields) into untyped memory (i.e. a []byte).
+	entriesGoAllocated = invariants.RaceEnabled || !cgoEnabled
+)
+
+var entryAllocPool = sync.Pool{
+	New: func() interface{} {
+		return newEntryAllocCache()
+	},
+}
+
+func entryAllocNew() *entry {
+	a := entryAllocPool.Get().(*entryAllocCache)
+	e := a.alloc()
+	entryAllocPool.Put(a)
+	return e
+}
+
+func entryAllocFree(e *entry) {
+	a := entryAllocPool.Get().(*entryAllocCache)
+	a.free(e)
+	entryAllocPool.Put(a)
+}
+
+type entryAllocCache struct {
+	entries []*entry
+}
+
+func newEntryAllocCache() *entryAllocCache {
+	c := &entryAllocCache{}
+	if !entriesGoAllocated {
+		// Note the use of a "real" finalizer here (as opposed to a build tag-gated
+		// no-op finalizer). Without the finalizer, objects released from the pool
+		// and subsequently GC'd by the Go runtime would fail to have their manually
+		// allocated memory freed, which results in a memory leak.
+		// lint:ignore SetFinalizer
+		runtime.SetFinalizer(c, freeEntryAllocCache)
+	}
+	return c
+}
+
+func freeEntryAllocCache(obj interface{}) {
+	c := obj.(*entryAllocCache)
+	for i, e := range c.entries {
+		c.dealloc(e)
+		c.entries[i] = nil
+	}
+}
+
+func (c *entryAllocCache) alloc() *entry {
+	n := len(c.entries)
+	if n == 0 {
+		if entriesGoAllocated {
+			return &entry{}
+		}
+		b := manual.New(entrySize)
+		return (*entry)(unsafe.Pointer(&b[0]))
+	}
+	e := c.entries[n-1]
+	c.entries = c.entries[:n-1]
+	return e
+}
+
+func (c *entryAllocCache) dealloc(e *entry) {
+	if !entriesGoAllocated {
+		buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(e))[:entrySize:entrySize]
+		manual.Free(buf)
+	}
+}
+
+func (c *entryAllocCache) free(e *entry) {
+	if len(c.entries) == entryAllocCacheLimit {
+		c.dealloc(e)
+		return
+	}
+	c.entries = append(c.entries, e)
+}

vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_normal.go

@@ -0,0 +1,59 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !tracing
+// +build !tracing
+
+package cache
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/cockroachdb/redact"
+)
+
+// refcnt provides an atomic reference count. This version is used when the
+// "tracing" build tag is not enabled. See refcnt_tracing.go for the "tracing"
+// enabled version.
+type refcnt struct {
+	val atomic.Int32
+}
+
+// initialize the reference count to the specified value.
+func (v *refcnt) init(val int32) {
+	v.val.Store(val)
+}
+
+func (v *refcnt) refs() int32 {
+	return v.val.Load()
+}
+
+func (v *refcnt) acquire() {
+	switch v := v.val.Add(1); {
+	case v <= 1:
+		panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
+	}
+}
+
+func (v *refcnt) release() bool {
+	switch v := v.val.Add(-1); {
+	case v < 0:
+		panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
+	case v == 0:
+		return true
+	default:
+		return false
+	}
+}
+
+func (v *refcnt) trace(msg string) {
+}
+
+func (v *refcnt) traces() string {
+	return ""
+}
+
+// Silence unused warning.
+var _ = (*refcnt)(nil).traces

vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_tracing.go

@@ -0,0 +1,66 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build tracing
+// +build tracing
+
+package cache
+
+import (
+	"fmt"
+	"runtime/debug"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+// refcnt provides an atomic reference count, along with a tracing facility for
+// debugging logic errors in manipulating the reference count. This version is
+// used when the "tracing" build tag is enabled.
+type refcnt struct {
+	val atomic.Int32
+	sync.Mutex
+	msgs []string
+}
+
+func (v *refcnt) init(val int32) {
+	v.val.Store(val)
+	v.trace("init")
+}
+
+func (v *refcnt) refs() int32 {
+	return v.val.Load()
+}
+
+func (v *refcnt) acquire() {
+	switch n := v.val.Add(1); {
+	case n <= 1:
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
+	}
+	v.trace("acquire")
+}
+
+func (v *refcnt) release() bool {
+	n := v.val.Add(-1)
+	switch {
+	case n < 0:
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
+	}
+	v.trace("release")
+	return n == 0
+}
+
+func (v *refcnt) trace(msg string) {
+	s := fmt.Sprintf("%s: refs=%d\n%s", msg, v.refs(), debug.Stack())
+	v.Lock()
+	v.msgs = append(v.msgs, s)
+	v.Unlock()
+}
+
+func (v *refcnt) traces() string {
+	v.Lock()
+	s := strings.Join(v.msgs, "\n")
+	v.Unlock()
+	return s
+}

vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go

@@ -0,0 +1,320 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package cache
+
+import (
+	"fmt"
+	"math/bits"
+	"os"
+	"runtime/debug"
+	"strings"
+	"time"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/manual"
+)
+
+var hashSeed = uint64(time.Now().UnixNano())
+
+// Fibonacci hash: https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/
+func robinHoodHash(k key, shift uint32) uint32 {
+	const m = 11400714819323198485
+	h := hashSeed
+	h ^= k.id * m
+	h ^= uint64(k.fileNum.FileNum()) * m
+	h ^= k.offset * m
+	return uint32(h >> shift)
+}
+
+type robinHoodEntry struct {
+	key key
+	// Note that value may point to a Go allocated object (if the "invariants"
+	// build tag was specified), even though the memory for the entry itself is
+	// manually managed. This is technically a volation of the Cgo pointer rules:
+	//
+	//   https://golang.org/cmd/cgo/#hdr-Passing_pointers
+	//
+	// Specifically, Go pointers should not be stored in C allocated memory. The
+	// reason for this rule is that the Go GC will not look at C allocated memory
+	// to find pointers to Go objects. If the only reference to a Go object is
+	// stored in C allocated memory, the object will be reclaimed. What makes
+	// this "safe" is that the Cache guarantees that there are other pointers to
+	// the entry and shard which will keep them alive. In particular, every Go
+	// allocated entry in the cache is referenced by the shard.entries map. And
+	// every shard is referenced by the Cache.shards map.
+	value *entry
+	// The distance the entry is from its desired position.
+	dist uint32
+}
+
+type robinHoodEntries struct {
+	ptr unsafe.Pointer
+	len uint32
+}
+
+func newRobinHoodEntries(n uint32) robinHoodEntries {
+	size := uintptr(n) * unsafe.Sizeof(robinHoodEntry{})
+	return robinHoodEntries{
+		ptr: unsafe.Pointer(&(manual.New(int(size)))[0]),
+		len: n,
+	}
+}
+
+func (e robinHoodEntries) at(i uint32) *robinHoodEntry {
+	return (*robinHoodEntry)(unsafe.Pointer(uintptr(e.ptr) +
+		uintptr(i)*unsafe.Sizeof(robinHoodEntry{})))
+}
+
+func (e robinHoodEntries) free() {
+	size := uintptr(e.len) * unsafe.Sizeof(robinHoodEntry{})
+	buf := (*[manual.MaxArrayLen]byte)(e.ptr)[:size:size]
+	manual.Free(buf)
+}
+
+// robinHoodMap is an implementation of Robin Hood hashing. Robin Hood hashing
+// is an open-address hash table using linear probing. The twist is that the
+// linear probe distance is reduced by moving existing entries when inserting
+// and deleting. This is accomplished by keeping track of how far an entry is
+// from its "desired" slot (hash of key modulo number of slots). During
+// insertion, if the new entry being inserted is farther from its desired slot
+// than the target entry, we swap the target and new entry. This effectively
+// steals from the "rich" target entry and gives to the "poor" new entry (thus
+// the origin of the name).
+//
+// An extension over the base Robin Hood hashing idea comes from
+// https://probablydance.com/2017/02/26/i-wrote-the-fastest-hashtable/. A cap
+// is placed on the max distance an entry can be from its desired slot. When
+// this threshold is reached during insertion, the size of the table is doubled
+// and insertion is restarted. Additionally, the entries slice is given "max
+// dist" extra entries on the end. The very last entry in the entries slice is
+// never used and acts as a sentinel which terminates loops. The previous
+// maxDist-1 entries act as the extra entries. For example, if the size of the
+// table is 2, maxDist is computed as 4 and the actual size of the entry slice
+// is 6.
+//
+//	+---+---+---+---+---+---+
+//	| 0 | 1 | 2 | 3 | 4 | 5 |
+//	+---+---+---+---+---+---+
+//	        ^
+//	       size
+//
+// In this scenario, the target entry for a key will always be in the range
+// [0,1]. Valid entries may reside in the range [0,4] due to the linear probing
+// of up to maxDist entries. The entry at index 5 will never contain a value,
+// and instead acts as a sentinel (its distance is always 0). The max distance
+// threshold is set to log2(num-entries). This ensures that retrieval is O(log
+// N), though note that N is the number of total entries, not the count of
+// valid entries.
+//
+// Deletion is implemented via the backward shift delete mechanism instead of
+// tombstones. This preserves the performance of the table in the presence of
+// deletions. See
+// http://codecapsule.com/2013/11/17/robin-hood-hashing-backward-shift-deletion
+// for details.
+type robinHoodMap struct {
+	entries robinHoodEntries
+	size    uint32
+	shift   uint32
+	count   uint32
+	maxDist uint32
+}
+
+func maxDistForSize(size uint32) uint32 {
+	desired := uint32(bits.Len32(size))
+	if desired < 4 {
+		desired = 4
+	}
+	return desired
+}
+
+func newRobinHoodMap(initialCapacity int) *robinHoodMap {
+	m := &robinHoodMap{}
+	m.init(initialCapacity)
+
+	// Note: this is a no-op if invariants are disabled or race is enabled.
+	invariants.SetFinalizer(m, func(obj interface{}) {
+		m := obj.(*robinHoodMap)
+		if m.entries.ptr != nil {
+			fmt.Fprintf(os.Stderr, "%p: robin-hood map not freed\n", m)
+			os.Exit(1)
+		}
+	})
+	return m
+}
+
+func (m *robinHoodMap) init(initialCapacity int) {
+	if initialCapacity < 1 {
+		initialCapacity = 1
+	}
+	targetSize := 1 << (uint(bits.Len(uint(2*initialCapacity-1))) - 1)
+	m.rehash(uint32(targetSize))
+}
+
+func (m *robinHoodMap) free() {
+	if m.entries.ptr != nil {
+		m.entries.free()
+		m.entries.ptr = nil
+	}
+}
+
+func (m *robinHoodMap) rehash(size uint32) {
+	oldEntries := m.entries
+
+	m.size = size
+	m.shift = uint32(64 - bits.Len32(m.size-1))
+	m.maxDist = maxDistForSize(size)
+	m.entries = newRobinHoodEntries(size + m.maxDist)
+	m.count = 0
+
+	for i := uint32(0); i < oldEntries.len; i++ {
+		e := oldEntries.at(i)
+		if e.value != nil {
+			m.Put(e.key, e.value)
+		}
+	}
+
+	if oldEntries.ptr != nil {
+		oldEntries.free()
+	}
+}
+
+// Find an entry containing the specified value. This is intended to be used
+// from debug and test code.
+func (m *robinHoodMap) findByValue(v *entry) *robinHoodEntry {
+	for i := uint32(0); i < m.entries.len; i++ {
+		e := m.entries.at(i)
+		if e.value == v {
+			return e
+		}
+	}
+	return nil
+}
+
+func (m *robinHoodMap) Count() int {
+	return int(m.count)
+}
+
+func (m *robinHoodMap) Put(k key, v *entry) {
+	maybeExists := true
+	n := robinHoodEntry{key: k, value: v, dist: 0}
+	for i := robinHoodHash(k, m.shift); ; i++ {
+		e := m.entries.at(i)
+		if maybeExists && k == e.key {
+			// Entry already exists: overwrite.
+			e.value = n.value
+			m.checkEntry(i)
+			return
+		}
+
+		if e.value == nil {
+			// Found an empty entry: insert here.
+			*e = n
+			m.count++
+			m.checkEntry(i)
+			return
+		}
+
+		if e.dist < n.dist {
+			// Swap the new entry with the current entry because the current is
+			// rich. We then continue to loop, looking for a new location for the
+			// current entry. Note that this is also the not-found condition for
+			// retrieval, which means that "k" is not present in the map. See Get().
+			n, *e = *e, n
+			m.checkEntry(i)
+			maybeExists = false
+		}
+
+		// The new entry gradually moves away from its ideal position.
+		n.dist++
+
+		// If we've reached the max distance threshold, grow the table and restart
+		// the insertion.
+		if n.dist == m.maxDist {
+			m.rehash(2 * m.size)
+			i = robinHoodHash(n.key, m.shift) - 1
+			n.dist = 0
+			maybeExists = false
+		}
+	}
+}
+
+func (m *robinHoodMap) Get(k key) *entry {
+	var dist uint32
+	for i := robinHoodHash(k, m.shift); ; i++ {
+		e := m.entries.at(i)
+		if k == e.key {
+			// Found.
+			return e.value
+		}
+		if e.dist < dist {
+			// Not found.
+			return nil
+		}
+		dist++
+	}
+}
+
+func (m *robinHoodMap) Delete(k key) {
+	var dist uint32
+	for i := robinHoodHash(k, m.shift); ; i++ {
+		e := m.entries.at(i)
+		if k == e.key {
+			m.checkEntry(i)
+			// We found the entry to delete. Shift the following entries backwards
+			// until the next empty value or entry with a zero distance. Note that
+			// empty values are guaranteed to have "dist == 0".
+			m.count--
+			for j := i + 1; ; j++ {
+				t := m.entries.at(j)
+				if t.dist == 0 {
+					*e = robinHoodEntry{}
+					return
+				}
+				e.key = t.key
+				e.value = t.value
+				e.dist = t.dist - 1
+				e = t
+				m.checkEntry(j)
+			}
+		}
+		if dist > e.dist {
+			// Not found.
+			return
+		}
+		dist++
+	}
+}
+
+func (m *robinHoodMap) checkEntry(i uint32) {
+	if invariants.Enabled {
+		e := m.entries.at(i)
+		if e.value != nil {
+			pos := robinHoodHash(e.key, m.shift)
+			if (uint32(i) - pos) != e.dist {
+				fmt.Fprintf(os.Stderr, "%d: invalid dist=%d, expected %d: %s\n%s",
+					i, e.dist, uint32(i)-pos, e.key, debug.Stack())
+				os.Exit(1)
+			}
+			if e.dist > m.maxDist {
+				fmt.Fprintf(os.Stderr, "%d: invalid dist=%d > maxDist=%d: %s\n%s",
+					i, e.dist, m.maxDist, e.key, debug.Stack())
+				os.Exit(1)
+			}
+		}
+	}
+}
+
+func (m *robinHoodMap) String() string {
+	var buf strings.Builder
+	fmt.Fprintf(&buf, "count: %d\n", m.count)
+	for i := uint32(0); i < m.entries.len; i++ {
+		e := m.entries.at(i)
+		if e.value != nil {
+			fmt.Fprintf(&buf, "%d: [%s,%p,%d]\n", i, e.key, e.value, e.dist)
+		}
+	}
+	return buf.String()
+}

vendor/github.com/cockroachdb/pebble/internal/cache/value.go

@@ -0,0 +1,46 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package cache
+
+// Value holds a reference counted immutable value.
+type Value struct {
+	buf []byte
+	// Reference count for the value. The value is freed when the reference count
+	// drops to zero.
+	ref refcnt
+}
+
+// Buf returns the buffer associated with the value. The contents of the buffer
+// should not be changed once the value has been added to the cache. Instead, a
+// new Value should be created and added to the cache to replace the existing
+// value.
+func (v *Value) Buf() []byte {
+	if v == nil {
+		return nil
+	}
+	return v.buf
+}
+
+// Truncate the buffer to the specified length. The buffer length should not be
+// changed once the value has been added to the cache as there may be
+// concurrent readers of the Value. Instead, a new Value should be created and
+// added to the cache to replace the existing value.
+func (v *Value) Truncate(n int) {
+	v.buf = v.buf[:n]
+}
+
+func (v *Value) refs() int32 {
+	return v.ref.refs()
+}
+
+func (v *Value) acquire() {
+	v.ref.acquire()
+}
+
+func (v *Value) release() {
+	if v != nil && v.ref.release() {
+		v.free()
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/cache/value_invariants.go

@@ -0,0 +1,55 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build (invariants && !race) || (tracing && !race)
+// +build invariants,!race tracing,!race
+
+package cache
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/manual"
+)
+
+// newValue creates a Value with a manually managed buffer of size n.
+//
+// This definition of newValue is used when either the "invariants" or
+// "tracing" build tags are specified. It hooks up a finalizer to the returned
+// Value that checks for memory leaks when the GC determines the Value is no
+// longer reachable.
+func newValue(n int) *Value {
+	if n == 0 {
+		return nil
+	}
+	b := manual.New(n)
+	v := &Value{buf: b}
+	v.ref.init(1)
+	// Note: this is a no-op if invariants and tracing are disabled or race is
+	// enabled.
+	invariants.SetFinalizer(v, func(obj interface{}) {
+		v := obj.(*Value)
+		if v.buf != nil {
+			fmt.Fprintf(os.Stderr, "%p: cache value was not freed: refs=%d\n%s",
+				v, v.refs(), v.ref.traces())
+			os.Exit(1)
+		}
+	})
+	return v
+}
+
+func (v *Value) free() {
+	// When "invariants" are enabled set the value contents to 0xff in order to
+	// cache use-after-free bugs.
+	for i := range v.buf {
+		v.buf[i] = 0xff
+	}
+	manual.Free(v.buf)
+	// Setting Value.buf to nil is needed for correctness of the leak checking
+	// that is performed when the "invariants" or "tracing" build tags are
+	// enabled.
+	v.buf = nil
+}

vendor/github.com/cockroachdb/pebble/internal/cache/value_normal.go

@@ -0,0 +1,57 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build (!invariants && !tracing) || race
+// +build !invariants,!tracing race
+
+package cache
+
+import (
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/manual"
+)
+
+const valueSize = int(unsafe.Sizeof(Value{}))
+
+func newValue(n int) *Value {
+	if n == 0 {
+		return nil
+	}
+
+	if !cgoEnabled {
+		// If Cgo is disabled then all memory is allocated from the Go heap and we
+		// can't play the trick below to combine the Value and buffer allocation.
+		v := &Value{buf: make([]byte, n)}
+		v.ref.init(1)
+		return v
+	}
+
+	// When we're not performing leak detection, the lifetime of the returned
+	// Value is exactly the lifetime of the backing buffer and we can manually
+	// allocate both.
+	//
+	// TODO(peter): It may be better to separate the allocation of the value and
+	// the buffer in order to reduce internal fragmentation in malloc. If the
+	// buffer is right at a power of 2, adding valueSize might push the
+	// allocation over into the next larger size.
+	b := manual.New(valueSize + n)
+	v := (*Value)(unsafe.Pointer(&b[0]))
+	v.buf = b[valueSize:]
+	v.ref.init(1)
+	return v
+}
+
+func (v *Value) free() {
+	if !cgoEnabled {
+		return
+	}
+
+	// When we're not performing leak detection, the Value and buffer were
+	// allocated contiguously.
+	n := valueSize + cap(v.buf)
+	buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(v))[:n:n]
+	v.buf = nil
+	manual.Free(buf)
+}

vendor/github.com/cockroachdb/pebble/internal/constants/constants.go

@@ -0,0 +1,17 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package constants
+
+const (
+	// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
+	oneIf64Bit = ^uint(0) >> 63
+
+	// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
+	// - MaxUint32 on 64-bit platforms;
+	// - MaxInt on 32-bit platforms.
+	// It is used when slices are limited to Uint32 on 64-bit platforms (the
+	// length limit for slices is naturally MaxInt on 32-bit platforms).
+	MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
+)

vendor/github.com/cockroachdb/pebble/internal/crc/crc.go

@@ -0,0 +1,42 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+// Package crc implements the checksum algorithm used throughout pebble.
+//
+// The algorithm is CRC-32 with Castagnoli's polynomial, followed by a bit
+// rotation and an additional delta. The additional processing is to lessen the
+// probability of arbitrary key/value data coincidentally containing bytes that
+// look like a checksum.
+//
+// To calculate the uint32 checksum of some data:
+//
+//	var u uint32 = crc.New(data).Value()
+//
+// In pebble, the uint32 value is then stored in little-endian format.
+package crc // import "github.com/cockroachdb/pebble/internal/crc"
+
+import "hash/crc32"
+
+var table = crc32.MakeTable(crc32.Castagnoli)
+
+// CRC is a small convenience wrapper for computing the CRC32 checksum used by
+// pebble. This is the same algorithm as used by RocksDB.
+type CRC uint32
+
+// New returns the result of adding the bytes to the zero-value CRC.
+func New(b []byte) CRC {
+	return CRC(0).Update(b)
+}
+
+// Update returns the result of adding the bytes to the CRC.
+func (c CRC) Update(b []byte) CRC {
+	return CRC(crc32.Update(uint32(c), table, b))
+}
+
+// Value returns the cooked CRC value. The additional processing is to lessen
+// the probability of arbitrary key/value data coincidentally containing bytes
+// that look like a checksum.
+func (c CRC) Value() uint32 {
+	return uint32(c>>15|c<<17) + 0xa282ead8
+}

vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go

@@ -0,0 +1,17 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package fastrand
+
+import _ "unsafe" // required by go:linkname
+
+// Uint32 returns a lock free uint32 value.
+//
+//go:linkname Uint32 runtime.fastrand
+func Uint32() uint32
+
+// Uint32n returns a lock free uint32 value in the interval [0, n).
+//
+//go:linkname Uint32n runtime.fastrandn
+func Uint32n(n uint32) uint32

vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go

@@ -0,0 +1,68 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package humanize
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/cockroachdb/redact"
+)
+
+func logn(n, b float64) float64 {
+	return math.Log(n) / math.Log(b)
+}
+
+func humanate(s uint64, base float64, suffixes []string) string {
+	if s < 10 {
+		return fmt.Sprintf("%d%s", s, suffixes[0])
+	}
+	e := math.Floor(logn(float64(s), base))
+	suffix := suffixes[int(e)]
+	val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10
+	f := "%.0f%s"
+	if val < 10 {
+		f = "%.1f%s"
+	}
+
+	return fmt.Sprintf(f, val, suffix)
+}
+
+type config struct {
+	base   float64
+	suffix []string
+}
+
+// Bytes produces human readable representations of byte values in IEC units.
+var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
+
+// Count produces human readable representations of unitless values in SI units.
+var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
+
+// Int64 produces a human readable representation of the value.
+func (c *config) Int64(s int64) FormattedString {
+	if s < 0 {
+		return FormattedString("-" + humanate(uint64(-s), c.base, c.suffix))
+	}
+	return FormattedString(humanate(uint64(s), c.base, c.suffix))
+}
+
+// Uint64 produces a human readable representation of the value.
+func (c *config) Uint64(s uint64) FormattedString {
+	return FormattedString(humanate(s, c.base, c.suffix))
+}
+
+// FormattedString represents a human readable representation of a value. It
+// implements the redact.SafeValue interface to signal that it represents a
+// a string that does not need to be redacted.
+type FormattedString string
+
+var _ redact.SafeValue = FormattedString("")
+
+// SafeValue implements redact.SafeValue.
+func (fs FormattedString) SafeValue() {}
+
+// String implements fmt.Stringer.
+func (fs FormattedString) String() string { return string(fs) }

vendor/github.com/cockroachdb/pebble/internal/intern/intern.go

@@ -0,0 +1,27 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package intern
+
+import "sync"
+
+var pool = sync.Pool{
+	New: func() interface{} {
+		return make(map[string]string)
+	},
+}
+
+// Bytes returns b converted to a string, interned.
+func Bytes(b []byte) string {
+	m := pool.Get().(map[string]string)
+	c, ok := m[string(b)]
+	if ok {
+		pool.Put(m)
+		return c
+	}
+	s := string(b)
+	m[s] = s
+	pool.Put(m)
+	return s
+}

vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package invalidating
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/fastrand"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
+// MaybeWrapIfInvariants does nothing in non-invariant builds.
+func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
+	if invariants.Enabled {
+		if fastrand.Uint32n(10) == 1 {
+			return NewIter(iter)
+		}
+	}
+	return iter
+}
+
+// iter tests unsafe key/value slice reuse by modifying the last
+// returned key/value to all 1s.
+type iter struct {
+	iter        base.InternalIterator
+	lastKey     *base.InternalKey
+	lastValue   base.LazyValue
+	ignoreKinds [base.InternalKeyKindMax + 1]bool
+	err         error
+}
+
+// Option configures the behavior of an invalidating iterator.
+type Option interface {
+	apply(*iter)
+}
+
+type funcOpt func(*iter)
+
+func (f funcOpt) apply(i *iter) { f(i) }
+
+// IgnoreKinds constructs an Option that configures an invalidating iterator to
+// skip trashing k/v pairs with the provided key kinds. Some iterators provided
+// key stability guarantees for specific key kinds.
+func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
+	return funcOpt(func(i *iter) {
+		for _, kind := range kinds {
+			i.ignoreKinds[kind] = true
+		}
+	})
+}
+
+// NewIter constructs a new invalidating iterator that wraps the provided
+// iterator, trashing buffers for previously returned keys.
+func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
+	i := &iter{iter: originalIterator}
+	for _, opt := range opts {
+		opt.apply(i)
+	}
+	return i
+}
+
+func (i *iter) update(
+	key *base.InternalKey, value base.LazyValue,
+) (*base.InternalKey, base.LazyValue) {
+	i.trashLastKV()
+	if key == nil {
+		i.lastKey = nil
+		i.lastValue = base.LazyValue{}
+		return nil, base.LazyValue{}
+	}
+
+	i.lastKey = &base.InternalKey{}
+	*i.lastKey = key.Clone()
+	i.lastValue = base.LazyValue{
+		ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
+	}
+	if value.Fetcher != nil {
+		fetcher := new(base.LazyFetcher)
+		*fetcher = *value.Fetcher
+		i.lastValue.Fetcher = fetcher
+	}
+	return i.lastKey, i.lastValue
+}
+
+func (i *iter) trashLastKV() {
+	if i.lastKey == nil {
+		return
+	}
+	if i.ignoreKinds[i.lastKey.Kind()] {
+		return
+	}
+
+	if i.lastKey != nil {
+		for j := range i.lastKey.UserKey {
+			i.lastKey.UserKey[j] = 0xff
+		}
+		i.lastKey.Trailer = 0xffffffffffffffff
+	}
+	for j := range i.lastValue.ValueOrHandle {
+		i.lastValue.ValueOrHandle[j] = 0xff
+	}
+	if i.lastValue.Fetcher != nil {
+		// Not all the LazyFetcher fields are visible, so we zero out the last
+		// value's Fetcher struct entirely.
+		*i.lastValue.Fetcher = base.LazyFetcher{}
+	}
+}
+
+func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekGE(key, flags))
+}
+
+func (i *iter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
+}
+
+func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekLT(key, flags))
+}
+
+func (i *iter) First() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.First())
+}
+
+func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Last())
+}
+
+func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Next())
+}
+
+func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Prev())
+}
+
+func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.NextPrefix(succKey))
+}
+
+func (i *iter) Error() error {
+	if err := i.iter.Error(); err != nil {
+		return err
+	}
+	return i.err
+}
+
+func (i *iter) Close() error {
+	return i.iter.Close()
+}
+
+func (i *iter) SetBounds(lower, upper []byte) {
+	i.iter.SetBounds(lower, upper)
+}
+
+func (i *iter) String() string {
+	return i.iter.String()
+}

vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_off.go

@@ -0,0 +1,14 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build (!invariants && !tracing) || race
+// +build !invariants,!tracing race
+
+package invariants
+
+// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
+// race builds or if neither the invariants or tracing build tags are
+// specified.
+func SetFinalizer(obj, finalizer interface{}) {
+}

vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_on.go

@@ -0,0 +1,17 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build (invariants && !race) || (tracing && !race)
+// +build invariants,!race tracing,!race
+
+package invariants
+
+import "runtime"
+
+// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
+// race builds or if neither the invariants or tracing build tags are
+// specified.
+func SetFinalizer(obj, finalizer interface{}) {
+	runtime.SetFinalizer(obj, finalizer)
+}

vendor/github.com/cockroachdb/pebble/internal/invariants/off.go

@@ -0,0 +1,11 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !invariants && !race
+// +build !invariants,!race
+
+package invariants
+
+// Enabled is true if we were built with the "invariants" or "race" build tags.
+const Enabled = false

vendor/github.com/cockroachdb/pebble/internal/invariants/on.go

@@ -0,0 +1,11 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build invariants || race
+// +build invariants race
+
+package invariants
+
+// Enabled is true if we were built with the "invariants" or "race" build tags.
+const Enabled = true

vendor/github.com/cockroachdb/pebble/internal/invariants/race_off.go

@@ -0,0 +1,11 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !race
+// +build !race
+
+package invariants
+
+// RaceEnabled is true if we were built with the "race" build tag.
+const RaceEnabled = false

vendor/github.com/cockroachdb/pebble/internal/invariants/race_on.go

@@ -0,0 +1,11 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build race
+// +build race
+
+package invariants
+
+// RaceEnabled is true if we were built with the "race" build tag.
+const RaceEnabled = true

vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go

@@ -0,0 +1,268 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// TODO(jackson): Consider removing this type and adding bounds enforcement
+// directly to the MergingIter. This type is probably too lightweight to warrant
+// its own type, but for now we implement it separately for expediency.
+
+// boundedIterPos records the position of the BoundedIter relative to the
+// underlying iterator's position. It's used to avoid Next/Prev-ing the iterator
+// if there can't possibly be another span within bounds, because the current
+// span overlaps the bound.
+//
+// Imagine bounds [a,c) and an iterator that seeks to a span [b,d). The span
+// [b,d) overlaps some portion of the iterator bounds, so the iterator must
+// return it. If the iterator is subsequently Nexted, Next can tell that the
+// iterator is exhausted without advancing the underlying iterator because the
+// current span's end bound of d is ≥ the upper bound of c. In this case, the
+// bounded iterator returns nil and records i.pos as posAtUpperLimit to remember
+// that the underlying iterator position does not match the current BoundedIter
+// position.
+type boundedIterPos int8
+
+const (
+	posAtLowerLimit boundedIterPos = -1
+	posAtIterSpan   boundedIterPos = 0
+	posAtUpperLimit boundedIterPos = +1
+)
+
+// BoundedIter implements FragmentIterator and enforces bounds.
+//
+// Like the point InternalIterator interface, the bounded iterator's forward
+// positioning routines (SeekGE, First, and Next) only check the upper bound.
+// The reverse positioning routines (SeekLT, Last, and Prev) only check the
+// lower bound. It is up to the caller to ensure that the forward positioning
+// routines respect the lower bound and the reverse positioning routines respect
+// the upper bound (i.e. calling SeekGE instead of First if there is a lower
+// bound, and SeekLT instead of Last if there is an upper bound).
+//
+// When the hasPrefix parameter indicates that the iterator is in prefix
+// iteration mode, BoundedIter elides any spans that do not overlap with the
+// prefix's keyspace. In prefix iteration mode, reverse iteration is disallowed,
+// except for an initial SeekLT with a seek key greater than or equal to the
+// prefix. In prefix iteration mode, the first seek must position the iterator
+// at or immediately before the first fragment covering a key greater than or
+// equal to the prefix.
+type BoundedIter struct {
+	iter      FragmentIterator
+	iterSpan  *Span
+	cmp       base.Compare
+	split     base.Split
+	lower     []byte
+	upper     []byte
+	hasPrefix *bool
+	prefix    *[]byte
+	pos       boundedIterPos
+}
+
+// Init initializes the bounded iterator.
+//
+// In addition to the iterator bounds, Init takes pointers to a boolean
+// indicating whether the iterator is in prefix iteration mode and the prefix
+// key if it is. This is used to exclude spans that are outside the iteration
+// prefix.
+//
+// hasPrefix and prefix are allowed to be nil, however if hasPrefix != nil,
+// prefix must also not be nil.
+func (i *BoundedIter) Init(
+	cmp base.Compare,
+	split base.Split,
+	iter FragmentIterator,
+	lower, upper []byte,
+	hasPrefix *bool,
+	prefix *[]byte,
+) {
+	*i = BoundedIter{
+		iter:      iter,
+		cmp:       cmp,
+		split:     split,
+		lower:     lower,
+		upper:     upper,
+		hasPrefix: hasPrefix,
+		prefix:    prefix,
+	}
+}
+
+var _ FragmentIterator = (*BoundedIter)(nil)
+
+// Seek calls.
+//
+// Seek calls check iterator bounds in the direction of the seek. Additionally,
+// if the iterator is in prefix iteration mode, seek calls check both start and
+// end bounds against the prefix's bounds. We check both bounds for defense in
+// depth. This optimization has been a source of various bugs due to various
+// other prefix iteration optimizations that can result in seek keys that don't
+// respect the prefix bounds.
+
+// SeekGE implements FragmentIterator.
+func (i *BoundedIter) SeekGE(key []byte) *Span {
+	s := i.iter.SeekGE(key)
+	s = i.checkPrefixSpanStart(s)
+	s = i.checkPrefixSpanEnd(s)
+	return i.checkForwardBound(s)
+}
+
+// SeekLT implements FragmentIterator.
+func (i *BoundedIter) SeekLT(key []byte) *Span {
+	s := i.iter.SeekLT(key)
+	s = i.checkPrefixSpanStart(s)
+	s = i.checkPrefixSpanEnd(s)
+	return i.checkBackwardBound(s)
+}
+
+// First implements FragmentIterator.
+func (i *BoundedIter) First() *Span {
+	s := i.iter.First()
+	s = i.checkPrefixSpanStart(s)
+	return i.checkForwardBound(s)
+}
+
+// Last implements FragmentIterator.
+func (i *BoundedIter) Last() *Span {
+	s := i.iter.Last()
+	s = i.checkPrefixSpanEnd(s)
+	return i.checkBackwardBound(s)
+}
+
+// Next implements FragmentIterator.
+func (i *BoundedIter) Next() *Span {
+	switch i.pos {
+	case posAtLowerLimit:
+		// The BoundedIter had previously returned nil, because it knew from
+		// i.iterSpan's bounds that there was no previous span. To Next, we only
+		// need to return the current iter span and reset i.pos to reflect that
+		// we're no longer positioned at the limit.
+		i.pos = posAtIterSpan
+		return i.iterSpan
+	case posAtIterSpan:
+		// If the span at the underlying iterator position extends to or beyond the
+		// upper bound, we can avoid advancing because the next span is necessarily
+		// out of bounds.
+		if i.iterSpan != nil && i.upper != nil && i.cmp(i.iterSpan.End, i.upper) >= 0 {
+			i.pos = posAtUpperLimit
+			return nil
+		}
+		// Similarly, if the span extends to the next prefix and we're in prefix
+		// iteration mode, we can avoid advancing.
+		if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
+			ei := i.split(i.iterSpan.End)
+			if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 {
+				i.pos = posAtUpperLimit
+				return nil
+			}
+		}
+		return i.checkForwardBound(i.checkPrefixSpanStart(i.iter.Next()))
+	case posAtUpperLimit:
+		// Already exhausted.
+		return nil
+	default:
+		panic("unreachable")
+	}
+}
+
+// Prev implements FragmentIterator.
+func (i *BoundedIter) Prev() *Span {
+	switch i.pos {
+	case posAtLowerLimit:
+		// Already exhausted.
+		return nil
+	case posAtIterSpan:
+		// If the span at the underlying iterator position extends to or beyond
+		// the lower bound, we can avoid advancing because the previous span is
+		// necessarily out of bounds.
+		if i.iterSpan != nil && i.lower != nil && i.cmp(i.iterSpan.Start, i.lower) <= 0 {
+			i.pos = posAtLowerLimit
+			return nil
+		}
+		// Similarly, if the span extends to or beyond the current prefix and
+		// we're in prefix iteration mode, we can avoid advancing.
+		if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
+			si := i.split(i.iterSpan.Start)
+			if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 {
+				i.pos = posAtLowerLimit
+				return nil
+			}
+		}
+		return i.checkBackwardBound(i.checkPrefixSpanEnd(i.iter.Prev()))
+	case posAtUpperLimit:
+		// The BoundedIter had previously returned nil, because it knew from
+		// i.iterSpan's bounds that there was no next span. To Prev, we only
+		// need to return the current iter span and reset i.pos to reflect that
+		// we're no longer positioned at the limit.
+		i.pos = posAtIterSpan
+		return i.iterSpan
+	default:
+		panic("unreachable")
+	}
+}
+
+// Error implements FragmentIterator.
+func (i *BoundedIter) Error() error {
+	return i.iter.Error()
+}
+
+// Close implements FragmentIterator.
+func (i *BoundedIter) Close() error {
+	return i.iter.Close()
+}
+
+// SetBounds modifies the FragmentIterator's bounds.
+func (i *BoundedIter) SetBounds(lower, upper []byte) {
+	i.lower, i.upper = lower, upper
+}
+
+func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span {
+	// Compare to the prefix's bounds, if in prefix iteration mode.
+	if span != nil && i.hasPrefix != nil && *i.hasPrefix {
+		si := i.split(span.Start)
+		if i.cmp(span.Start[:si], *i.prefix) > 0 {
+			// This span starts at a prefix that sorts after our current prefix.
+			span = nil
+		}
+	}
+	return span
+}
+
+// checkForwardBound enforces the upper bound, returning nil if the provided
+// span is wholly outside the upper bound. It also updates i.pos and i.iterSpan
+// to reflect the new iterator position.
+func (i *BoundedIter) checkForwardBound(span *Span) *Span {
+	// Compare to the upper bound.
+	if span != nil && i.upper != nil && i.cmp(span.Start, i.upper) >= 0 {
+		span = nil
+	}
+	i.iterSpan = span
+	if i.pos != posAtIterSpan {
+		i.pos = posAtIterSpan
+	}
+	return span
+}
+
+func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span {
+	// Compare to the prefix's bounds, if in prefix iteration mode.
+	if span != nil && i.hasPrefix != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
+		// This span ends before the current prefix.
+		span = nil
+	}
+	return span
+}
+
+// checkBackward enforces the lower bound, returning nil if the provided span is
+// wholly outside the lower bound.  It also updates i.pos and i.iterSpan to
+// reflect the new iterator position.
+func (i *BoundedIter) checkBackwardBound(span *Span) *Span {
+	// Compare to the lower bound.
+	if span != nil && i.lower != nil && i.cmp(span.End, i.lower) <= 0 {
+		span = nil
+	}
+	i.iterSpan = span
+	if i.pos != posAtIterSpan {
+		i.pos = posAtIterSpan
+	}
+	return span
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -0,0 +1,539 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import (
+	"bytes"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/bytealloc"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
+// that DefragmentingIter will reuse. Buffers larger than this will be
+// discarded and reallocated as necessary.
+const bufferReuseMaxCapacity = 10 << 10 // 10 KB
+
+// keysReuseMaxCapacity is the maximum capacity of a []keyspan.Key buffer that
+// DefragmentingIter will reuse. Buffers larger than this will be discarded and
+// reallocated as necessary.
+const keysReuseMaxCapacity = 100
+
+// DefragmentMethod configures the defragmentation performed by the
+// DefragmentingIter.
+type DefragmentMethod interface {
+	// ShouldDefragment takes two abutting spans and returns whether the two
+	// spans should be combined into a single, defragmented Span.
+	ShouldDefragment(equal base.Equal, left, right *Span) bool
+}
+
+// The DefragmentMethodFunc type is an adapter to allow the use of ordinary
+// functions as DefragmentMethods. If f is a function with the appropriate
+// signature, DefragmentMethodFunc(f) is a DefragmentMethod that calls f.
+type DefragmentMethodFunc func(equal base.Equal, left, right *Span) bool
+
+// ShouldDefragment calls f(equal, left, right).
+func (f DefragmentMethodFunc) ShouldDefragment(equal base.Equal, left, right *Span) bool {
+	return f(equal, left, right)
+}
+
+// DefragmentInternal configures a DefragmentingIter to defragment spans
+// only if they have identical keys. It requires spans' keys to be sorted in
+// trailer descending order.
+//
+// This defragmenting method is intended for use in compactions that may see
+// internal range keys fragments that may now be joined, because the state that
+// required their fragmentation has been dropped.
+var DefragmentInternal DefragmentMethod = DefragmentMethodFunc(func(equal base.Equal, a, b *Span) bool {
+	if a.KeysOrder != ByTrailerDesc || b.KeysOrder != ByTrailerDesc {
+		panic("pebble: span keys unexpectedly not in trailer descending order")
+	}
+	if len(a.Keys) != len(b.Keys) {
+		return false
+	}
+	for i := range a.Keys {
+		if a.Keys[i].Trailer != b.Keys[i].Trailer {
+			return false
+		}
+		if !equal(a.Keys[i].Suffix, b.Keys[i].Suffix) {
+			return false
+		}
+		if !bytes.Equal(a.Keys[i].Value, b.Keys[i].Value) {
+			return false
+		}
+	}
+	return true
+})
+
+// DefragmentReducer merges the current and next Key slices, returning a new Key
+// slice.
+//
+// Implementations should modify and return `cur` to save on allocations, or
+// consider allocating a new slice, as the `cur` slice may be retained by the
+// DefragmentingIter and mutated. The `next` slice must not be mutated.
+//
+// The incoming slices are sorted by (SeqNum, Kind) descending. The output slice
+// must also have this sort order.
+type DefragmentReducer func(cur, next []Key) []Key
+
+// StaticDefragmentReducer is a no-op DefragmentReducer that simply returns the
+// current key slice, effectively retaining the first set of keys encountered
+// for a defragmented span.
+//
+// This reducer can be used, for example, when the set of Keys for each Span
+// being reduced is not expected to change, and therefore the keys from the
+// first span encountered can be used without considering keys in subsequent
+// spans.
+var StaticDefragmentReducer DefragmentReducer = func(cur, _ []Key) []Key {
+	return cur
+}
+
+// iterPos is an enum indicating the position of the defragmenting iter's
+// wrapped iter. The defragmenting iter must look ahead or behind when
+// defragmenting forward or backwards respectively, and this enum records that
+// current position.
+type iterPos int8
+
+const (
+	iterPosPrev iterPos = -1
+	iterPosCurr iterPos = 0
+	iterPosNext iterPos = +1
+)
+
+// DefragmentingIter wraps a key span iterator, defragmenting physical
+// fragmentation during iteration.
+//
+// During flushes and compactions, keys applied over a span may be split at
+// sstable boundaries. This fragmentation can produce internal key bounds that
+// do not match any of the bounds ever supplied to a user operation. This
+// physical fragmentation is necessary to avoid excessively wide sstables.
+//
+// The defragmenting iterator undoes this physical fragmentation, joining spans
+// with abutting bounds and equal state. The defragmenting iterator takes a
+// DefragmentMethod to determine what is "equal state" for a span. The
+// DefragmentMethod is a function type, allowing arbitrary comparisons between
+// Span keys.
+//
+// Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may
+// land on a physical fragment in the middle of several fragments that must be
+// defragmented. A seek that lands in a fragment straddling the seek key must
+// first degfragment in the opposite direction of iteration to find the
+// beginning of the defragmented span, and then defragments in the iteration
+// direction, ensuring it's found a whole defragmented span.
+type DefragmentingIter struct {
+	// DefragmentingBuffers holds buffers used for copying iterator state.
+	*DefragmentingBuffers
+	comparer *base.Comparer
+	equal    base.Equal
+	iter     FragmentIterator
+	iterSpan *Span
+	iterPos  iterPos
+
+	// curr holds the span at the current iterator position.
+	curr Span
+
+	// method is a comparison function for two spans. method is called when two
+	// spans are abutting to determine whether they may be defragmented.
+	// method does not itself check for adjacency for the two spans.
+	method DefragmentMethod
+
+	// reduce is the reducer function used to collect Keys across all spans that
+	// constitute a defragmented span.
+	reduce DefragmentReducer
+}
+
+// DefragmentingBuffers holds buffers used for copying iterator state.
+type DefragmentingBuffers struct {
+	// currBuf is a buffer for use when copying user keys for curr. currBuf is
+	// cleared between positioning methods.
+	currBuf bytealloc.A
+	// keysBuf is a buffer for use when copying Keys for DefragmentingIter.curr.
+	keysBuf []Key
+	// keyBuf is a buffer specifically for the defragmented start key when
+	// defragmenting backwards or the defragmented end key when defragmenting
+	// forwards. These bounds are overwritten repeatedly during defragmentation,
+	// and the defragmentation routines overwrite keyBuf repeatedly to store
+	// these extended bounds.
+	keyBuf []byte
+}
+
+// PrepareForReuse discards any excessively large buffers.
+func (bufs *DefragmentingBuffers) PrepareForReuse() {
+	if cap(bufs.currBuf) > bufferReuseMaxCapacity {
+		bufs.currBuf = nil
+	}
+	if cap(bufs.keyBuf) > bufferReuseMaxCapacity {
+		bufs.keyBuf = nil
+	}
+	if cap(bufs.keysBuf) > keysReuseMaxCapacity {
+		bufs.keysBuf = nil
+	}
+}
+
+// Assert that *DefragmentingIter implements the FragmentIterator interface.
+var _ FragmentIterator = (*DefragmentingIter)(nil)
+
+// Init initializes the defragmenting iter using the provided defragment
+// method.
+func (i *DefragmentingIter) Init(
+	comparer *base.Comparer,
+	iter FragmentIterator,
+	equal DefragmentMethod,
+	reducer DefragmentReducer,
+	bufs *DefragmentingBuffers,
+) {
+	*i = DefragmentingIter{
+		DefragmentingBuffers: bufs,
+		comparer:             comparer,
+		equal:                comparer.Equal,
+		iter:                 iter,
+		method:               equal,
+		reduce:               reducer,
+	}
+}
+
+// Error returns any accumulated error.
+func (i *DefragmentingIter) Error() error {
+	return i.iter.Error()
+}
+
+// Close closes the underlying iterators.
+func (i *DefragmentingIter) Close() error {
+	return i.iter.Close()
+}
+
+// SeekGE moves the iterator to the first span covering a key greater than or
+// equal to the given key. This is equivalent to seeking to the first span with
+// an end key greater than the given key.
+func (i *DefragmentingIter) SeekGE(key []byte) *Span {
+	i.iterSpan = i.iter.SeekGE(key)
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	} else if i.iterSpan.Empty() {
+		i.iterPos = iterPosCurr
+		return i.iterSpan
+	}
+	// If the span starts strictly after key, we know there mustn't be an
+	// earlier span that ends at i.iterSpan.Start, otherwise i.iter would've
+	// returned that span instead.
+	if i.comparer.Compare(i.iterSpan.Start, key) > 0 {
+		return i.defragmentForward()
+	}
+
+	// The span we landed on has a Start bound ≤ key. There may be additional
+	// fragments before this span. Defragment backward to find the start of the
+	// defragmented span.
+	i.defragmentBackward()
+
+	// Defragmenting backward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
+	if i.iterPos == iterPosPrev {
+		// Next once back onto the span.
+		i.iterSpan = i.iter.Next()
+	}
+	// Defragment the full span from its start.
+	return i.defragmentForward()
+}
+
+// SeekLT moves the iterator to the last span covering a key less than the
+// given key. This is equivalent to seeking to the last span with a start
+// key less than the given key.
+func (i *DefragmentingIter) SeekLT(key []byte) *Span {
+	i.iterSpan = i.iter.SeekLT(key)
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	} else if i.iterSpan.Empty() {
+		i.iterPos = iterPosCurr
+		return i.iterSpan
+	}
+	// If the span ends strictly before key, we know there mustn't be a later
+	// span that starts at i.iterSpan.End, otherwise i.iter would've returned
+	// that span instead.
+	if i.comparer.Compare(i.iterSpan.End, key) < 0 {
+		return i.defragmentBackward()
+	}
+
+	// The span we landed on has a End bound ≥ key. There may be additional
+	// fragments after this span. Defragment forward to find the end of the
+	// defragmented span.
+	i.defragmentForward()
+
+	// Defragmenting forward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
+	if i.iterPos == iterPosNext {
+		// Prev once back onto the span.
+		i.iterSpan = i.iter.Prev()
+	}
+	// Defragment the full span from its end.
+	return i.defragmentBackward()
+}
+
+// First seeks the iterator to the first span and returns it.
+func (i *DefragmentingIter) First() *Span {
+	i.iterSpan = i.iter.First()
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	}
+	return i.defragmentForward()
+}
+
+// Last seeks the iterator to the last span and returns it.
+func (i *DefragmentingIter) Last() *Span {
+	i.iterSpan = i.iter.Last()
+	if i.iterSpan == nil {
+		i.iterPos = iterPosCurr
+		return nil
+	}
+	return i.defragmentBackward()
+}
+
+// Next advances to the next span and returns it.
+func (i *DefragmentingIter) Next() *Span {
+	switch i.iterPos {
+	case iterPosPrev:
+		// Switching directions; The iterator is currently positioned over the
+		// last span of the previous set of fragments. In the below diagram,
+		// the iterator is positioned over the last span that contributes to
+		// the defragmented x position. We want to be positioned over the first
+		// span that contributes to the z position.
+		//
+		//   x x x y y y z z z
+		//       ^       ^
+		//      old     new
+		//
+		// Next once to move onto y, defragment forward to land on the first z
+		// position.
+		i.iterSpan = i.iter.Next()
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
+			panic("pebble: invariant violation: no next span while switching directions")
+		}
+		// We're now positioned on the first span that was defragmented into the
+		// current iterator position. Skip over the rest of the current iterator
+		// position's constitutent fragments. In the above example, this would
+		// land on the first 'z'.
+		i.defragmentForward()
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
+
+		// Now that we're positioned over the first of the next set of
+		// fragments, defragment forward.
+		return i.defragmentForward()
+	case iterPosCurr:
+		// iterPosCurr is only used when the iter is exhausted or when the iterator
+		// is at an empty span.
+		if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
+			panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
+		}
+
+		i.iterSpan = i.iter.Next()
+		if i.iterSpan == nil {
+			return nil
+		}
+		return i.defragmentForward()
+	case iterPosNext:
+		// Already at the next span.
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
+		return i.defragmentForward()
+	default:
+		panic("unreachable")
+	}
+}
+
+// Prev steps back to the previous span and returns it.
+func (i *DefragmentingIter) Prev() *Span {
+	switch i.iterPos {
+	case iterPosPrev:
+		// Already at the previous span.
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
+		return i.defragmentBackward()
+	case iterPosCurr:
+		// iterPosCurr is only used when the iter is exhausted or when the iterator
+		// is at an empty span.
+		if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
+			panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
+		}
+
+		i.iterSpan = i.iter.Prev()
+		if i.iterSpan == nil {
+			return nil
+		}
+		return i.defragmentBackward()
+	case iterPosNext:
+		// Switching directions; The iterator is currently positioned over the
+		// first fragment of the next set of fragments. In the below diagram,
+		// the iterator is positioned over the first span that contributes to
+		// the defragmented z position. We want to be positioned over the last
+		// span that contributes to the x position.
+		//
+		//   x x x y y y z z z
+		//       ^       ^
+		//      new     old
+		//
+		// Prev once to move onto y, defragment backward to land on the last x
+		// position.
+		i.iterSpan = i.iter.Prev()
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
+			panic("pebble: invariant violation: no previous span while switching directions")
+		}
+		// We're now positioned on the last span that was defragmented into the
+		// current iterator position. Skip over the rest of the current iterator
+		// position's constitutent fragments. In the above example, this would
+		// land on the last 'x'.
+		i.defragmentBackward()
+
+		// Now that we're positioned over the last of the prev set of
+		// fragments, defragment backward.
+		if i.iterSpan == nil {
+			i.iterPos = iterPosCurr
+			return nil
+		}
+		return i.defragmentBackward()
+	default:
+		panic("unreachable")
+	}
+}
+
+// checkEqual checks the two spans for logical equivalence. It uses the passed-in
+// DefragmentMethod and ensures both spans are NOT empty; not defragmenting empty
+// spans is an optimization that lets us load fewer sstable blocks.
+func (i *DefragmentingIter) checkEqual(left, right *Span) bool {
+	return (!left.Empty() && !right.Empty()) && i.method.ShouldDefragment(i.equal, i.iterSpan, &i.curr)
+}
+
+// defragmentForward defragments spans in the forward direction, starting from
+// i.iter's current position. The span at the current position must be non-nil,
+// but may be Empty().
+func (i *DefragmentingIter) defragmentForward() *Span {
+	if i.iterSpan.Empty() {
+		// An empty span will never be equal to another span; see checkEqual for
+		// why. To avoid loading non-empty range keys further ahead by calling Next,
+		// return early.
+		i.iterPos = iterPosCurr
+		return i.iterSpan
+	}
+	i.saveCurrent()
+
+	i.iterPos = iterPosNext
+	i.iterSpan = i.iter.Next()
+	for i.iterSpan != nil {
+		if !i.equal(i.curr.End, i.iterSpan.Start) {
+			// Not a continuation.
+			break
+		}
+		if !i.checkEqual(i.iterSpan, &i.curr) {
+			// Not a continuation.
+			break
+		}
+		i.keyBuf = append(i.keyBuf[:0], i.iterSpan.End...)
+		i.curr.End = i.keyBuf
+		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
+		i.iterSpan = i.iter.Next()
+	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
+	i.curr.Keys = i.keysBuf
+	return &i.curr
+}
+
+// defragmentBackward defragments spans in the backward direction, starting from
+// i.iter's current position. The span at the current position must be non-nil,
+// but may be Empty().
+func (i *DefragmentingIter) defragmentBackward() *Span {
+	if i.iterSpan.Empty() {
+		// An empty span will never be equal to another span; see checkEqual for
+		// why. To avoid loading non-empty range keys further ahead by calling Next,
+		// return early.
+		i.iterPos = iterPosCurr
+		return i.iterSpan
+	}
+	i.saveCurrent()
+
+	i.iterPos = iterPosPrev
+	i.iterSpan = i.iter.Prev()
+	for i.iterSpan != nil {
+		if !i.equal(i.curr.Start, i.iterSpan.End) {
+			// Not a continuation.
+			break
+		}
+		if !i.checkEqual(i.iterSpan, &i.curr) {
+			// Not a continuation.
+			break
+		}
+		i.keyBuf = append(i.keyBuf[:0], i.iterSpan.Start...)
+		i.curr.Start = i.keyBuf
+		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
+		i.iterSpan = i.iter.Prev()
+	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
+	i.curr.Keys = i.keysBuf
+	return &i.curr
+}
+
+func (i *DefragmentingIter) saveCurrent() {
+	i.currBuf.Reset()
+	i.keysBuf = i.keysBuf[:0]
+	i.keyBuf = i.keyBuf[:0]
+	if i.iterSpan == nil {
+		return
+	}
+	i.curr = Span{
+		Start:     i.saveBytes(i.iterSpan.Start),
+		End:       i.saveBytes(i.iterSpan.End),
+		KeysOrder: i.iterSpan.KeysOrder,
+	}
+	for j := range i.iterSpan.Keys {
+		i.keysBuf = append(i.keysBuf, Key{
+			Trailer: i.iterSpan.Keys[j].Trailer,
+			Suffix:  i.saveBytes(i.iterSpan.Keys[j].Suffix),
+			Value:   i.saveBytes(i.iterSpan.Keys[j].Value),
+		})
+	}
+	i.curr.Keys = i.keysBuf
+}
+
+func (i *DefragmentingIter) saveBytes(b []byte) []byte {
+	if b == nil {
+		return nil
+	}
+	i.currBuf, b = i.currBuf.Copy(b)
+	return b
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/doc.go

@@ -0,0 +1,13 @@
+// Package keyspan provides facilities for sorting, fragmenting and
+// iterating over spans of user keys.
+//
+// A Span represents a range of user key space with an inclusive start
+// key and exclusive end key. A span may hold any number of Keys which are
+// applied over the entirety of the span's keyspace.
+//
+// Spans are used within Pebble as an in-memory representation of range
+// deletion tombstones, and range key sets, unsets and deletes. Spans
+// are fragmented at overlapping key boundaries by the Fragmenter type.
+// This package's various iteration facilities require these
+// non-overlapping fragmented spans.
+package keyspan

vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go

@@ -0,0 +1,115 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// FilterFunc defines a transform from the input Span into the output Span. The
+// function returns true if the Span should be returned by the iterator, and
+// false if the Span should be skipped. The FilterFunc is permitted to mutate
+// the output Span, for example, to elice certain keys, or update the Span's
+// bounds if so desired. The output Span's Keys slice may be reused to reduce
+// allocations.
+type FilterFunc func(in *Span, out *Span) (keep bool)
+
+// filteringIter is a FragmentIterator that uses a FilterFunc to select which
+// Spans from the input iterator are returned in the output.
+//
+// A note on Span lifetimes: as the FilterFunc reuses a Span with a mutable
+// slice of Keys to reduce allocations, Spans returned by this iterator are only
+// valid until the next relative or absolute positioning method is called.
+type filteringIter struct {
+	iter     FragmentIterator
+	filterFn FilterFunc
+	cmp      base.Compare
+
+	// span is a mutable Span passed to the filterFn. The filterFn is free to
+	// mutate this Span. The slice of Keys in the Span is reused with every call
+	// to the filterFn.
+	span Span
+}
+
+var _ FragmentIterator = (*filteringIter)(nil)
+
+// Filter returns a new filteringIter that will filter the Spans from the
+// provided child iterator using the provided FilterFunc.
+func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
+	return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
+}
+
+// SeekGE implements FragmentIterator.
+func (i *filteringIter) SeekGE(key []byte) *Span {
+	span := i.filter(i.iter.SeekGE(key), +1)
+	// i.filter could return a span that's less than key, _if_ the filterFunc
+	// (which has no knowledge of the seek key) mutated the span to end at a key
+	// less than or equal to `key`. Detect this case and next/invalidate the iter.
+	if span != nil && i.cmp(span.End, key) <= 0 {
+		return i.Next()
+	}
+	return span
+}
+
+// SeekLT implements FragmentIterator.
+func (i *filteringIter) SeekLT(key []byte) *Span {
+	span := i.filter(i.iter.SeekLT(key), -1)
+	// i.filter could return a span that's >= key, _if_ the filterFunc (which has
+	// no knowledge of the seek key) mutated the span to start at a key greater
+	// than or equal to `key`. Detect this case and prev/invalidate the iter.
+	if span != nil && i.cmp(span.Start, key) >= 0 {
+		return i.Prev()
+	}
+	return span
+}
+
+// First implements FragmentIterator.
+func (i *filteringIter) First() *Span {
+	return i.filter(i.iter.First(), +1)
+}
+
+// Last implements FragmentIterator.
+func (i *filteringIter) Last() *Span {
+	return i.filter(i.iter.Last(), -1)
+}
+
+// Next implements FragmentIterator.
+func (i *filteringIter) Next() *Span {
+	return i.filter(i.iter.Next(), +1)
+}
+
+// Prev implements FragmentIterator.
+func (i *filteringIter) Prev() *Span {
+	return i.filter(i.iter.Prev(), -1)
+}
+
+// Error implements FragmentIterator.
+func (i *filteringIter) Error() error {
+	return i.iter.Error()
+}
+
+// Close implements FragmentIterator.
+func (i *filteringIter) Close() error {
+	return i.iter.Close()
+}
+
+// filter uses the filterFn (if configured) to filter and possibly mutate the
+// given Span. If the current Span is to be skipped, the iterator continues
+// iterating in the given direction until it lands on a Span that should be
+// returned, or the iterator becomes invalid.
+func (i *filteringIter) filter(span *Span, dir int8) *Span {
+	if i.filterFn == nil {
+		return span
+	}
+	for i.Error() == nil && span != nil {
+		if keep := i.filterFn(span, &i.span); keep {
+			return &i.span
+		}
+		if dir == +1 {
+			span = i.iter.Next()
+		} else {
+			span = i.iter.Prev()
+		}
+	}
+	return span
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go

@@ -0,0 +1,483 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+type spansByStartKey struct {
+	cmp base.Compare
+	buf []Span
+}
+
+func (v *spansByStartKey) Len() int { return len(v.buf) }
+func (v *spansByStartKey) Less(i, j int) bool {
+	return v.cmp(v.buf[i].Start, v.buf[j].Start) < 0
+}
+func (v *spansByStartKey) Swap(i, j int) {
+	v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
+}
+
+type spansByEndKey struct {
+	cmp base.Compare
+	buf []Span
+}
+
+func (v *spansByEndKey) Len() int { return len(v.buf) }
+func (v *spansByEndKey) Less(i, j int) bool {
+	return v.cmp(v.buf[i].End, v.buf[j].End) < 0
+}
+func (v *spansByEndKey) Swap(i, j int) {
+	v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
+}
+
+// keysBySeqNumKind sorts spans by the start key's sequence number in
+// descending order. If two spans have equal sequence number, they're compared
+// by key kind in descending order. This ordering matches the ordering of
+// base.InternalCompare among keys with matching user keys.
+type keysBySeqNumKind []Key
+
+func (v *keysBySeqNumKind) Len() int           { return len(*v) }
+func (v *keysBySeqNumKind) Less(i, j int) bool { return (*v)[i].Trailer > (*v)[j].Trailer }
+func (v *keysBySeqNumKind) Swap(i, j int)      { (*v)[i], (*v)[j] = (*v)[j], (*v)[i] }
+
+// Sort the spans by start key. This is the ordering required by the
+// Fragmenter. Usually spans are naturally sorted by their start key,
+// but that isn't true for range deletion tombstones in the legacy
+// range-del-v1 block format.
+func Sort(cmp base.Compare, spans []Span) {
+	sorter := spansByStartKey{
+		cmp: cmp,
+		buf: spans,
+	}
+	sort.Sort(&sorter)
+}
+
+// Fragmenter fragments a set of spans such that overlapping spans are
+// split at their overlap points. The fragmented spans are output to the
+// supplied Output function.
+type Fragmenter struct {
+	Cmp    base.Compare
+	Format base.FormatKey
+	// Emit is called to emit a fragmented span and its keys. Every key defined
+	// within the emitted Span applies to the entirety of the Span's key span.
+	// Keys are ordered in decreasing order of their sequence numbers, and if
+	// equal, decreasing order of key kind.
+	Emit func(Span)
+	// pending contains the list of pending fragments that have not been
+	// flushed to the block writer. Note that the spans have not been
+	// fragmented on the end keys yet. That happens as the spans are
+	// flushed. All pending spans have the same Start.
+	pending []Span
+	// doneBuf is used to buffer completed span fragments when flushing to a
+	// specific key (e.g. TruncateAndFlushTo). It is cached in the Fragmenter to
+	// allow reuse.
+	doneBuf []Span
+	// sortBuf is used to sort fragments by end key when flushing.
+	sortBuf spansByEndKey
+	// flushBuf is used to sort keys by (seqnum,kind) before emitting.
+	flushBuf keysBySeqNumKind
+	// flushedKey is the key that fragments have been flushed up to. Any
+	// additional spans added to the fragmenter must have a start key >=
+	// flushedKey. A nil value indicates flushedKey has not been set.
+	flushedKey []byte
+	finished   bool
+}
+
+func (f *Fragmenter) checkInvariants(buf []Span) {
+	for i := 1; i < len(buf); i++ {
+		if f.Cmp(buf[i].Start, buf[i].End) >= 0 {
+			panic(fmt.Sprintf("pebble: empty pending span invariant violated: %s", buf[i]))
+		}
+		if f.Cmp(buf[i-1].Start, buf[i].Start) != 0 {
+			panic(fmt.Sprintf("pebble: pending span invariant violated: %s %s",
+				f.Format(buf[i-1].Start), f.Format(buf[i].Start)))
+		}
+	}
+}
+
+// Add adds a span to the fragmenter. Spans may overlap and the
+// fragmenter will internally split them. The spans must be presented in
+// increasing start key order. That is, Add must be called with a series
+// of spans like:
+//
+//	a---e
+//	  c---g
+//	  c-----i
+//	         j---n
+//	         j-l
+//
+// We need to fragment the spans at overlap points. In the above
+// example, we'd create:
+//
+//	a-c-e
+//	  c-e-g
+//	  c-e-g-i
+//	         j-l-n
+//	         j-l
+//
+// The fragments need to be output sorted by start key, and for equal start
+// keys, sorted by descending sequence number. This last part requires a mild
+// bit of care as the fragments are not created in descending sequence number
+// order.
+//
+// Once a start key has been seen, we know that we'll never see a smaller
+// start key and can thus flush all of the fragments that lie before that
+// start key.
+//
+// Walking through the example above, we start with:
+//
+//	a---e
+//
+// Next we add [c,g) resulting in:
+//
+//	a-c-e
+//	  c---g
+//
+// The fragment [a,c) is flushed leaving the pending spans as:
+//
+//	c-e
+//	c---g
+//
+// The next span is [c,i):
+//
+//	c-e
+//	c---g
+//	c-----i
+//
+// No fragments are flushed. The next span is [j,n):
+//
+//	c-e
+//	c---g
+//	c-----i
+//	       j---n
+//
+// The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments
+// by their end key, then split the fragments on the end keys:
+//
+//	c-e
+//	c-e-g
+//	c-e---i
+//
+// The [c,e) fragments all get flushed leaving:
+//
+//	e-g
+//	e---i
+//
+// This process continues until there are no more fragments to flush.
+//
+// WARNING: the slices backing Start, End, Keys, Key.Suffix and Key.Value are
+// all retained after this method returns and should not be modified. This is
+// safe for spans that are added from a memtable or batch. It is partially
+// unsafe for a span read from an sstable. Specifically, the Keys slice of a
+// Span returned during sstable iteration is only valid until the next iterator
+// operation. The stability of the user keys depend on whether the block is
+// prefix compressed, and in practice Pebble never prefix compresses range
+// deletion and range key blocks, so these keys are stable. Because of this key
+// stability, typically callers only need to perform a shallow clone of the Span
+// before Add-ing it to the fragmenter.
+//
+// Add requires the provided span's keys are sorted in Trailer descending order.
+func (f *Fragmenter) Add(s Span) {
+	if f.finished {
+		panic("pebble: span fragmenter already finished")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span keys unexpectedly not in trailer descending order")
+	}
+	if f.flushedKey != nil {
+		switch c := f.Cmp(s.Start, f.flushedKey); {
+		case c < 0:
+			panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
+				f.Format(s.Start), f.Format(f.flushedKey)))
+		}
+	}
+	if f.Cmp(s.Start, s.End) >= 0 {
+		// An empty span, we can ignore it.
+		return
+	}
+	if invariants.RaceEnabled {
+		f.checkInvariants(f.pending)
+		defer func() { f.checkInvariants(f.pending) }()
+	}
+
+	if len(f.pending) > 0 {
+		// Since all of the pending spans have the same start key, we only need
+		// to compare against the first one.
+		switch c := f.Cmp(f.pending[0].Start, s.Start); {
+		case c > 0:
+			panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
+				f.Format(f.pending[0].Start), f.Format(s.Start)))
+		case c == 0:
+			// The new span has the same start key as the existing pending
+			// spans. Add it to the pending buffer.
+			f.pending = append(f.pending, s)
+			return
+		}
+
+		// At this point we know that the new start key is greater than the pending
+		// spans start keys.
+		f.truncateAndFlush(s.Start)
+	}
+
+	f.pending = append(f.pending, s)
+}
+
+// Cover is returned by Framenter.Covers and describes a span's relationship to
+// a key at a particular snapshot.
+type Cover int8
+
+const (
+	// NoCover indicates the tested key does not fall within the span's bounds,
+	// or the span contains no keys with sequence numbers higher than the key's.
+	NoCover Cover = iota
+	// CoversInvisibly indicates the tested key does fall within the span's
+	// bounds and the span contains at least one key with a higher sequence
+	// number, but none visible at the provided snapshot.
+	CoversInvisibly
+	// CoversVisibly indicates the tested key does fall within the span's
+	// bounds, and the span constains at least one key with a sequence number
+	// higher than the key's sequence number that is visible at the provided
+	// snapshot.
+	CoversVisibly
+)
+
+// Covers returns an enum indicating whether the specified key is covered by one
+// of the pending keys. The provided key must be consistent with the ordering of
+// the spans. That is, it is invalid to specify a key here that is out of order
+// with the span start keys passed to Add.
+func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) Cover {
+	if f.finished {
+		panic("pebble: span fragmenter already finished")
+	}
+	if len(f.pending) == 0 {
+		return NoCover
+	}
+
+	if f.Cmp(f.pending[0].Start, key.UserKey) > 0 {
+		panic(fmt.Sprintf("pebble: keys must be in order: %s > %s",
+			f.Format(f.pending[0].Start), key.Pretty(f.Format)))
+	}
+
+	cover := NoCover
+	seqNum := key.SeqNum()
+	for _, s := range f.pending {
+		if f.Cmp(key.UserKey, s.End) < 0 {
+			// NB: A range deletion tombstone does not delete a point operation
+			// at the same sequence number, and broadly a span is not considered
+			// to cover a point operation at the same sequence number.
+
+			for i := range s.Keys {
+				if kseq := s.Keys[i].SeqNum(); kseq > seqNum {
+					// This key from the span has a higher sequence number than
+					// `key`. It covers `key`, although the span's key might not
+					// be visible if its snapshot is too high.
+					//
+					// Batch keys are always be visible.
+					if kseq < snapshot || kseq&base.InternalKeySeqNumBatch != 0 {
+						return CoversVisibly
+					}
+					// s.Keys[i] is not visible.
+					cover = CoversInvisibly
+				}
+			}
+		}
+	}
+	return cover
+}
+
+// Empty returns true if all fragments added so far have finished flushing.
+func (f *Fragmenter) Empty() bool {
+	return f.finished || len(f.pending) == 0
+}
+
+// TruncateAndFlushTo flushes all of the fragments with a start key <= key,
+// truncating spans to the specified end key. Used during compaction to force
+// emitting of spans which straddle an sstable boundary. Consider
+// the scenario:
+//
+//	a---------k#10
+//	     f#8
+//	     f#7
+//
+// Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will
+// flush this span:
+//
+//	a-------g#10
+//	     f#8
+//	     f#7
+//
+// And leave this one in f.pending:
+//
+//	g----k#10
+//
+// WARNING: The fragmenter could hold on to the specified end key. Ensure it's
+// a safe byte slice that could outlast the current sstable output, and one
+// that will never be modified.
+func (f *Fragmenter) TruncateAndFlushTo(key []byte) {
+	if f.finished {
+		panic("pebble: span fragmenter already finished")
+	}
+	if f.flushedKey != nil {
+		switch c := f.Cmp(key, f.flushedKey); {
+		case c < 0:
+			panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
+				f.Format(key), f.Format(f.flushedKey)))
+		}
+	}
+	if invariants.RaceEnabled {
+		f.checkInvariants(f.pending)
+		defer func() { f.checkInvariants(f.pending) }()
+	}
+	if len(f.pending) > 0 {
+		// Since all of the pending spans have the same start key, we only need
+		// to compare against the first one.
+		switch c := f.Cmp(f.pending[0].Start, key); {
+		case c > 0:
+			panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
+				f.Format(f.pending[0].Start), f.Format(key)))
+		case c == 0:
+			return
+		}
+	}
+	f.truncateAndFlush(key)
+}
+
+// Start returns the start key of the first span in the pending buffer, or nil
+// if there are no pending spans. The start key of all pending spans is the same
+// as that of the first one.
+func (f *Fragmenter) Start() []byte {
+	if len(f.pending) > 0 {
+		return f.pending[0].Start
+	}
+	return nil
+}
+
+// Flushes all pending spans up to key (exclusive).
+//
+// WARNING: The specified key is stored without making a copy, so all callers
+// must ensure it is safe.
+func (f *Fragmenter) truncateAndFlush(key []byte) {
+	f.flushedKey = append(f.flushedKey[:0], key...)
+	done := f.doneBuf[:0]
+	pending := f.pending
+	f.pending = f.pending[:0]
+
+	// pending and f.pending share the same underlying storage. As we iterate
+	// over pending we append to f.pending, but only one entry is appended in
+	// each iteration, after we have read the entry being overwritten.
+	for _, s := range pending {
+		if f.Cmp(key, s.End) < 0 {
+			//   s: a--+--e
+			// new:    c------
+			if f.Cmp(s.Start, key) < 0 {
+				done = append(done, Span{
+					Start: s.Start,
+					End:   key,
+					Keys:  s.Keys,
+				})
+			}
+			f.pending = append(f.pending, Span{
+				Start: key,
+				End:   s.End,
+				Keys:  s.Keys,
+			})
+		} else {
+			//   s: a-----e
+			// new:       e----
+			done = append(done, s)
+		}
+	}
+
+	f.doneBuf = done[:0]
+	f.flush(done, nil)
+}
+
+// flush a group of range spans to the block. The spans are required to all have
+// the same start key. We flush all span fragments until startKey > lastKey. If
+// lastKey is nil, all span fragments are flushed. The specification of a
+// non-nil lastKey occurs for range deletion tombstones during compaction where
+// we want to flush (but not truncate) all range tombstones that start at or
+// before the first key in the next sstable. Consider:
+//
+//	a---e#10
+//	a------h#9
+//
+// If a compaction splits the sstables at key c we want the first sstable to
+// contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally
+// produce a tombstone [e,h)#9, but we don't need to output that tombstone to
+// the first sstable.
+func (f *Fragmenter) flush(buf []Span, lastKey []byte) {
+	if invariants.RaceEnabled {
+		f.checkInvariants(buf)
+	}
+
+	// Sort the spans by end key. This will allow us to walk over the spans and
+	// easily determine the next split point (the smallest end-key).
+	f.sortBuf.cmp = f.Cmp
+	f.sortBuf.buf = buf
+	sort.Sort(&f.sortBuf)
+
+	// Loop over the spans, splitting by end key.
+	for len(buf) > 0 {
+		// A prefix of spans will end at split. remove represents the count of
+		// that prefix.
+		remove := 1
+		split := buf[0].End
+		f.flushBuf = append(f.flushBuf[:0], buf[0].Keys...)
+
+		for i := 1; i < len(buf); i++ {
+			if f.Cmp(split, buf[i].End) == 0 {
+				remove++
+			}
+			f.flushBuf = append(f.flushBuf, buf[i].Keys...)
+		}
+
+		sort.Sort(&f.flushBuf)
+
+		f.Emit(Span{
+			Start: buf[0].Start,
+			End:   split,
+			// Copy the sorted keys to a new slice.
+			//
+			// This allocation is an unfortunate side effect of the Fragmenter and
+			// the expectation that the spans it produces are available in-memory
+			// indefinitely.
+			//
+			// Eventually, we should be able to replace the fragmenter with the
+			// keyspan.MergingIter which will perform just-in-time
+			// fragmentation, and only guaranteeing the memory lifetime for the
+			// current span. The MergingIter fragments while only needing to
+			// access one Span per level. It only accesses the Span at the
+			// current position for each level. During compactions, we can write
+			// these spans to sstables without retaining previous Spans.
+			Keys: append([]Key(nil), f.flushBuf...),
+		})
+
+		if lastKey != nil && f.Cmp(split, lastKey) > 0 {
+			break
+		}
+
+		// Adjust the start key for every remaining span.
+		buf = buf[remove:]
+		for i := range buf {
+			buf[i].Start = split
+		}
+	}
+}
+
+// Finish flushes any remaining fragments to the output. It is an error to call
+// this if any other spans will be added.
+func (f *Fragmenter) Finish() {
+	if f.finished {
+		panic("pebble: span fragmenter already finished")
+	}
+	f.flush(f.pending, nil)
+	f.finished = true
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/get.go

@@ -0,0 +1,53 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Get returns the newest span that contains the target key. If no span
+// contains the target key, an empty span is returned. The snapshot
+// parameter controls the visibility of spans (only spans older than the
+// snapshot sequence number are visible). The iterator must contain
+// fragmented spans: no span may overlap another.
+func Get(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
+	// NB: We use SeekLT in order to land on the proper span for a search
+	// key that resides in the middle of a span. Consider the scenario:
+	//
+	//     a---e
+	//         e---i
+	//
+	// The spans are indexed by their start keys `a` and `e`. If the
+	// search key is `c` we want to land on the span [a,e). If we were
+	// to use SeekGE then the search key `c` would land on the span
+	// [e,i) and we'd have to backtrack. The one complexity here is what
+	// happens for the search key `e`. In that case SeekLT will land us
+	// on the span [a,e) and we'll have to move forward.
+	iterSpan := iter.SeekLT(key)
+	if iterSpan == nil {
+		iterSpan = iter.Next()
+		if iterSpan == nil {
+			// The iterator is empty.
+			return nil
+		}
+		if cmp(key, iterSpan.Start) < 0 {
+			// The search key lies before the first span.
+			return nil
+		}
+	}
+
+	// Invariant: key > iterSpan.Start
+	if cmp(key, iterSpan.End) >= 0 {
+		// The current span lies before the search key. Advance the iterator
+		// once to potentially land on a key with a start key exactly equal to
+		// key. (See the comment at the beginning of this function.)
+		iterSpan = iter.Next()
+		if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
+			// We've run out of spans or we've moved on to a span which
+			// starts after our search key.
+			return nil
+		}
+	}
+	return iterSpan
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go

@@ -0,0 +1,118 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// InternalIteratorShim is a temporary iterator type used as a shim between
+// keyspan.MergingIter and base.InternalIterator. It's used temporarily for
+// range deletions during compactions, allowing range deletions to be
+// interleaved by a compaction input iterator.
+//
+// TODO(jackson): This type should be removed, and the usages converted to using
+// an InterleavingIterator type that interleaves keyspan.Spans from a
+// keyspan.FragmentIterator with point keys.
+type InternalIteratorShim struct {
+	miter   MergingIter
+	mbufs   MergingBuffers
+	span    *Span
+	iterKey base.InternalKey
+}
+
+// Assert that InternalIteratorShim implements InternalIterator.
+var _ base.InternalIterator = &InternalIteratorShim{}
+
+// Init initializes the internal iterator shim to merge the provided fragment
+// iterators.
+func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) {
+	i.miter.Init(cmp, noopTransform, &i.mbufs, iters...)
+}
+
+// Span returns the span containing the full set of keys over the key span at
+// the current iterator position.
+func (i *InternalIteratorShim) Span() *Span {
+	return i.span
+}
+
+// SeekGE implements (base.InternalIterator).SeekGE.
+func (i *InternalIteratorShim) SeekGE(
+	key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+// SeekPrefixGE implements (base.InternalIterator).SeekPrefixGE.
+func (i *InternalIteratorShim) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+// SeekLT implements (base.InternalIterator).SeekLT.
+func (i *InternalIteratorShim) SeekLT(
+	key []byte, flags base.SeekLTFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+// First implements (base.InternalIterator).First.
+func (i *InternalIteratorShim) First() (*base.InternalKey, base.LazyValue) {
+	i.span = i.miter.First()
+	for i.span != nil && i.span.Empty() {
+		i.span = i.miter.Next()
+	}
+	if i.span == nil {
+		return nil, base.LazyValue{}
+	}
+	i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
+	return &i.iterKey, base.MakeInPlaceValue(i.span.End)
+}
+
+// Last implements (base.InternalIterator).Last.
+func (i *InternalIteratorShim) Last() (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+// Next implements (base.InternalIterator).Next.
+func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) {
+	i.span = i.miter.Next()
+	for i.span != nil && i.span.Empty() {
+		i.span = i.miter.Next()
+	}
+	if i.span == nil {
+		return nil, base.LazyValue{}
+	}
+	i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
+	return &i.iterKey, base.MakeInPlaceValue(i.span.End)
+}
+
+// NextPrefix implements (base.InternalIterator).NextPrefix.
+func (i *InternalIteratorShim) NextPrefix([]byte) (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+// Prev implements (base.InternalIterator).Prev.
+func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) {
+	panic("unimplemented")
+}
+
+// Error implements (base.InternalIterator).Error.
+func (i *InternalIteratorShim) Error() error {
+	return i.miter.Error()
+}
+
+// Close implements (base.InternalIterator).Close.
+func (i *InternalIteratorShim) Close() error {
+	return i.miter.Close()
+}
+
+// SetBounds implements (base.InternalIterator).SetBounds.
+func (i *InternalIteratorShim) SetBounds(lower, upper []byte) {
+}
+
+// String implements fmt.Stringer.
+func (i *InternalIteratorShim) String() string {
+	return i.miter.String()
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -0,0 +1,220 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/manifest"
+)
+
+// FragmentIterator defines an iterator interface over spans. The spans
+// surfaced by a FragmentIterator must be non-overlapping. This is achieved by
+// fragmenting spans at overlap points (see Fragmenter).
+//
+// A Span returned by a FragmentIterator is only valid until the next
+// positioning method. Some implementations (eg, keyspan.Iter) may provide
+// longer lifetimes but implementations need only guarantee stability until the
+// next positioning method.
+type FragmentIterator interface {
+	// SeekGE moves the iterator to the first span covering a key greater than
+	// or equal to the given key. This is equivalent to seeking to the first
+	// span with an end key greater than the given key.
+	SeekGE(key []byte) *Span
+
+	// SeekLT moves the iterator to the last span covering a key less than the
+	// given key. This is equivalent to seeking to the last span with a start
+	// key less than the given key.
+	SeekLT(key []byte) *Span
+
+	// First moves the iterator to the first span.
+	First() *Span
+
+	// Last moves the iterator to the last span.
+	Last() *Span
+
+	// Next moves the iterator to the next span.
+	//
+	// It is valid to call Next when the iterator is positioned before the first
+	// key/value pair due to either a prior call to SeekLT or Prev which
+	// returned an invalid span. It is not allowed to call Next when the
+	// previous call to SeekGE, SeekPrefixGE or Next returned an invalid span.
+	Next() *Span
+
+	// Prev moves the iterator to the previous span.
+	//
+	// It is valid to call Prev when the iterator is positioned after the last
+	// key/value pair due to either a prior call to SeekGE or Next which
+	// returned an invalid span. It is not allowed to call Prev when the
+	// previous call to SeekLT or Prev returned an invalid span.
+	Prev() *Span
+
+	// Error returns any accumulated error.
+	//
+	// TODO(jackson): Lift errors into return values on the positioning methods.
+	Error() error
+
+	// Close closes the iterator and returns any accumulated error. Exhausting
+	// the iterator is not considered to be an error. It is valid to call Close
+	// multiple times. Other methods should not be called after the iterator has
+	// been closed.
+	Close() error
+}
+
+// TableNewSpanIter creates a new iterator for range key spans for the given
+// file.
+type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
+
+// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
+// per-sstable span iterators.
+type SpanIterOptions struct {
+	// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
+	// when iterating over range keys.
+	RangeKeyFilters []base.BlockPropertyFilter
+}
+
+// Iter is an iterator over a set of fragmented spans.
+type Iter struct {
+	cmp   base.Compare
+	spans []Span
+	index int
+}
+
+// Iter implements the FragmentIterator interface.
+var _ FragmentIterator = (*Iter)(nil)
+
+// NewIter returns a new iterator over a set of fragmented spans.
+func NewIter(cmp base.Compare, spans []Span) *Iter {
+	i := &Iter{}
+	i.Init(cmp, spans)
+	return i
+}
+
+// Count returns the number of spans contained by Iter.
+func (i *Iter) Count() int {
+	return len(i.spans)
+}
+
+// Init initializes an Iter with the provided spans.
+func (i *Iter) Init(cmp base.Compare, spans []Span) {
+	*i = Iter{
+		cmp:   cmp,
+		spans: spans,
+		index: -1,
+	}
+}
+
+// SeekGE implements FragmentIterator.SeekGE.
+func (i *Iter) SeekGE(key []byte) *Span {
+	// NB: manually inlined sort.Search is ~5% faster.
+	//
+	// Define f(j) = false iff the span i.spans[j] is strictly before `key`
+	// (equivalently, i.spans[j].End ≤ key.)
+	//
+	// Define f(-1) == false and f(n) == true.
+	// Invariant: f(index-1) == false, f(upper) == true.
+	i.index = 0
+	upper := len(i.spans)
+	for i.index < upper {
+		h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
+		// i.index ≤ h < upper
+		if i.cmp(key, i.spans[h].End) >= 0 {
+			i.index = h + 1 // preserves f(i-1) == false
+		} else {
+			upper = h // preserves f(j) == true
+		}
+	}
+
+	// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
+	// true => answer is i.index.
+	if i.index >= len(i.spans) {
+		return nil
+	}
+	return &i.spans[i.index]
+}
+
+// SeekLT implements FragmentIterator.SeekLT.
+func (i *Iter) SeekLT(key []byte) *Span {
+	// NB: manually inlined sort.Search is ~5% faster.
+	//
+	// Define f(-1) == false and f(n) == true.
+	// Invariant: f(index-1) == false, f(upper) == true.
+	i.index = 0
+	upper := len(i.spans)
+	for i.index < upper {
+		h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
+		// i.index ≤ h < upper
+		if i.cmp(key, i.spans[h].Start) > 0 {
+			i.index = h + 1 // preserves f(i-1) == false
+		} else {
+			upper = h // preserves f(j) == true
+		}
+	}
+	// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
+	// true => answer is i.index.
+
+	// Since keys are strictly increasing, if i.index > 0 then i.index-1 will be
+	// the largest whose key is < the key sought.
+	i.index--
+	if i.index < 0 {
+		return nil
+	}
+	return &i.spans[i.index]
+}
+
+// First implements FragmentIterator.First.
+func (i *Iter) First() *Span {
+	if len(i.spans) == 0 {
+		return nil
+	}
+	i.index = 0
+	return &i.spans[i.index]
+}
+
+// Last implements FragmentIterator.Last.
+func (i *Iter) Last() *Span {
+	if len(i.spans) == 0 {
+		return nil
+	}
+	i.index = len(i.spans) - 1
+	return &i.spans[i.index]
+}
+
+// Next implements FragmentIterator.Next.
+func (i *Iter) Next() *Span {
+	if i.index >= len(i.spans) {
+		return nil
+	}
+	i.index++
+	if i.index >= len(i.spans) {
+		return nil
+	}
+	return &i.spans[i.index]
+}
+
+// Prev implements FragmentIterator.Prev.
+func (i *Iter) Prev() *Span {
+	if i.index < 0 {
+		return nil
+	}
+	i.index--
+	if i.index < 0 {
+		return nil
+	}
+	return &i.spans[i.index]
+}
+
+// Error implements FragmentIterator.Error.
+func (i *Iter) Error() error {
+	return nil
+}
+
+// Close implements FragmentIterator.Close.
+func (i *Iter) Close() error {
+	return nil
+}
+
+func (i *Iter) String() string {
+	return "fragmented-spans"
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go

@@ -0,0 +1,521 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/manifest"
+)
+
+// LevelIter provides a merged view of spans from sstables in a level.
+// It takes advantage of level invariants to only have one sstable span block
+// open at one time, opened using the newIter function passed in.
+type LevelIter struct {
+	cmp base.Compare
+	// Denotes the kind of key the level iterator should read. If the key type
+	// is KeyTypePoint, the level iterator will read range tombstones (which
+	// only affect point keys). If the key type is KeyTypeRange, the level
+	// iterator will read range keys. It is invalid to configure an iterator
+	// with the KeyTypePointAndRange key type.
+	//
+	// If key type is KeyTypePoint, no straddle spans are emitted between files,
+	// and point key bounds are used to find files instead of range key bounds.
+	//
+	// TODO(bilal): Straddle spans can safely be produced in rangedel mode once
+	// we can guarantee that we will never read sstables in a level that split
+	// user keys across them. This might be guaranteed in a future release, but
+	// as of CockroachDB 22.2 it is not guaranteed, so to be safe disable it when
+	// keyType == KeyTypePoint
+	keyType manifest.KeyType
+	// The LSM level this LevelIter is initialized for. Used in logging.
+	level manifest.Level
+	// The below fields are used to fill in gaps between adjacent files' range
+	// key spaces. This is an optimization to avoid unnecessarily loading files
+	// in cases where range keys are sparse and rare. dir is set by every
+	// positioning operation, straddleDir is set to dir whenever a straddling
+	// Span is synthesized and the last positioning operation returned a
+	// synthesized straddle span.
+	//
+	// Note that when a straddle span is initialized, iterFile is modified to
+	// point to the next file in the straddleDir direction. A change of direction
+	// on a straddle key therefore necessitates the value of iterFile to be
+	// reverted.
+	dir         int
+	straddle    Span
+	straddleDir int
+	// The iter for the current file (iterFile). It is nil under any of the
+	// following conditions:
+	// - files.Current() == nil
+	// - err != nil
+	// - straddleDir != 0, in which case iterFile is not nil and points to the
+	//   next file (in the straddleDir direction).
+	// - some other constraint, like the bounds in opts, caused the file at index to not
+	//   be relevant to the iteration.
+	iter FragmentIterator
+	// iterFile holds the current file.
+	// INVARIANT: iterFile = files.Current()
+	iterFile *manifest.FileMetadata
+	newIter  TableNewSpanIter
+	files    manifest.LevelIterator
+	err      error
+
+	// The options that were passed in.
+	tableOpts SpanIterOptions
+
+	// TODO(bilal): Add InternalIteratorStats.
+}
+
+// LevelIter implements the keyspan.FragmentIterator interface.
+var _ FragmentIterator = (*LevelIter)(nil)
+
+// NewLevelIter returns a LevelIter.
+func NewLevelIter(
+	opts SpanIterOptions,
+	cmp base.Compare,
+	newIter TableNewSpanIter,
+	files manifest.LevelIterator,
+	level manifest.Level,
+	keyType manifest.KeyType,
+) *LevelIter {
+	l := &LevelIter{}
+	l.Init(opts, cmp, newIter, files, level, keyType)
+	return l
+}
+
+// Init initializes a LevelIter.
+func (l *LevelIter) Init(
+	opts SpanIterOptions,
+	cmp base.Compare,
+	newIter TableNewSpanIter,
+	files manifest.LevelIterator,
+	level manifest.Level,
+	keyType manifest.KeyType,
+) {
+	l.err = nil
+	l.level = level
+	l.tableOpts = opts
+	l.cmp = cmp
+	l.iterFile = nil
+	l.newIter = newIter
+	switch keyType {
+	case manifest.KeyTypePoint:
+		l.keyType = keyType
+		l.files = files.Filter(keyType)
+	case manifest.KeyTypeRange:
+		l.keyType = keyType
+		l.files = files.Filter(keyType)
+	default:
+		panic(fmt.Sprintf("unsupported key type: %v", keyType))
+	}
+}
+
+func (l *LevelIter) findFileGE(key []byte) *manifest.FileMetadata {
+	// Find the earliest file whose largest key is >= key.
+	//
+	// If the earliest file has its largest key == key and that largest key is a
+	// range deletion sentinel, we know that we manufactured this sentinel to convert
+	// the exclusive range deletion end key into an inclusive key (reminder: [start, end)#seqnum
+	// is the form of a range deletion sentinel which can contribute a largest key = end#sentinel).
+	// In this case we don't return this as the earliest file since there is nothing actually
+	// equal to key in it.
+
+	m := l.files.SeekGE(l.cmp, key)
+	for m != nil {
+		largestKey := m.LargestRangeKey
+		if l.keyType == manifest.KeyTypePoint {
+			largestKey = m.LargestPointKey
+		}
+		if !largestKey.IsExclusiveSentinel() || l.cmp(largestKey.UserKey, key) != 0 {
+			break
+		}
+		m = l.files.Next()
+	}
+	return m
+}
+
+func (l *LevelIter) findFileLT(key []byte) *manifest.FileMetadata {
+	// Find the last file whose smallest key is < key.
+	return l.files.SeekLT(l.cmp, key)
+}
+
+type loadFileReturnIndicator int8
+
+const (
+	noFileLoaded loadFileReturnIndicator = iota
+	fileAlreadyLoaded
+	newFileLoaded
+)
+
+func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileReturnIndicator {
+	indicator := noFileLoaded
+	if l.iterFile == file {
+		if l.err != nil {
+			return noFileLoaded
+		}
+		if l.iter != nil {
+			// We are already at the file, but we would need to check for bounds.
+			// Set indicator accordingly.
+			indicator = fileAlreadyLoaded
+		}
+		// We were already at file, but don't have an iterator, probably because the file was
+		// beyond the iteration bounds. It may still be, but it is also possible that the bounds
+		// have changed. We handle that below.
+	}
+
+	// Note that LevelIter.Close() can be called multiple times.
+	if indicator != fileAlreadyLoaded {
+		if err := l.Close(); err != nil {
+			return noFileLoaded
+		}
+	}
+
+	l.iterFile = file
+	if file == nil {
+		return noFileLoaded
+	}
+	if indicator != fileAlreadyLoaded {
+		l.iter, l.err = l.newIter(file, l.tableOpts)
+		indicator = newFileLoaded
+	}
+	if l.err != nil {
+		return noFileLoaded
+	}
+	return indicator
+}
+
+// SeekGE implements keyspan.FragmentIterator.
+func (l *LevelIter) SeekGE(key []byte) *Span {
+	l.dir = +1
+	l.straddle = Span{}
+	l.straddleDir = 0
+	l.err = nil // clear cached iteration error
+
+	f := l.findFileGE(key)
+	if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
+		// Peek at the previous file.
+		prevFile := l.files.Prev()
+		l.files.Next()
+		if prevFile != nil {
+			// We could unconditionally return an empty span between the seek key and
+			// f.SmallestRangeKey, however if this span is to the left of all range
+			// keys on this level, it could lead to inconsistent behaviour in relative
+			// positioning operations. Consider this example, with a b-c range key:
+			//
+			// SeekGE(a) -> a-b:{}
+			// Next() -> b-c{(#5,RANGEKEYSET,@4,foo)}
+			// Prev() -> nil
+			//
+			// Iterators higher up in the iterator stack rely on this sort of relative
+			// positioning consistency.
+			//
+			// TODO(bilal): Investigate ways to be able to return straddle spans in
+			// cases similar to the above, while still retaining correctness.
+			// Return a straddling key instead of loading the file.
+			l.iterFile = f
+			if err := l.Close(); err != nil {
+				return l.verify(nil)
+			}
+			l.straddleDir = +1
+			l.straddle = Span{
+				Start: prevFile.LargestRangeKey.UserKey,
+				End:   f.SmallestRangeKey.UserKey,
+				Keys:  nil,
+			}
+			return l.verify(&l.straddle)
+		}
+	}
+	loadFileIndicator := l.loadFile(f, +1)
+	if loadFileIndicator == noFileLoaded {
+		return l.verify(nil)
+	}
+	if span := l.iter.SeekGE(key); span != nil {
+		return l.verify(span)
+	}
+	return l.skipEmptyFileForward()
+}
+
+// SeekLT implements keyspan.FragmentIterator.
+func (l *LevelIter) SeekLT(key []byte) *Span {
+	l.dir = -1
+	l.straddle = Span{}
+	l.straddleDir = 0
+	l.err = nil // clear cached iteration error
+
+	f := l.findFileLT(key)
+	if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
+		// Peek at the next file.
+		nextFile := l.files.Next()
+		l.files.Prev()
+		if nextFile != nil {
+			// We could unconditionally return an empty span between f.LargestRangeKey
+			// and the seek key, however if this span is to the right of all range keys
+			// on this level, it could lead to inconsistent behaviour in relative
+			// positioning operations. Consider this example, with a b-c range key:
+			//
+			// SeekLT(d) -> c-d:{}
+			// Prev() -> b-c{(#5,RANGEKEYSET,@4,foo)}
+			// Next() -> nil
+			//
+			// Iterators higher up in the iterator stack rely on this sort of relative
+			// positioning consistency.
+			//
+			// TODO(bilal): Investigate ways to be able to return straddle spans in
+			// cases similar to the above, while still retaining correctness.
+			// Return a straddling key instead of loading the file.
+			l.iterFile = f
+			if err := l.Close(); err != nil {
+				return l.verify(nil)
+			}
+			l.straddleDir = -1
+			l.straddle = Span{
+				Start: f.LargestRangeKey.UserKey,
+				End:   nextFile.SmallestRangeKey.UserKey,
+				Keys:  nil,
+			}
+			return l.verify(&l.straddle)
+		}
+	}
+	if l.loadFile(f, -1) == noFileLoaded {
+		return l.verify(nil)
+	}
+	if span := l.iter.SeekLT(key); span != nil {
+		return l.verify(span)
+	}
+	return l.skipEmptyFileBackward()
+}
+
+// First implements keyspan.FragmentIterator.
+func (l *LevelIter) First() *Span {
+	l.dir = +1
+	l.straddle = Span{}
+	l.straddleDir = 0
+	l.err = nil // clear cached iteration error
+
+	if l.loadFile(l.files.First(), +1) == noFileLoaded {
+		return l.verify(nil)
+	}
+	if span := l.iter.First(); span != nil {
+		return l.verify(span)
+	}
+	return l.skipEmptyFileForward()
+}
+
+// Last implements keyspan.FragmentIterator.
+func (l *LevelIter) Last() *Span {
+	l.dir = -1
+	l.straddle = Span{}
+	l.straddleDir = 0
+	l.err = nil // clear cached iteration error
+
+	if l.loadFile(l.files.Last(), -1) == noFileLoaded {
+		return l.verify(nil)
+	}
+	if span := l.iter.Last(); span != nil {
+		return l.verify(span)
+	}
+	return l.skipEmptyFileBackward()
+}
+
+// Next implements keyspan.FragmentIterator.
+func (l *LevelIter) Next() *Span {
+	if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) {
+		return l.verify(nil)
+	}
+	if l.iter == nil && l.iterFile == nil {
+		// l.dir <= 0
+		return l.First()
+	}
+	l.dir = +1
+
+	if l.iter != nil {
+		if span := l.iter.Next(); span != nil {
+			return l.verify(span)
+		}
+	}
+	return l.skipEmptyFileForward()
+}
+
+// Prev implements keyspan.FragmentIterator.
+func (l *LevelIter) Prev() *Span {
+	if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) {
+		return l.verify(nil)
+	}
+	if l.iter == nil && l.iterFile == nil {
+		// l.dir >= 0
+		return l.Last()
+	}
+	l.dir = -1
+
+	if l.iter != nil {
+		if span := l.iter.Prev(); span != nil {
+			return l.verify(span)
+		}
+	}
+	return l.skipEmptyFileBackward()
+}
+
+func (l *LevelIter) skipEmptyFileForward() *Span {
+	if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
+		l.iterFile != nil && l.iter != nil {
+		// We were at a file that had spans. Check if the next file that has
+		// spans is not directly adjacent to the current file i.e. there is a
+		// gap in the span keyspace between the two files. In that case, synthesize
+		// a "straddle span" in l.straddle and return that.
+		//
+		// Straddle spans are not created in rangedel mode.
+		if err := l.Close(); err != nil {
+			l.err = err
+			return l.verify(nil)
+		}
+		startKey := l.iterFile.LargestRangeKey.UserKey
+		// Resetting l.iterFile without loading the file into l.iter is okay and
+		// does not change the logic in loadFile() as long as l.iter is also nil;
+		// which it should be due to the Close() call above.
+		l.iterFile = l.files.Next()
+		if l.iterFile == nil {
+			return l.verify(nil)
+		}
+		endKey := l.iterFile.SmallestRangeKey.UserKey
+		if l.cmp(startKey, endKey) < 0 {
+			// There is a gap between the two files. Synthesize a straddling span
+			// to avoid unnecessarily loading the next file.
+			l.straddle = Span{
+				Start: startKey,
+				End:   endKey,
+			}
+			l.straddleDir = +1
+			return l.verify(&l.straddle)
+		}
+	} else if l.straddleDir < 0 {
+		// We were at a straddle key, but are now changing directions. l.iterFile
+		// was already moved backward by skipEmptyFileBackward, so advance it
+		// forward.
+		l.iterFile = l.files.Next()
+	}
+	l.straddle = Span{}
+	l.straddleDir = 0
+	var span *Span
+	for span.Empty() {
+		fileToLoad := l.iterFile
+		if l.keyType == manifest.KeyTypePoint {
+			// We haven't iterated to the next file yet if we're in point key
+			// (rangedel) mode.
+			fileToLoad = l.files.Next()
+		}
+		if l.loadFile(fileToLoad, +1) == noFileLoaded {
+			return l.verify(nil)
+		}
+		span = l.iter.First()
+		// In rangedel mode, we can expect to get empty files that we'd need to
+		// skip over, but not in range key mode.
+		if l.keyType == manifest.KeyTypeRange {
+			break
+		}
+	}
+	return l.verify(span)
+}
+
+func (l *LevelIter) skipEmptyFileBackward() *Span {
+	// We were at a file that had spans. Check if the previous file that has
+	// spans is not directly adjacent to the current file i.e. there is a
+	// gap in the span keyspace between the two files. In that case, synthesize
+	// a "straddle span" in l.straddle and return that.
+	//
+	// Straddle spans are not created in rangedel mode.
+	if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
+		l.iterFile != nil && l.iter != nil {
+		if err := l.Close(); err != nil {
+			l.err = err
+			return l.verify(nil)
+		}
+		endKey := l.iterFile.SmallestRangeKey.UserKey
+		// Resetting l.iterFile without loading the file into l.iter is okay and
+		// does not change the logic in loadFile() as long as l.iter is also nil;
+		// which it should be due to the Close() call above.
+		l.iterFile = l.files.Prev()
+		if l.iterFile == nil {
+			return l.verify(nil)
+		}
+		startKey := l.iterFile.LargestRangeKey.UserKey
+		if l.cmp(startKey, endKey) < 0 {
+			// There is a gap between the two files. Synthesize a straddling span
+			// to avoid unnecessarily loading the next file.
+			l.straddle = Span{
+				Start: startKey,
+				End:   endKey,
+			}
+			l.straddleDir = -1
+			return l.verify(&l.straddle)
+		}
+	} else if l.straddleDir > 0 {
+		// We were at a straddle key, but are now changing directions. l.iterFile
+		// was already advanced forward by skipEmptyFileForward, so move it
+		// backward.
+		l.iterFile = l.files.Prev()
+	}
+	l.straddle = Span{}
+	l.straddleDir = 0
+	var span *Span
+	for span.Empty() {
+		fileToLoad := l.iterFile
+		if l.keyType == manifest.KeyTypePoint {
+			fileToLoad = l.files.Prev()
+		}
+		if l.loadFile(fileToLoad, -1) == noFileLoaded {
+			return l.verify(nil)
+		}
+		span = l.iter.Last()
+		// In rangedel mode, we can expect to get empty files that we'd need to
+		// skip over, but not in range key mode as the filter on the FileMetadata
+		// should guarantee we always get a non-empty file.
+		if l.keyType == manifest.KeyTypeRange {
+			break
+		}
+	}
+	return l.verify(span)
+}
+
+// verify is invoked whenever a span is returned from an iterator positioning
+// method to a caller. During invariant builds, it asserts invariants to the
+// caller.
+func (l *LevelIter) verify(s *Span) *Span {
+	// NB: Do not add any logic outside the invariants.Enabled conditional to
+	// ensure that verify is always compiled away in production builds.
+	if invariants.Enabled {
+		if f := l.files.Current(); f != l.iterFile {
+			panic(fmt.Sprintf("LevelIter.files.Current (%s) and l.iterFile (%s) diverged",
+				f, l.iterFile))
+		}
+	}
+	return s
+}
+
+// Error implements keyspan.FragmentIterator.
+func (l *LevelIter) Error() error {
+	if l.err != nil || l.iter == nil {
+		return l.err
+	}
+	return l.iter.Error()
+}
+
+// Close implements keyspan.FragmentIterator.
+func (l *LevelIter) Close() error {
+	if l.iter != nil {
+		l.err = l.iter.Close()
+		l.iter = nil
+	}
+	return l.err
+}
+
+// String implements keyspan.FragmentIterator.
+func (l *LevelIter) String() string {
+	if l.iterFile != nil {
+		return fmt.Sprintf("%s: fileNum=%s", l.level, l.iterFile.FileNum)
+	}
+	return fmt.Sprintf("%s: fileNum=<nil>", l.level)
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go

@@ -0,0 +1,48 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// SeekLE seeks to the span that contains or is before the target key.
+func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
+	// NB: We use SeekLT in order to land on the proper span for a search
+	// key that resides in the middle of a span. Consider the scenario:
+	//
+	//     a---e
+	//         e---i
+	//
+	// The spans are indexed by their start keys `a` and `e`. If the
+	// search key is `c` we want to land on the span [a,e). If we were to
+	// use SeekGE then the search key `c` would land on the span [e,i) and
+	// we'd have to backtrack. The one complexity here is what happens for the
+	// search key `e`. In that case SeekLT will land us on the span [a,e)
+	// and we'll have to move forward.
+	iterSpan := iter.SeekLT(key)
+
+	if iterSpan == nil {
+		// Advance the iterator once to see if the next span has a start key
+		// equal to key.
+		iterSpan = iter.Next()
+		if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
+			// The iterator is exhausted or we've hit the next span.
+			return nil
+		}
+	} else {
+		// Invariant: key > iterSpan.Start
+		if cmp(key, iterSpan.End) >= 0 {
+			// The current span lies entirely before the search key. Check to see if
+			// the next span contains the search key. If it doesn't, we'll backup
+			// and return to our earlier candidate.
+			iterSpan = iter.Next()
+			if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
+				// The next span is past our search key or there is no next span. Go
+				// back.
+				iterSpan = iter.Prev()
+			}
+		}
+	}
+	return iterSpan
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go

@@ -0,0 +1,467 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan // import "github.com/cockroachdb/pebble/internal/keyspan"
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+// Span represents a set of keys over a span of user key space. All of the keys
+// within a Span are applied across the span's key span indicated by Start and
+// End. Each internal key applied over the user key span appears as a separate
+// Key, with its own kind and sequence number. Optionally, each Key may also
+// have a Suffix and/or Value.
+//
+// Note that the start user key is inclusive and the end user key is exclusive.
+//
+// Currently the only supported key kinds are:
+//
+//	RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
+type Span struct {
+	// Start and End encode the user key range of all the contained items, with
+	// an inclusive start key and exclusive end key. Both Start and End must be
+	// non-nil, or both nil if representing an invalid Span.
+	Start, End []byte
+	// Keys holds the set of keys applied over the [Start, End) user key range.
+	// Keys is sorted by (SeqNum, Kind) descending, unless otherwise specified
+	// by the context. If SeqNum and Kind are equal, the order of Keys is
+	// undefined. Keys may be empty, even if Start and End are non-nil.
+	//
+	// Keys are a decoded representation of the internal keys stored in batches
+	// or sstable blocks. A single internal key in a range key block may produce
+	// several decoded Keys.
+	Keys      []Key
+	KeysOrder KeysOrder
+}
+
+// KeysOrder describes the ordering of Keys within a Span.
+type KeysOrder int8
+
+const (
+	// ByTrailerDesc indicates a Span's keys are sorted by Trailer descending.
+	// This is the default ordering, and the ordering used during physical
+	// storage.
+	ByTrailerDesc KeysOrder = iota
+	// BySuffixAsc indicates a Span's keys are sorted by Suffix ascending. This
+	// ordering is used during user iteration of range keys.
+	BySuffixAsc
+)
+
+// Key represents a single key applied over a span of user keys. A Key is
+// contained by a Span which specifies the span of user keys over which the Key
+// is applied.
+type Key struct {
+	// Trailer contains the key kind and sequence number.
+	Trailer uint64
+	// Suffix holds an optional suffix associated with the key. This is only
+	// non-nil for RANGEKEYSET and RANGEKEYUNSET keys.
+	Suffix []byte
+	// Value holds a logical value associated with the Key. It is NOT the
+	// internal value stored in a range key or range deletion block.  This is
+	// only non-nil for RANGEKEYSET keys.
+	Value []byte
+}
+
+// SeqNum returns the sequence number component of the key.
+func (k Key) SeqNum() uint64 {
+	return k.Trailer >> 8
+}
+
+// VisibleAt returns true if the provided key is visible at the provided
+// snapshot sequence number. It interprets batch sequence numbers as always
+// visible, because non-visible batch span keys are filtered when they're
+// fragmented.
+func (k Key) VisibleAt(snapshot uint64) bool {
+	seq := k.SeqNum()
+	return seq < snapshot || seq&base.InternalKeySeqNumBatch != 0
+}
+
+// Kind returns the kind component of the key.
+func (k Key) Kind() base.InternalKeyKind {
+	return base.InternalKeyKind(k.Trailer & 0xff)
+}
+
+// Equal returns true if this Key is equal to the given key. Two keys are said
+// to be equal if the two Keys have equal trailers, suffix and value. Suffix
+// comparison uses the provided base.Compare func. Value comparison is bytewise.
+func (k Key) Equal(equal base.Equal, b Key) bool {
+	return k.Trailer == b.Trailer &&
+		equal(k.Suffix, b.Suffix) &&
+		bytes.Equal(k.Value, b.Value)
+}
+
+// Valid returns true if the span is defined.
+func (s *Span) Valid() bool {
+	return s.Start != nil && s.End != nil
+}
+
+// Empty returns true if the span does not contain any keys. An empty span may
+// still be Valid. A non-empty span must be Valid.
+//
+// An Empty span may be produced by Visible, or be produced by iterators in
+// order to surface the gaps between keys.
+func (s *Span) Empty() bool {
+	return s == nil || len(s.Keys) == 0
+}
+
+// SmallestKey returns the smallest internal key defined by the span's keys.
+// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
+// contains no keys or its keys are sorted in a different order.
+func (s *Span) SmallestKey() base.InternalKey {
+	if len(s.Keys) == 0 {
+		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	// The first key has the highest (sequence number,kind) tuple.
+	return base.InternalKey{
+		UserKey: s.Start,
+		Trailer: s.Keys[0].Trailer,
+	}
+}
+
+// LargestKey returns the largest internal key defined by the span's keys. The
+// returned key will always be a "sentinel key" at the end boundary. The
+// "sentinel key" models the exclusive end boundary by returning an InternalKey
+// with the maximal sequence number, ensuring all InternalKeys with the same
+// user key sort after the sentinel key.
+//
+// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
+// contains no keys or its keys are sorted in a different order.
+func (s *Span) LargestKey() base.InternalKey {
+	if len(s.Keys) == 0 {
+		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	// The last key has the lowest (sequence number,kind) tuple.
+	kind := s.Keys[len(s.Keys)-1].Kind()
+	return base.MakeExclusiveSentinelKey(kind, s.End)
+}
+
+// SmallestSeqNum returns the smallest sequence number of a key contained within
+// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
+// the span contains no keys or its keys are sorted in a different order.
+func (s *Span) SmallestSeqNum() uint64 {
+	if len(s.Keys) == 0 {
+		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+
+	return s.Keys[len(s.Keys)-1].SeqNum()
+}
+
+// LargestSeqNum returns the largest sequence number of a key contained within
+// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
+// the span contains no keys or its keys are sorted in a different order.
+func (s *Span) LargestSeqNum() uint64 {
+	if len(s.Keys) == 0 {
+		panic("pebble: Span contains no keys")
+	} else if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	return s.Keys[0].SeqNum()
+}
+
+// TODO(jackson): Replace most of the calls to Visible with more targeted calls
+// that avoid the need to construct a new Span.
+
+// Visible returns a span with the subset of keys visible at the provided
+// sequence number. It requires the Span's keys be in ByTrailerDesc order. It
+// panics if the span's keys are sorted in a different order.
+//
+// Visible may incur an allocation, so callers should prefer targeted,
+// non-allocating methods when possible.
+func (s Span) Visible(snapshot uint64) Span {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+
+	ret := Span{Start: s.Start, End: s.End}
+	if len(s.Keys) == 0 {
+		return ret
+	}
+
+	// Keys from indexed batches may force an allocation. The Keys slice is
+	// ordered by sequence number, so ordinarily we can return the trailing
+	// subslice containing keys with sequence numbers less than `seqNum`.
+	//
+	// However, batch keys are special. Only visible batch keys are included
+	// when an Iterator's batch spans are fragmented. They must always be
+	// visible.
+	//
+	// Batch keys can create a sandwich of visible batch keys at the beginning
+	// of the slice and visible committed keys at the end of the slice, forcing
+	// us to allocate a new slice and copy the contents.
+	//
+	// Care is taking to only incur an allocation only when batch keys and
+	// visible keys actually sandwich non-visible keys.
+
+	// lastBatchIdx and lastNonVisibleIdx are set to the last index of a batch
+	// key and a non-visible key respectively.
+	lastBatchIdx := -1
+	lastNonVisibleIdx := -1
+	for i := range s.Keys {
+		if seqNum := s.Keys[i].SeqNum(); seqNum&base.InternalKeySeqNumBatch != 0 {
+			// Batch key. Always visible.
+			lastBatchIdx = i
+		} else if seqNum >= snapshot {
+			// This key is not visible.
+			lastNonVisibleIdx = i
+		}
+	}
+
+	// In the following comments: b = batch, h = hidden, v = visible (committed).
+	switch {
+	case lastNonVisibleIdx == -1:
+		// All keys are visible.
+		//
+		// [b b b], [v v v] and [b b b v v v]
+		ret.Keys = s.Keys
+	case lastBatchIdx == -1:
+		// There are no batch keys, so we can return the continuous subslice
+		// starting after the last non-visible Key.
+		//
+		// h h h [v v v]
+		ret.Keys = s.Keys[lastNonVisibleIdx+1:]
+	case lastNonVisibleIdx == len(s.Keys)-1:
+		// While we have a batch key and non-visible keys, there are no
+		// committed visible keys. The 'sandwich' is missing the bottom layer,
+		// so we can return the continuous sublice at the beginning.
+		//
+		// [b b b] h h h
+		ret.Keys = s.Keys[0 : lastBatchIdx+1]
+	default:
+		// This is the problematic sandwich case. Allocate a new slice, copying
+		// the batch keys and the visible keys into it.
+		//
+		// [b b b] h h h [v v v]
+		ret.Keys = make([]Key, (lastBatchIdx+1)+(len(s.Keys)-lastNonVisibleIdx-1))
+		copy(ret.Keys, s.Keys[:lastBatchIdx+1])
+		copy(ret.Keys[lastBatchIdx+1:], s.Keys[lastNonVisibleIdx+1:])
+	}
+	return ret
+}
+
+// VisibleAt returns true if the span contains a key visible at the provided
+// snapshot. Keys with sequence numbers with the batch bit set are treated as
+// always visible.
+//
+// VisibleAt requires the Span's keys be in ByTrailerDesc order. It panics if
+// the span's keys are sorted in a different order.
+func (s *Span) VisibleAt(snapshot uint64) bool {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	if len(s.Keys) == 0 {
+		return false
+	} else if first := s.Keys[0].SeqNum(); first&base.InternalKeySeqNumBatch != 0 {
+		// Only visible batch keys are included when an Iterator's batch spans
+		// are fragmented. They must always be visible.
+		return true
+	} else {
+		// Otherwise we check the last key. Since keys are ordered decreasing in
+		// sequence number, the last key has the lowest sequence number of any
+		// of the span's keys. If any of the keys are visible, the last key must
+		// be visible. Or put differently: if the last key is not visible, then
+		// no key is visible.
+		return s.Keys[len(s.Keys)-1].SeqNum() < snapshot
+	}
+}
+
+// ShallowClone returns the span with a Keys slice owned by the span itself.
+// None of the key byte slices are cloned (see Span.DeepClone).
+func (s *Span) ShallowClone() Span {
+	c := Span{
+		Start:     s.Start,
+		End:       s.End,
+		Keys:      make([]Key, len(s.Keys)),
+		KeysOrder: s.KeysOrder,
+	}
+	copy(c.Keys, s.Keys)
+	return c
+}
+
+// DeepClone clones the span, creating copies of all contained slices. DeepClone
+// is intended for non-production code paths like tests, the level checker, etc
+// because it is allocation heavy.
+func (s *Span) DeepClone() Span {
+	c := Span{
+		Start:     make([]byte, len(s.Start)),
+		End:       make([]byte, len(s.End)),
+		Keys:      make([]Key, len(s.Keys)),
+		KeysOrder: s.KeysOrder,
+	}
+	copy(c.Start, s.Start)
+	copy(c.End, s.End)
+	for i := range s.Keys {
+		c.Keys[i].Trailer = s.Keys[i].Trailer
+		if len(s.Keys[i].Suffix) > 0 {
+			c.Keys[i].Suffix = make([]byte, len(s.Keys[i].Suffix))
+			copy(c.Keys[i].Suffix, s.Keys[i].Suffix)
+		}
+		if len(s.Keys[i].Value) > 0 {
+			c.Keys[i].Value = make([]byte, len(s.Keys[i].Value))
+			copy(c.Keys[i].Value, s.Keys[i].Value)
+		}
+	}
+	return c
+}
+
+// Contains returns true if the specified key resides within the span's bounds.
+func (s *Span) Contains(cmp base.Compare, key []byte) bool {
+	return cmp(s.Start, key) <= 0 && cmp(key, s.End) < 0
+}
+
+// Covers returns true if the span covers keys at seqNum.
+//
+// Covers requires the Span's keys be in ByTrailerDesc order. It panics if the
+// span's keys are sorted in a different order.
+func (s Span) Covers(seqNum uint64) bool {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	return !s.Empty() && s.Keys[0].SeqNum() > seqNum
+}
+
+// CoversAt returns true if the span contains a key that is visible at the
+// provided snapshot sequence number, and that key's sequence number is higher
+// than seqNum.
+//
+// Keys with sequence numbers with the batch bit set are treated as always
+// visible.
+//
+// CoversAt requires the Span's keys be in ByTrailerDesc order. It panics if the
+// span's keys are sorted in a different order.
+func (s *Span) CoversAt(snapshot, seqNum uint64) bool {
+	if s.KeysOrder != ByTrailerDesc {
+		panic("pebble: span's keys unexpectedly not in trailer order")
+	}
+	// NB: A key is visible at `snapshot` if its sequence number is strictly
+	// less than `snapshot`. See base.Visible.
+	for i := range s.Keys {
+		if kseq := s.Keys[i].SeqNum(); kseq&base.InternalKeySeqNumBatch != 0 {
+			// Only visible batch keys are included when an Iterator's batch spans
+			// are fragmented. They must always be visible.
+			return kseq > seqNum
+		} else if kseq < snapshot {
+			return kseq > seqNum
+		}
+	}
+	return false
+}
+
+// String returns a string representation of the span.
+func (s Span) String() string {
+	return fmt.Sprint(prettySpan{Span: s, formatKey: base.DefaultFormatter})
+}
+
+// Pretty returns a formatter for the span.
+func (s Span) Pretty(f base.FormatKey) fmt.Formatter {
+	// TODO(jackson): Take a base.FormatValue to format Key.Value too.
+	return prettySpan{s, f}
+}
+
+type prettySpan struct {
+	Span
+	formatKey base.FormatKey
+}
+
+func (s prettySpan) Format(fs fmt.State, c rune) {
+	if !s.Valid() {
+		fmt.Fprintf(fs, "<invalid>")
+		return
+	}
+	fmt.Fprintf(fs, "%s-%s:{", s.formatKey(s.Start), s.formatKey(s.End))
+	for i, k := range s.Keys {
+		if i > 0 {
+			fmt.Fprint(fs, " ")
+		}
+		fmt.Fprintf(fs, "(#%d,%s", k.SeqNum(), k.Kind())
+		if len(k.Suffix) > 0 || len(k.Value) > 0 {
+			fmt.Fprintf(fs, ",%s", k.Suffix)
+		}
+		if len(k.Value) > 0 {
+			fmt.Fprintf(fs, ",%s", k.Value)
+		}
+		fmt.Fprint(fs, ")")
+	}
+	fmt.Fprintf(fs, "}")
+}
+
+// SortKeysByTrailer sorts a keys slice by trailer.
+func SortKeysByTrailer(keys *[]Key) {
+	// NB: keys is a pointer to a slice instead of a slice to avoid `sorted`
+	// escaping to the heap.
+	sorted := (*keysBySeqNumKind)(keys)
+	sort.Sort(sorted)
+}
+
+// KeysBySuffix implements sort.Interface, sorting its member Keys slice to by
+// Suffix in the order dictated by Cmp.
+type KeysBySuffix struct {
+	Cmp  base.Compare
+	Keys []Key
+}
+
+func (s *KeysBySuffix) Len() int           { return len(s.Keys) }
+func (s *KeysBySuffix) Less(i, j int) bool { return s.Cmp(s.Keys[i].Suffix, s.Keys[j].Suffix) < 0 }
+func (s *KeysBySuffix) Swap(i, j int)      { s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] }
+
+// ParseSpan parses the string representation of a Span. It's intended for
+// tests. ParseSpan panics if passed a malformed span representation.
+func ParseSpan(input string) Span {
+	var s Span
+	parts := strings.FieldsFunc(input, func(r rune) bool {
+		switch r {
+		case '-', ':', '{', '}':
+			return true
+		default:
+			return unicode.IsSpace(r)
+		}
+	})
+	s.Start, s.End = []byte(parts[0]), []byte(parts[1])
+
+	// Each of the remaining parts represents a single Key.
+	s.Keys = make([]Key, 0, len(parts)-2)
+	for _, p := range parts[2:] {
+		keyFields := strings.FieldsFunc(p, func(r rune) bool {
+			switch r {
+			case '#', ',', '(', ')':
+				return true
+			default:
+				return unicode.IsSpace(r)
+			}
+		})
+
+		var k Key
+		// Parse the sequence number.
+		seqNum, err := strconv.ParseUint(keyFields[0], 10, 64)
+		if err != nil {
+			panic(fmt.Sprintf("invalid sequence number: %q: %s", keyFields[0], err))
+		}
+		// Parse the key kind.
+		kind := base.ParseKind(keyFields[1])
+		k.Trailer = base.MakeTrailer(seqNum, kind)
+		// Parse the optional suffix.
+		if len(keyFields) >= 3 {
+			k.Suffix = []byte(keyFields[2])
+		}
+		// Parse the optional value.
+		if len(keyFields) >= 4 {
+			k.Value = []byte(keyFields[3])
+		}
+		s.Keys = append(s.Keys, k)
+	}
+	return s
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go

@@ -0,0 +1,50 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Transformer defines a transformation to be applied to a Span.
+type Transformer interface {
+	// Transform takes a Span as input and writes the transformed Span to the
+	// provided output *Span pointer. The output Span's Keys slice may be reused
+	// by Transform to reduce allocations.
+	Transform(cmp base.Compare, in Span, out *Span) error
+}
+
+// The TransformerFunc type is an adapter to allow the use of ordinary functions
+// as Transformers. If f is a function with the appropriate signature,
+// TransformerFunc(f) is a Transformer that calls f.
+type TransformerFunc func(base.Compare, Span, *Span) error
+
+// Transform calls f(cmp, in, out).
+func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
+	return tf(cmp, in, out)
+}
+
+var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+	dst.Start, dst.End = s.Start, s.End
+	dst.Keys = append(dst.Keys[:0], s.Keys...)
+	return nil
+})
+
+// VisibleTransform filters keys that are invisible at the provided snapshot
+// sequence number.
+func VisibleTransform(snapshot uint64) Transformer {
+	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+		dst.Start, dst.End = s.Start, s.End
+		dst.Keys = dst.Keys[:0]
+		for _, k := range s.Keys {
+			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
+			// because a batch's visible span keys are filtered when they're
+			// fragmented. There's no requirement to enforce visibility at
+			// iteration time.
+			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
+				dst.Keys = append(dst.Keys, k)
+			}
+		}
+		return nil
+	})
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -0,0 +1,73 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Truncate creates a new iterator where every span in the supplied iterator is
+// truncated to be contained within the range [lower, upper). If start and end
+// are specified, filter out any spans that are completely outside those bounds.
+func Truncate(
+	cmp base.Compare,
+	iter FragmentIterator,
+	lower, upper []byte,
+	start, end *base.InternalKey,
+	panicOnUpperTruncate bool,
+) FragmentIterator {
+	return Filter(iter, func(in *Span, out *Span) (keep bool) {
+		out.Start, out.End = in.Start, in.End
+		out.Keys = append(out.Keys[:0], in.Keys...)
+
+		// Ignore this span if it lies completely outside start, end. Note that
+		// end endInclusive indicated whether end is inclusive.
+		//
+		// The comparison between s.End and start is by user key only, as
+		// the span is exclusive at s.End, so comparing by user keys
+		// is sufficient.
+		if start != nil && cmp(in.End, start.UserKey) <= 0 {
+			return false
+		}
+		if end != nil {
+			v := cmp(in.Start, end.UserKey)
+			switch {
+			case v > 0:
+				// Wholly outside the end bound. Skip it.
+				return false
+			case v == 0:
+				// This span begins at the same user key as `end`. Whether or
+				// not any of the keys contained within the span are relevant is
+				// dependent on Trailers. Any keys contained within the span
+				// with trailers larger than end cover the small sliver of
+				// keyspace between [k#inf, k#<end-seqnum>]. Since keys are
+				// sorted descending by Trailer within the span, we need to find
+				// the prefix of keys with larger trailers.
+				for i := range in.Keys {
+					if in.Keys[i].Trailer < end.Trailer {
+						out.Keys = out.Keys[:i]
+						break
+					}
+				}
+			default:
+				// Wholly within the end bound. Keep it.
+			}
+		}
+
+		var truncated bool
+		// Truncate the bounds to lower and upper.
+		if cmp(in.Start, lower) < 0 {
+			out.Start = lower
+		}
+		if cmp(in.End, upper) > 0 {
+			truncated = true
+			out.End = upper
+		}
+
+		if panicOnUpperTruncate && truncated {
+			panic("pebble: upper bound should not be truncated")
+		}
+
+		return !out.Empty() && cmp(out.Start, out.End) < 0
+	}, cmp)
+}

vendor/github.com/cockroachdb/pebble/internal/manifest/level.go

@@ -0,0 +1,46 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package manifest
+
+import "fmt"
+
+const (
+	// 3 bits are necessary to represent level values from 0-6.
+	levelBits = 3
+	levelMask = (1 << levelBits) - 1
+	// invalidSublevel denotes an invalid or non-applicable sublevel.
+	invalidSublevel = -1
+)
+
+// Level encodes a level and optional sublevel for use in log and error
+// messages. The encoding has the property that Level(0) ==
+// L0Sublevel(invalidSublevel).
+type Level uint32
+
+func makeLevel(level, sublevel int) Level {
+	return Level(((sublevel + 1) << levelBits) | level)
+}
+
+// LevelToInt returns the int representation of a Level
+func LevelToInt(l Level) int {
+	return int(l) & levelMask
+}
+
+// L0Sublevel returns a Level representing the specified L0 sublevel.
+func L0Sublevel(sublevel int) Level {
+	if sublevel < 0 {
+		panic(fmt.Sprintf("invalid L0 sublevel: %d", sublevel))
+	}
+	return makeLevel(0, sublevel)
+}
+
+func (l Level) String() string {
+	level := int(l) & levelMask
+	sublevel := (int(l) >> levelBits) - 1
+	if sublevel != invalidSublevel {
+		return fmt.Sprintf("L%d.%d", level, sublevel)
+	}
+	return fmt.Sprintf("L%d", level)
+}

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -0,0 +1,748 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package manifest
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// LevelMetadata contains metadata for all of the files within
+// a level of the LSM.
+type LevelMetadata struct {
+	level     int
+	totalSize uint64
+	// NumVirtual is the number of virtual sstables in the level.
+	NumVirtual uint64
+	// VirtualSize is the size of the virtual sstables in the level.
+	VirtualSize uint64
+	tree        btree
+}
+
+// clone makes a copy of the level metadata, implicitly increasing the ref
+// count of every file contained within lm.
+func (lm *LevelMetadata) clone() LevelMetadata {
+	return LevelMetadata{
+		level:       lm.level,
+		totalSize:   lm.totalSize,
+		NumVirtual:  lm.NumVirtual,
+		VirtualSize: lm.VirtualSize,
+		tree:        lm.tree.Clone(),
+	}
+}
+
+func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
+	return lm.tree.Release()
+}
+
+func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetadata {
+	bcmp := btreeCmpSeqNum
+	if level > 0 {
+		bcmp = btreeCmpSmallestKey(cmp)
+	}
+	var lm LevelMetadata
+	lm.level = level
+	lm.tree, _ = makeBTree(bcmp, files)
+	for _, f := range files {
+		lm.totalSize += f.Size
+		if f.Virtual {
+			lm.NumVirtual++
+			lm.VirtualSize += f.Size
+		}
+	}
+	return lm
+}
+
+func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
+	var t btree
+	t.cmp = cmp
+	for _, f := range files {
+		t.Insert(f)
+	}
+	return t, newLevelSlice(t.Iter())
+}
+
+func (lm *LevelMetadata) insert(f *FileMetadata) error {
+	if err := lm.tree.Insert(f); err != nil {
+		return err
+	}
+	lm.totalSize += f.Size
+	if f.Virtual {
+		lm.NumVirtual++
+		lm.VirtualSize += f.Size
+	}
+	return nil
+}
+
+func (lm *LevelMetadata) remove(f *FileMetadata) bool {
+	lm.totalSize -= f.Size
+	if f.Virtual {
+		lm.NumVirtual--
+		lm.VirtualSize -= f.Size
+	}
+	return lm.tree.Delete(f)
+}
+
+// Empty indicates whether there are any files in the level.
+func (lm *LevelMetadata) Empty() bool {
+	return lm.tree.Count() == 0
+}
+
+// Len returns the number of files within the level.
+func (lm *LevelMetadata) Len() int {
+	return lm.tree.Count()
+}
+
+// Size returns the cumulative size of all the files within the level.
+func (lm *LevelMetadata) Size() uint64 {
+	return lm.totalSize
+}
+
+// Iter constructs a LevelIterator over the entire level.
+func (lm *LevelMetadata) Iter() LevelIterator {
+	return LevelIterator{iter: lm.tree.Iter()}
+}
+
+// Slice constructs a slice containing the entire level.
+func (lm *LevelMetadata) Slice() LevelSlice {
+	return newLevelSlice(lm.tree.Iter())
+}
+
+// Find finds the provided file in the level if it exists.
+func (lm *LevelMetadata) Find(cmp base.Compare, m *FileMetadata) *LevelFile {
+	iter := lm.Iter()
+	if lm.level != 0 {
+		// If lm holds files for levels >0, we can narrow our search by binary
+		// searching by bounds.
+		o := overlaps(iter, cmp, m.Smallest.UserKey,
+			m.Largest.UserKey, m.Largest.IsExclusiveSentinel())
+		iter = o.Iter()
+	}
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f == m {
+			lf := iter.Take()
+			return &lf
+		}
+	}
+	return nil
+}
+
+// Annotation lazily calculates and returns the annotation defined by
+// Annotator. The Annotator is used as the key for pre-calculated
+// values, so equal Annotators must be used to avoid duplicate computations
+// and cached annotations. Annotation must not be called concurrently, and in
+// practice this is achieved by requiring callers to hold DB.mu.
+func (lm *LevelMetadata) Annotation(annotator Annotator) interface{} {
+	if lm.Empty() {
+		return annotator.Zero(nil)
+	}
+	v, _ := lm.tree.root.Annotation(annotator)
+	return v
+}
+
+// InvalidateAnnotation clears any cached annotations defined by Annotator. The
+// Annotator is used as the key for pre-calculated values, so equal Annotators
+// must be used to clear the appropriate cached annotation. InvalidateAnnotation
+// must not be called concurrently, and in practice this is achieved by
+// requiring callers to hold DB.mu.
+func (lm *LevelMetadata) InvalidateAnnotation(annotator Annotator) {
+	if lm.Empty() {
+		return
+	}
+	lm.tree.root.InvalidateAnnotation(annotator)
+}
+
+// LevelFile holds a file's metadata along with its position
+// within a level of the LSM.
+type LevelFile struct {
+	*FileMetadata
+	slice LevelSlice
+}
+
+// Slice constructs a LevelSlice containing only this file.
+func (lf LevelFile) Slice() LevelSlice {
+	return lf.slice
+}
+
+// NewLevelSliceSeqSorted constructs a LevelSlice over the provided files,
+// sorted by the L0 sequence number sort order.
+// TODO(jackson): Can we improve this interface or avoid needing to export
+// a slice constructor like this?
+func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
+	tr, slice := makeBTree(btreeCmpSeqNum, files)
+	tr.Release()
+	slice.verifyInvariants()
+	return slice
+}
+
+// NewLevelSliceKeySorted constructs a LevelSlice over the provided files,
+// sorted by the files smallest keys.
+// TODO(jackson): Can we improve this interface or avoid needing to export
+// a slice constructor like this?
+func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice {
+	tr, slice := makeBTree(btreeCmpSmallestKey(cmp), files)
+	tr.Release()
+	slice.verifyInvariants()
+	return slice
+}
+
+// NewLevelSliceSpecificOrder constructs a LevelSlice over the provided files,
+// ordering the files by their order in the provided slice. It's used in
+// tests.
+// TODO(jackson): Update tests to avoid requiring this and remove it.
+func NewLevelSliceSpecificOrder(files []*FileMetadata) LevelSlice {
+	tr, slice := makeBTree(btreeCmpSpecificOrder(files), files)
+	tr.Release()
+	slice.verifyInvariants()
+	return slice
+}
+
+// newLevelSlice constructs a new LevelSlice backed by iter.
+func newLevelSlice(iter iterator) LevelSlice {
+	s := LevelSlice{iter: iter}
+	if iter.r != nil {
+		s.length = iter.r.subtreeCount
+	}
+	s.verifyInvariants()
+	return s
+}
+
+// newBoundedLevelSlice constructs a new LevelSlice backed by iter and bounded
+// by the provided start and end bounds. The provided startBound and endBound
+// iterators must be iterators over the same B-Tree. Both start and end bounds
+// are inclusive.
+func newBoundedLevelSlice(iter iterator, startBound, endBound *iterator) LevelSlice {
+	s := LevelSlice{
+		iter:  iter,
+		start: startBound,
+		end:   endBound,
+	}
+	if iter.valid() {
+		s.length = endBound.countLeft() - startBound.countLeft()
+		// NB: The +1 is a consequence of the end bound being inclusive.
+		if endBound.valid() {
+			s.length++
+		}
+		// NB: A slice that's empty due to its bounds may have an endBound
+		// positioned before the startBound due to the inclusive bounds.
+		// TODO(jackson): Consider refactoring the end boundary to be exclusive;
+		// it would simplify some areas (eg, here) and complicate others (eg,
+		// Reslice-ing to grow compactions).
+		if s.length < 0 {
+			s.length = 0
+		}
+	}
+	s.verifyInvariants()
+	return s
+}
+
+// LevelSlice contains a slice of the files within a level of the LSM.
+// A LevelSlice is immutable once created, but may be used to construct a
+// mutable LevelIterator over the slice's files.
+//
+// LevelSlices should be constructed through one of the existing constructors,
+// not manually initialized.
+type LevelSlice struct {
+	iter   iterator
+	length int
+	// start and end form the inclusive bounds of a slice of files within a
+	// level of the LSM. They may be nil if the entire B-Tree backing iter is
+	// accessible.
+	start *iterator
+	end   *iterator
+}
+
+func (ls LevelSlice) verifyInvariants() {
+	if invariants.Enabled {
+		i := ls.Iter()
+		var length int
+		for f := i.First(); f != nil; f = i.Next() {
+			length++
+		}
+		if ls.length != length {
+			panic(fmt.Sprintf("LevelSlice %s has length %d value; actual length is %d", ls, ls.length, length))
+		}
+	}
+}
+
+// Each invokes fn for each element in the slice.
+func (ls LevelSlice) Each(fn func(*FileMetadata)) {
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		fn(f)
+	}
+}
+
+// String implements fmt.Stringer.
+func (ls LevelSlice) String() string {
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "%d files: ", ls.length)
+	ls.Each(func(f *FileMetadata) {
+		if buf.Len() > 0 {
+			fmt.Fprintf(&buf, " ")
+		}
+		fmt.Fprint(&buf, f)
+	})
+	return buf.String()
+}
+
+// Empty indicates whether the slice contains any files.
+func (ls *LevelSlice) Empty() bool {
+	return emptyWithBounds(ls.iter, ls.start, ls.end)
+}
+
+// Iter constructs a LevelIterator that iterates over the slice.
+func (ls *LevelSlice) Iter() LevelIterator {
+	return LevelIterator{
+		start: ls.start,
+		end:   ls.end,
+		iter:  ls.iter.clone(),
+	}
+}
+
+// Len returns the number of files in the slice. Its runtime is constant.
+func (ls *LevelSlice) Len() int {
+	return ls.length
+}
+
+// SizeSum sums the size of all files in the slice. Its runtime is linear in
+// the length of the slice.
+func (ls *LevelSlice) SizeSum() uint64 {
+	var sum uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		sum += f.Size
+	}
+	return sum
+}
+
+// NumVirtual returns the number of virtual sstables in the level. Its runtime is
+// linear in the length of the slice.
+func (ls *LevelSlice) NumVirtual() uint64 {
+	var n uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			n++
+		}
+	}
+	return n
+}
+
+// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
+// level.
+func (ls *LevelSlice) VirtualSizeSum() uint64 {
+	var sum uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			sum += f.Size
+		}
+	}
+	return sum
+}
+
+// Reslice constructs a new slice backed by the same underlying level, with
+// new start and end positions. Reslice invokes the provided function, passing
+// two LevelIterators: one positioned to i's inclusive start and one
+// positioned to i's inclusive end. The resliceFunc may move either iterator
+// forward or backwards, including beyond the callee's original bounds to
+// capture additional files from the underlying level. Reslice constructs and
+// returns a new LevelSlice with the final bounds of the iterators after
+// calling resliceFunc.
+func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelSlice {
+	if ls.iter.r == nil {
+		return ls
+	}
+	var start, end LevelIterator
+	if ls.start == nil {
+		start.iter = ls.iter.clone()
+		start.iter.first()
+	} else {
+		start.iter = ls.start.clone()
+	}
+	if ls.end == nil {
+		end.iter = ls.iter.clone()
+		end.iter.last()
+	} else {
+		end.iter = ls.end.clone()
+	}
+	resliceFunc(&start, &end)
+	return newBoundedLevelSlice(start.iter.clone(), &start.iter, &end.iter)
+}
+
+// KeyType is used to specify the type of keys we're looking for in
+// LevelIterator positioning operations. Files not containing any keys of the
+// desired type are skipped.
+type KeyType int8
+
+const (
+	// KeyTypePointAndRange denotes a search among the entire keyspace, including
+	// both point keys and range keys. No sstables are skipped.
+	KeyTypePointAndRange KeyType = iota
+	// KeyTypePoint denotes a search among the point keyspace. SSTables with no
+	// point keys will be skipped. Note that the point keyspace includes rangedels.
+	KeyTypePoint
+	// KeyTypeRange denotes a search among the range keyspace. SSTables with no
+	// range keys will be skipped.
+	KeyTypeRange
+)
+
+type keyTypeAnnotator struct{}
+
+var _ Annotator = keyTypeAnnotator{}
+
+func (k keyTypeAnnotator) Zero(dst interface{}) interface{} {
+	var val *KeyType
+	if dst != nil {
+		val = dst.(*KeyType)
+	} else {
+		val = new(KeyType)
+	}
+	*val = KeyTypePoint
+	return val
+}
+
+func (k keyTypeAnnotator) Accumulate(m *FileMetadata, dst interface{}) (interface{}, bool) {
+	v := dst.(*KeyType)
+	switch *v {
+	case KeyTypePoint:
+		if m.HasRangeKeys {
+			*v = KeyTypePointAndRange
+		}
+	case KeyTypePointAndRange:
+		// Do nothing.
+	default:
+		panic("unexpected key type")
+	}
+	return v, true
+}
+
+func (k keyTypeAnnotator) Merge(src interface{}, dst interface{}) interface{} {
+	v := dst.(*KeyType)
+	srcVal := src.(*KeyType)
+	switch *v {
+	case KeyTypePoint:
+		if *srcVal == KeyTypePointAndRange {
+			*v = KeyTypePointAndRange
+		}
+	case KeyTypePointAndRange:
+		// Do nothing.
+	default:
+		panic("unexpected key type")
+	}
+	return v
+}
+
+// LevelIterator iterates over a set of files' metadata. Its zero value is an
+// empty iterator.
+type LevelIterator struct {
+	iter   iterator
+	start  *iterator
+	end    *iterator
+	filter KeyType
+}
+
+func (i LevelIterator) String() string {
+	var buf bytes.Buffer
+	iter := i.iter.clone()
+	iter.first()
+	iter.prev()
+	if i.iter.pos == -1 {
+		fmt.Fprint(&buf, "(<start>)*")
+	}
+	iter.next()
+	for ; iter.valid(); iter.next() {
+		if buf.Len() > 0 {
+			fmt.Fprint(&buf, "   ")
+		}
+
+		if i.start != nil && cmpIter(iter, *i.start) == 0 {
+			fmt.Fprintf(&buf, " [ ")
+		}
+		isCurrentPos := cmpIter(iter, i.iter) == 0
+		if isCurrentPos {
+			fmt.Fprint(&buf, " ( ")
+		}
+		fmt.Fprint(&buf, iter.cur().String())
+		if isCurrentPos {
+			fmt.Fprint(&buf, " )*")
+		}
+		if i.end != nil && cmpIter(iter, *i.end) == 0 {
+			fmt.Fprintf(&buf, " ]")
+		}
+	}
+	if i.iter.n != nil && i.iter.pos >= i.iter.n.count {
+		if buf.Len() > 0 {
+			fmt.Fprint(&buf, "   ")
+		}
+		fmt.Fprint(&buf, "(<end>)*")
+	}
+	return buf.String()
+}
+
+// Clone copies the iterator, returning an independent iterator at the same
+// position.
+func (i *LevelIterator) Clone() LevelIterator {
+	if i.iter.r == nil {
+		return *i
+	}
+	// The start and end iterators are not cloned and are treated as
+	// immutable.
+	return LevelIterator{
+		iter:   i.iter.clone(),
+		start:  i.start,
+		end:    i.end,
+		filter: i.filter,
+	}
+}
+
+// Current returns the item at the current iterator position.
+//
+// Current is deprecated. Callers should instead use the return value of a
+// positioning operation.
+func (i *LevelIterator) Current() *FileMetadata {
+	if !i.iter.valid() ||
+		(i.end != nil && cmpIter(i.iter, *i.end) > 0) ||
+		(i.start != nil && cmpIter(i.iter, *i.start) < 0) {
+		return nil
+	}
+	return i.iter.cur()
+}
+
+func (i *LevelIterator) empty() bool {
+	return emptyWithBounds(i.iter, i.start, i.end)
+}
+
+// Filter clones the iterator and sets the desired KeyType as the key to filter
+// files on.
+func (i *LevelIterator) Filter(keyType KeyType) LevelIterator {
+	l := i.Clone()
+	l.filter = keyType
+	return l
+}
+
+func emptyWithBounds(i iterator, start, end *iterator) bool {
+	// If i.r is nil, the iterator was constructed from an empty btree.
+	// If the end bound is before the start bound, the bounds represent an
+	// empty slice of the B-Tree.
+	return i.r == nil || (start != nil && end != nil && cmpIter(*end, *start) < 0)
+}
+
+// First seeks to the first file in the iterator and returns it.
+func (i *LevelIterator) First() *FileMetadata {
+	if i.empty() {
+		return nil
+	}
+	if i.start != nil {
+		i.iter = i.start.clone()
+	} else {
+		i.iter.first()
+	}
+	if !i.iter.valid() {
+		return nil
+	}
+	return i.skipFilteredForward(i.iter.cur())
+}
+
+// Last seeks to the last file in the iterator and returns it.
+func (i *LevelIterator) Last() *FileMetadata {
+	if i.empty() {
+		return nil
+	}
+	if i.end != nil {
+		i.iter = i.end.clone()
+	} else {
+		i.iter.last()
+	}
+	if !i.iter.valid() {
+		return nil
+	}
+	return i.skipFilteredBackward(i.iter.cur())
+}
+
+// Next advances the iterator to the next file and returns it.
+func (i *LevelIterator) Next() *FileMetadata {
+	if i.iter.r == nil {
+		return nil
+	}
+	if invariants.Enabled && (i.iter.pos >= i.iter.n.count || (i.end != nil && cmpIter(i.iter, *i.end) > 0)) {
+		panic("pebble: cannot next forward-exhausted iterator")
+	}
+	i.iter.next()
+	if !i.iter.valid() {
+		return nil
+	}
+	return i.skipFilteredForward(i.iter.cur())
+}
+
+// Prev moves the iterator the previous file and returns it.
+func (i *LevelIterator) Prev() *FileMetadata {
+	if i.iter.r == nil {
+		return nil
+	}
+	if invariants.Enabled && (i.iter.pos < 0 || (i.start != nil && cmpIter(i.iter, *i.start) < 0)) {
+		panic("pebble: cannot prev backward-exhausted iterator")
+	}
+	i.iter.prev()
+	if !i.iter.valid() {
+		return nil
+	}
+	return i.skipFilteredBackward(i.iter.cur())
+}
+
+// SeekGE seeks to the first file in the iterator's file set with a largest
+// user key greater than or equal to the provided user key. The iterator must
+// have been constructed from L1+, because it requires the underlying files to
+// be sorted by user keys and non-overlapping.
+func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
+	// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
+	if i.iter.r == nil {
+		return nil
+	}
+	m := i.seek(func(m *FileMetadata) bool {
+		return cmp(m.Largest.UserKey, userKey) >= 0
+	})
+	if i.filter != KeyTypePointAndRange && m != nil {
+		b, ok := m.LargestBound(i.filter)
+		if !ok {
+			m = i.Next()
+		} else if c := cmp(b.UserKey, userKey); c < 0 || c == 0 && b.IsExclusiveSentinel() {
+			// This file does not contain any keys of the type ≥ lower. It
+			// should be filtered, even though it does contain point keys.
+			m = i.Next()
+		}
+	}
+	return i.skipFilteredForward(m)
+}
+
+// SeekLT seeks to the last file in the iterator's file set with a smallest
+// user key less than the provided user key. The iterator must have been
+// constructed from L1+, because it requires the underlying files to be sorted
+// by user keys and non-overlapping.
+func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata {
+	// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
+	if i.iter.r == nil {
+		return nil
+	}
+	i.seek(func(m *FileMetadata) bool {
+		return cmp(m.Smallest.UserKey, userKey) >= 0
+	})
+	m := i.Prev()
+	// Although i.Prev() guarantees that the current file contains keys of the
+	// relevant type, it doesn't guarantee that the keys of the relevant type
+	// are < userKey.
+	if i.filter != KeyTypePointAndRange && m != nil {
+		b, ok := m.SmallestBound(i.filter)
+		if !ok {
+			panic("unreachable")
+		}
+		if c := cmp(b.UserKey, userKey); c >= 0 {
+			// This file does not contain any keys of the type ≥ lower. It
+			// should be filtered, even though it does contain point keys.
+			m = i.Prev()
+		}
+	}
+	return i.skipFilteredBackward(m)
+}
+
+// skipFilteredForward takes the file metadata at the iterator's current
+// position, and skips forward if the current key-type filter (i.filter)
+// excludes the file. It skips until it finds an unfiltered file or exhausts the
+// level. If lower is != nil, skipFilteredForward skips any files that do not
+// contain keys with the provided key-type ≥ lower.
+//
+// skipFilteredForward also enforces the upper bound, returning nil if at any
+// point the upper bound is exceeded.
+func (i *LevelIterator) skipFilteredForward(meta *FileMetadata) *FileMetadata {
+	for meta != nil && !meta.ContainsKeyType(i.filter) {
+		i.iter.next()
+		if !i.iter.valid() {
+			meta = nil
+		} else {
+			meta = i.iter.cur()
+		}
+	}
+	if meta != nil && i.end != nil && cmpIter(i.iter, *i.end) > 0 {
+		// Exceeded upper bound.
+		meta = nil
+	}
+	return meta
+}
+
+// skipFilteredBackward takes the file metadata at the iterator's current
+// position, and skips backward if the current key-type filter (i.filter)
+// excludes the file. It skips until it finds an unfiltered file or exhausts the
+// level. If upper is != nil, skipFilteredBackward skips any files that do not
+// contain keys with the provided key-type < upper.
+//
+// skipFilteredBackward also enforces the lower bound, returning nil if at any
+// point the lower bound is exceeded.
+func (i *LevelIterator) skipFilteredBackward(meta *FileMetadata) *FileMetadata {
+	for meta != nil && !meta.ContainsKeyType(i.filter) {
+		i.iter.prev()
+		if !i.iter.valid() {
+			meta = nil
+		} else {
+			meta = i.iter.cur()
+		}
+	}
+	if meta != nil && i.start != nil && cmpIter(i.iter, *i.start) < 0 {
+		// Exceeded lower bound.
+		meta = nil
+	}
+	return meta
+}
+
+func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
+	i.iter.seek(fn)
+
+	// i.iter.seek seeked in the unbounded underlying B-Tree. If the iterator
+	// has start or end bounds, we may have exceeded them. Reset to the bounds
+	// if necessary.
+	//
+	// NB: The LevelIterator and LevelSlice semantics require that a bounded
+	// LevelIterator/LevelSlice containing files x0, x1, ..., xn behave
+	// identically to an unbounded LevelIterator/LevelSlice of a B-Tree
+	// containing x0, x1, ..., xn. In other words, any files outside the
+	// LevelIterator's bounds should not influence the iterator's behavior.
+	// When seeking, this means a SeekGE that seeks beyond the end bound,
+	// followed by a Prev should return the last element within bounds.
+	if i.end != nil && cmpIter(i.iter, *i.end) > 0 {
+		i.iter = i.end.clone()
+		// Since seek(fn) positioned beyond i.end, we know there is nothing to
+		// return within bounds.
+		i.iter.next()
+		return nil
+	} else if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
+		i.iter = i.start.clone()
+	}
+	if !i.iter.valid() {
+		return nil
+	}
+	return i.iter.cur()
+}
+
+// Take constructs a LevelFile containing the file at the iterator's current
+// position. Take panics if the iterator is not currently positioned over a
+// file.
+func (i *LevelIterator) Take() LevelFile {
+	m := i.Current()
+	if m == nil {
+		panic("Take called on invalid LevelIterator")
+	}
+	// LevelSlice's start and end fields are immutable and are positioned to
+	// the same position for a LevelFile because they're inclusive, so we can
+	// share one iterator stack between the two bounds.
+	boundsIter := i.iter.clone()
+	s := newBoundedLevelSlice(i.iter.clone(), &boundsIter, &boundsIter)
+	return LevelFile{
+		FileMetadata: m,
+		slice:        s,
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/manual/manual.go

@@ -0,0 +1,60 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package manual
+
+// #include <stdlib.h>
+import "C"
+import "unsafe"
+
+// The go:linkname directives provides backdoor access to private functions in
+// the runtime. Below we're accessing the throw function.
+
+//go:linkname throw runtime.throw
+func throw(s string)
+
+// TODO(peter): Rather than relying an C malloc/free, we could fork the Go
+// runtime page allocator and allocate large chunks of memory using mmap or
+// similar.
+
+// New allocates a slice of size n. The returned slice is from manually managed
+// memory and MUST be released by calling Free. Failure to do so will result in
+// a memory leak.
+func New(n int) []byte {
+	if n == 0 {
+		return make([]byte, 0)
+	}
+	// We need to be conscious of the Cgo pointer passing rules:
+	//
+	//   https://golang.org/cmd/cgo/#hdr-Passing_pointers
+	//
+	//   ...
+	//   Note: the current implementation has a bug. While Go code is permitted
+	//   to write nil or a C pointer (but not a Go pointer) to C memory, the
+	//   current implementation may sometimes cause a runtime error if the
+	//   contents of the C memory appear to be a Go pointer. Therefore, avoid
+	//   passing uninitialized C memory to Go code if the Go code is going to
+	//   store pointer values in it. Zero out the memory in C before passing it
+	//   to Go.
+	ptr := C.calloc(C.size_t(n), 1)
+	if ptr == nil {
+		// NB: throw is like panic, except it guarantees the process will be
+		// terminated. The call below is exactly what the Go runtime invokes when
+		// it cannot allocate memory.
+		throw("out of memory")
+	}
+	// Interpret the C pointer as a pointer to a Go array, then slice.
+	return (*[MaxArrayLen]byte)(unsafe.Pointer(ptr))[:n:n]
+}
+
+// Free frees the specified slice.
+func Free(b []byte) {
+	if cap(b) != 0 {
+		if len(b) == 0 {
+			b = b[:cap(b)]
+		}
+		ptr := unsafe.Pointer(&b[0])
+		C.free(ptr)
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/manual/manual_32bit.go

@@ -0,0 +1,13 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build 386 || amd64p32 || arm || armbe || ppc || sparc
+// +build 386 amd64p32 arm armbe ppc sparc
+
+package manual
+
+const (
+	// MaxArrayLen is a safe maximum length for slices on this architecture.
+	MaxArrayLen = 1<<31 - 1
+)

vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go

@@ -0,0 +1,13 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64 || loong64
+// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64 loong64
+
+package manual
+
+const (
+	// MaxArrayLen is a safe maximum length for slices on this architecture.
+	MaxArrayLen = 1<<50 - 1
+)

vendor/github.com/cockroachdb/pebble/internal/manual/manual_mips.go

@@ -0,0 +1,13 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build mips || mipsle || mips64p32 || mips64p32le
+// +build mips mipsle mips64p32 mips64p32le
+
+package manual
+
+const (
+	// MaxArrayLen is a safe maximum length for slices on this architecture.
+	MaxArrayLen = 1 << 30
+)

vendor/github.com/cockroachdb/pebble/internal/manual/manual_nocgo.go

@@ -0,0 +1,20 @@
+// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !cgo
+// +build !cgo
+
+package manual
+
+// Provides versions of New and Free when cgo is not available (e.g. cross
+// compilation).
+
+// New allocates a slice of size n.
+func New(n int) []byte {
+	return make([]byte, n)
+}
+
+// Free frees the specified slice.
+func Free(b []byte) {
+}

vendor/github.com/cockroachdb/pebble/internal/private/batch.go

@@ -0,0 +1,19 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package private
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/keyspan"
+)
+
+// BatchSort is a hook for constructing iterators over the point and range
+// mutations contained in a batch in sorted order. It is intended for testing
+// use only.
+var BatchSort func(interface{}) (
+	points base.InternalIterator,
+	rangeDels keyspan.FragmentIterator,
+	rangeKeys keyspan.FragmentIterator,
+)