Initial commit

vendor/github.com/cockroachdb/pebble/sstable/block_property.go

@@ -0,0 +1,820 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"encoding/binary"
+	"fmt"
+	"math"
+	"sync"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/rangekey"
+)
+
+// Block properties are an optional user-facing feature that can be used to
+// filter data blocks (and whole sstables) from an Iterator before they are
+// loaded. They do not apply to range delete blocks. These are expected to
+// very concisely represent a set of some attribute value contained within the
+// key or value, such that the set includes all the attribute values in the
+// block. This has some similarities with OLAP pruning approaches that
+// maintain min-max attribute values for some column (which concisely
+// represent a set), that is then used to prune at query time. In Pebble's
+// case, data blocks are small, typically 25-50KB, so these properties should
+// reduce their precision in order to be concise -- a good rule of thumb is to
+// not consume more than 50-100 bytes across all properties maintained for a
+// block, i.e., a 500x reduction compared to loading the data block.
+//
+// A block property must be assigned a unique name, which is encoded and
+// stored in the sstable. This name must be unique among all user-properties
+// encoded in an sstable.
+//
+// A property is represented as a []byte. A nil value or empty byte slice are
+// considered semantically identical. The caller is free to choose the
+// semantics of an empty byte slice e.g. they could use it to represent the
+// empty set or the universal set, whichever they think is more common and
+// therefore better to encode more concisely. The serialization of the
+// property for the various Finish*() calls in a BlockPropertyCollector
+// implementation should be identical, since the corresponding
+// BlockPropertyFilter implementation is not told the context in which it is
+// deserializing the property.
+//
+// Block properties are more general than table properties and should be
+// preferred over using table properties. A BlockPropertyCollector can achieve
+// identical behavior to table properties by returning the nil slice from
+// FinishDataBlock and FinishIndexBlock, and interpret them as the universal
+// set in BlockPropertyFilter, and return a non-universal set in FinishTable.
+//
+// Block property filtering is nondeterministic because the separation of keys
+// into blocks is nondeterministic. Clients use block-property filters to
+// implement efficient application of a filter F that applies to key-value pairs
+// (abbreviated as kv-filter). Consider correctness defined as surfacing exactly
+// the same key-value pairs that would be surfaced if one applied the filter F
+// above normal iteration. With this correctness definition, block property
+// filtering may introduce two kinds of errors:
+//
+//   a) Block property filtering that uses a kv-filter may produce additional
+//      key-value pairs that don't satisfy the filter because of the separation
+//      of keys into blocks. Clients may remove these extra key-value pairs by
+//      re-applying the kv filter while reading results back from Pebble.
+//
+//   b) Block property filtering may surface deleted key-value pairs if the
+//      kv filter is not a strict function of the key's user key. A block
+//      containing k.DEL may be filtered, while a block containing the deleted
+//      key k.SET may not be filtered, if the kv filter applies to one but not
+//      the other.
+//
+//      This error may be avoided trivially by using a kv filter that is a pure
+//      function of the user key. A filter that examines values or key kinds
+//      requires care to ensure F(k.SET, <value>) = F(k.DEL) = F(k.SINGLEDEL).
+//
+// The combination of range deletions and filtering by table-level properties
+// add another opportunity for deleted point keys to be surfaced. The pebble
+// Iterator stack takes care to correctly apply filtered tables' range deletions
+// to lower tables, preventing this form of nondeterministic error.
+//
+// In addition to the non-determinism discussed in (b), which limits the use
+// of properties over values, we now have support for values that are not
+// stored together with the key, and may not even be retrieved during
+// compactions. If Pebble is configured with such value separation, block
+// properties must only apply to the key, and will be provided a nil value.
+
+// BlockPropertyCollector is used when writing a sstable.
+//
+//   - All calls to Add are included in the next FinishDataBlock, after which
+//     the next data block is expected to start.
+//
+//   - The index entry generated for the data block, which contains the return
+//     value from FinishDataBlock, is not immediately included in the current
+//     index block. It is included when AddPrevDataBlockToIndexBlock is called.
+//     An alternative would be to return an opaque handle from FinishDataBlock
+//     and pass it to a new AddToIndexBlock method, which requires more
+//     plumbing, and passing of an interface{} results in a undesirable heap
+//     allocation. AddPrevDataBlockToIndexBlock must be called before keys are
+//     added to the new data block.
+type BlockPropertyCollector interface {
+	// Name returns the name of the block property collector.
+	Name() string
+	// Add is called with each new entry added to a data block in the sstable.
+	// The callee can assume that these are in sorted order.
+	Add(key InternalKey, value []byte) error
+	// FinishDataBlock is called when all the entries have been added to a
+	// data block. Subsequent Add calls will be for the next data block. It
+	// returns the property value for the finished block.
+	FinishDataBlock(buf []byte) ([]byte, error)
+	// AddPrevDataBlockToIndexBlock adds the entry corresponding to the
+	// previous FinishDataBlock to the current index block.
+	AddPrevDataBlockToIndexBlock()
+	// FinishIndexBlock is called when an index block, containing all the
+	// key-value pairs since the last FinishIndexBlock, will no longer see new
+	// entries. It returns the property value for the index block.
+	FinishIndexBlock(buf []byte) ([]byte, error)
+	// FinishTable is called when the sstable is finished, and returns the
+	// property value for the sstable.
+	FinishTable(buf []byte) ([]byte, error)
+}
+
+// SuffixReplaceableBlockCollector is an extension to the BlockPropertyCollector
+// interface that allows a block property collector to indicate that it supports
+// being *updated* during suffix replacement, i.e. when an existing SST in which
+// all keys have the same key suffix is updated to have a new suffix.
+//
+// A collector which supports being updated in such cases must be able to derive
+// its updated value from its old value and the change being made to the suffix,
+// without needing to be passed each updated K/V.
+//
+// For example, a collector that only inspects values would can simply copy its
+// previously computed property as-is, since key-suffix replacement does not
+// change values, while a collector that depends only on key suffixes, like one
+// which collected mvcc-timestamp bounds from timestamp-suffixed keys, can just
+// set its new bounds from the new suffix, as it is common to all keys, without
+// needing to recompute it from every key.
+//
+// An implementation of DataBlockIntervalCollector can also implement this
+// interface, in which case the BlockPropertyCollector returned by passing it to
+// NewBlockIntervalCollector will also implement this interface automatically.
+type SuffixReplaceableBlockCollector interface {
+	// UpdateKeySuffixes is called when a block is updated to change the suffix of
+	// all keys in the block, and is passed the old value for that prop, if any,
+	// for that block as well as the old and new suffix.
+	UpdateKeySuffixes(oldProp []byte, oldSuffix, newSuffix []byte) error
+}
+
+// BlockPropertyFilter is used in an Iterator to filter sstables and blocks
+// within the sstable. It should not maintain any per-sstable state, and must
+// be thread-safe.
+type BlockPropertyFilter = base.BlockPropertyFilter
+
+// BoundLimitedBlockPropertyFilter implements the block-property filter but
+// imposes an additional constraint on its usage, requiring that only blocks
+// containing exclusively keys between its lower and upper bounds may be
+// filtered. The bounds may be change during iteration, so the filter doesn't
+// expose the bounds, instead implementing KeyIsWithin[Lower,Upper]Bound methods
+// for performing bound comparisons.
+//
+// To be used, a BoundLimitedBlockPropertyFilter must be supplied directly
+// through NewBlockPropertiesFilterer's dedicated parameter. If supplied through
+// the ordinary slice of block property filters, this filter's bounds will be
+// ignored.
+//
+// The current [lower,upper) bounds of the filter are unknown, because they may
+// be changing. During forward iteration the lower bound is externally
+// guaranteed, meaning Intersects only returns false if the sstable iterator is
+// already known to be positioned at a key ≥ lower. The sstable iterator is then
+// only responsible for ensuring filtered blocks also meet the upper bound, and
+// should only allow a block to be filtered if all its keys are < upper. The
+// sstable iterator may invoke KeyIsWithinUpperBound(key) to perform this check,
+// where key is an inclusive upper bound on the block's keys.
+//
+// During backward iteration the upper bound is externally guaranteed, and
+// Intersects only returns false if the sstable iterator is already known to be
+// positioned at a key < upper. The sstable iterator is responsible for ensuring
+// filtered blocks also meet the lower bound, enforcing that a block is only
+// filtered if all its keys are ≥ lower. This check is made through passing the
+// block's inclusive lower bound to KeyIsWithinLowerBound.
+//
+// Implementations may become active or inactive through implementing Intersects
+// to return true whenever the filter is disabled.
+//
+// Usage of BoundLimitedBlockPropertyFilter is subtle, and Pebble consumers
+// should not implement this interface directly. This interface is an internal
+// detail in the implementation of block-property range-key masking.
+type BoundLimitedBlockPropertyFilter interface {
+	BlockPropertyFilter
+
+	// KeyIsWithinLowerBound tests whether the provided internal key falls
+	// within the current lower bound of the filter. A true return value
+	// indicates that the filter may be used to filter blocks that exclusively
+	// contain keys ≥ `key`, so long as the blocks' keys also satisfy the upper
+	// bound.
+	KeyIsWithinLowerBound(key []byte) bool
+	// KeyIsWithinUpperBound tests whether the provided internal key falls
+	// within the current upper bound of the filter. A true return value
+	// indicates that the filter may be used to filter blocks that exclusively
+	// contain keys ≤ `key`, so long as the blocks' keys also satisfy the lower
+	// bound.
+	KeyIsWithinUpperBound(key []byte) bool
+}
+
+// BlockIntervalCollector is a helper implementation of BlockPropertyCollector
+// for users who want to represent a set of the form [lower,upper) where both
+// lower and upper are uint64, and lower <= upper.
+//
+// The set is encoded as:
+// - Two varint integers, (lower,upper-lower), when upper-lower > 0
+// - Nil, when upper-lower=0
+//
+// Users must not expect this to preserve differences between empty sets --
+// they will all get turned into the semantically equivalent [0,0).
+//
+// A BlockIntervalCollector that collects over point and range keys needs to
+// have both the point and range DataBlockIntervalCollector specified, since
+// point and range keys are fed to the BlockIntervalCollector in an interleaved
+// fashion, independently of one another. This also implies that the
+// DataBlockIntervalCollectors for point and range keys should be references to
+// independent instances, rather than references to the same collector, as point
+// and range keys are tracked independently.
+type BlockIntervalCollector struct {
+	name   string
+	points DataBlockIntervalCollector
+	ranges DataBlockIntervalCollector
+
+	blockInterval interval
+	indexInterval interval
+	tableInterval interval
+}
+
+var _ BlockPropertyCollector = &BlockIntervalCollector{}
+
+// DataBlockIntervalCollector is the interface used by BlockIntervalCollector
+// that contains the actual logic pertaining to the property. It only
+// maintains state for the current data block, and resets that state in
+// FinishDataBlock. This interface can be used to reduce parsing costs.
+type DataBlockIntervalCollector interface {
+	// Add is called with each new entry added to a data block in the sstable.
+	// The callee can assume that these are in sorted order.
+	Add(key InternalKey, value []byte) error
+	// FinishDataBlock is called when all the entries have been added to a
+	// data block. Subsequent Add calls will be for the next data block. It
+	// returns the [lower, upper) for the finished block.
+	FinishDataBlock() (lower uint64, upper uint64, err error)
+}
+
+// NewBlockIntervalCollector constructs a BlockIntervalCollector with the given
+// name. The BlockIntervalCollector makes use of the given point and range key
+// DataBlockIntervalCollectors when encountering point and range keys,
+// respectively.
+//
+// The caller may pass a nil DataBlockIntervalCollector for one of the point or
+// range key collectors, in which case keys of those types will be ignored. This
+// allows for flexible construction of BlockIntervalCollectors that operate on
+// just point keys, just range keys, or both point and range keys.
+//
+// If both point and range keys are to be tracked, two independent collectors
+// should be provided, rather than the same collector passed in twice (see the
+// comment on BlockIntervalCollector for more detail)
+func NewBlockIntervalCollector(
+	name string, pointCollector, rangeCollector DataBlockIntervalCollector,
+) BlockPropertyCollector {
+	if pointCollector == nil && rangeCollector == nil {
+		panic("sstable: at least one interval collector must be provided")
+	}
+	bic := BlockIntervalCollector{
+		name:   name,
+		points: pointCollector,
+		ranges: rangeCollector,
+	}
+	if _, ok := pointCollector.(SuffixReplaceableBlockCollector); ok {
+		return &suffixReplacementBlockCollectorWrapper{bic}
+	}
+	return &bic
+}
+
+// Name implements the BlockPropertyCollector interface.
+func (b *BlockIntervalCollector) Name() string {
+	return b.name
+}
+
+// Add implements the BlockPropertyCollector interface.
+func (b *BlockIntervalCollector) Add(key InternalKey, value []byte) error {
+	if rangekey.IsRangeKey(key.Kind()) {
+		if b.ranges != nil {
+			return b.ranges.Add(key, value)
+		}
+	} else if b.points != nil {
+		return b.points.Add(key, value)
+	}
+	return nil
+}
+
+// FinishDataBlock implements the BlockPropertyCollector interface.
+func (b *BlockIntervalCollector) FinishDataBlock(buf []byte) ([]byte, error) {
+	if b.points == nil {
+		return buf, nil
+	}
+	var err error
+	b.blockInterval.lower, b.blockInterval.upper, err = b.points.FinishDataBlock()
+	if err != nil {
+		return buf, err
+	}
+	buf = b.blockInterval.encode(buf)
+	b.tableInterval.union(b.blockInterval)
+	return buf, nil
+}
+
+// AddPrevDataBlockToIndexBlock implements the BlockPropertyCollector
+// interface.
+func (b *BlockIntervalCollector) AddPrevDataBlockToIndexBlock() {
+	b.indexInterval.union(b.blockInterval)
+	b.blockInterval = interval{}
+}
+
+// FinishIndexBlock implements the BlockPropertyCollector interface.
+func (b *BlockIntervalCollector) FinishIndexBlock(buf []byte) ([]byte, error) {
+	buf = b.indexInterval.encode(buf)
+	b.indexInterval = interval{}
+	return buf, nil
+}
+
+// FinishTable implements the BlockPropertyCollector interface.
+func (b *BlockIntervalCollector) FinishTable(buf []byte) ([]byte, error) {
+	// If the collector is tracking range keys, the range key interval is union-ed
+	// with the point key interval for the table.
+	if b.ranges != nil {
+		var rangeInterval interval
+		var err error
+		rangeInterval.lower, rangeInterval.upper, err = b.ranges.FinishDataBlock()
+		if err != nil {
+			return buf, err
+		}
+		b.tableInterval.union(rangeInterval)
+	}
+	return b.tableInterval.encode(buf), nil
+}
+
+type interval struct {
+	lower uint64
+	upper uint64
+}
+
+func (i interval) encode(buf []byte) []byte {
+	if i.lower < i.upper {
+		var encoded [binary.MaxVarintLen64 * 2]byte
+		n := binary.PutUvarint(encoded[:], i.lower)
+		n += binary.PutUvarint(encoded[n:], i.upper-i.lower)
+		buf = append(buf, encoded[:n]...)
+	}
+	return buf
+}
+
+func (i *interval) decode(buf []byte) error {
+	if len(buf) == 0 {
+		*i = interval{}
+		return nil
+	}
+	var n int
+	i.lower, n = binary.Uvarint(buf)
+	if n <= 0 || n >= len(buf) {
+		return base.CorruptionErrorf("cannot decode interval from buf %x", buf)
+	}
+	pos := n
+	i.upper, n = binary.Uvarint(buf[pos:])
+	pos += n
+	if pos != len(buf) || n <= 0 {
+		return base.CorruptionErrorf("cannot decode interval from buf %x", buf)
+	}
+	// Delta decode.
+	i.upper += i.lower
+	if i.upper < i.lower {
+		return base.CorruptionErrorf("unexpected overflow, upper %d < lower %d", i.upper, i.lower)
+	}
+	return nil
+}
+
+func (i *interval) union(x interval) {
+	if x.lower >= x.upper {
+		// x is the empty set.
+		return
+	}
+	if i.lower >= i.upper {
+		// i is the empty set.
+		*i = x
+		return
+	}
+	// Both sets are non-empty.
+	if x.lower < i.lower {
+		i.lower = x.lower
+	}
+	if x.upper > i.upper {
+		i.upper = x.upper
+	}
+}
+
+func (i interval) intersects(x interval) bool {
+	if i.lower >= i.upper || x.lower >= x.upper {
+		// At least one of the sets is empty.
+		return false
+	}
+	// Neither set is empty.
+	return i.upper > x.lower && i.lower < x.upper
+}
+
+type suffixReplacementBlockCollectorWrapper struct {
+	BlockIntervalCollector
+}
+
+// UpdateKeySuffixes implements the SuffixReplaceableBlockCollector interface.
+func (w *suffixReplacementBlockCollectorWrapper) UpdateKeySuffixes(
+	oldProp []byte, from, to []byte,
+) error {
+	return w.BlockIntervalCollector.points.(SuffixReplaceableBlockCollector).UpdateKeySuffixes(oldProp, from, to)
+}
+
+// BlockIntervalFilter is an implementation of BlockPropertyFilter when the
+// corresponding collector is a BlockIntervalCollector. That is, the set is of
+// the form [lower, upper).
+type BlockIntervalFilter struct {
+	name           string
+	filterInterval interval
+}
+
+var _ BlockPropertyFilter = (*BlockIntervalFilter)(nil)
+
+// NewBlockIntervalFilter constructs a BlockPropertyFilter that filters blocks
+// based on an interval property collected by BlockIntervalCollector and the
+// given [lower, upper) bounds. The given name specifies the
+// BlockIntervalCollector's properties to read.
+func NewBlockIntervalFilter(name string, lower uint64, upper uint64) *BlockIntervalFilter {
+	b := new(BlockIntervalFilter)
+	b.Init(name, lower, upper)
+	return b
+}
+
+// Init initializes (or re-initializes, clearing previous state) an existing
+// BLockPropertyFilter to filter blocks based on an interval property collected
+// by BlockIntervalCollector and the given [lower, upper) bounds. The given name
+// specifies the BlockIntervalCollector's properties to read.
+func (b *BlockIntervalFilter) Init(name string, lower, upper uint64) {
+	*b = BlockIntervalFilter{
+		name:           name,
+		filterInterval: interval{lower: lower, upper: upper},
+	}
+}
+
+// Name implements the BlockPropertyFilter interface.
+func (b *BlockIntervalFilter) Name() string {
+	return b.name
+}
+
+// Intersects implements the BlockPropertyFilter interface.
+func (b *BlockIntervalFilter) Intersects(prop []byte) (bool, error) {
+	var i interval
+	if err := i.decode(prop); err != nil {
+		return false, err
+	}
+	return i.intersects(b.filterInterval), nil
+}
+
+// SetInterval adjusts the [lower, upper) bounds used by the filter. It is not
+// generally safe to alter the filter while it's in use, except as part of the
+// implementation of BlockPropertyFilterMask.SetSuffix used for range-key
+// masking.
+func (b *BlockIntervalFilter) SetInterval(lower, upper uint64) {
+	b.filterInterval = interval{lower: lower, upper: upper}
+}
+
+// When encoding block properties for each block, we cannot afford to encode
+// the name. Instead, the name is mapped to a shortID, in the scope of that
+// sstable, and the shortID is encoded. Since we use a uint8, there is a limit
+// of 256 block property collectors per sstable.
+type shortID uint8
+
+type blockPropertiesEncoder struct {
+	propsBuf []byte
+	scratch  []byte
+}
+
+func (e *blockPropertiesEncoder) getScratchForProp() []byte {
+	return e.scratch[:0]
+}
+
+func (e *blockPropertiesEncoder) resetProps() {
+	e.propsBuf = e.propsBuf[:0]
+}
+
+func (e *blockPropertiesEncoder) addProp(id shortID, scratch []byte) {
+	const lenID = 1
+	lenProp := uvarintLen(uint32(len(scratch)))
+	n := lenID + lenProp + len(scratch)
+	if cap(e.propsBuf)-len(e.propsBuf) < n {
+		size := len(e.propsBuf) + 2*n
+		if size < 2*cap(e.propsBuf) {
+			size = 2 * cap(e.propsBuf)
+		}
+		buf := make([]byte, len(e.propsBuf), size)
+		copy(buf, e.propsBuf)
+		e.propsBuf = buf
+	}
+	pos := len(e.propsBuf)
+	b := e.propsBuf[pos : pos+lenID]
+	b[0] = byte(id)
+	pos += lenID
+	b = e.propsBuf[pos : pos+lenProp]
+	n = binary.PutUvarint(b, uint64(len(scratch)))
+	pos += n
+	b = e.propsBuf[pos : pos+len(scratch)]
+	pos += len(scratch)
+	copy(b, scratch)
+	e.propsBuf = e.propsBuf[0:pos]
+	e.scratch = scratch
+}
+
+func (e *blockPropertiesEncoder) unsafeProps() []byte {
+	return e.propsBuf
+}
+
+func (e *blockPropertiesEncoder) props() []byte {
+	buf := make([]byte, len(e.propsBuf))
+	copy(buf, e.propsBuf)
+	return buf
+}
+
+type blockPropertiesDecoder struct {
+	props []byte
+}
+
+func (d *blockPropertiesDecoder) done() bool {
+	return len(d.props) == 0
+}
+
+// REQUIRES: !done()
+func (d *blockPropertiesDecoder) next() (id shortID, prop []byte, err error) {
+	const lenID = 1
+	id = shortID(d.props[0])
+	propLen, m := binary.Uvarint(d.props[lenID:])
+	n := lenID + m
+	if m <= 0 || propLen == 0 || (n+int(propLen)) > len(d.props) {
+		return 0, nil, base.CorruptionErrorf("corrupt block property length")
+	}
+	prop = d.props[n : n+int(propLen)]
+	d.props = d.props[n+int(propLen):]
+	return id, prop, nil
+}
+
+// BlockPropertiesFilterer provides filtering support when reading an sstable
+// in the context of an iterator that has a slice of BlockPropertyFilters.
+// After the call to NewBlockPropertiesFilterer, the caller must call
+// IntersectsUserPropsAndFinishInit to check if the sstable intersects with
+// the filters. If it does intersect, this function also finishes initializing
+// the BlockPropertiesFilterer using the shortIDs for the relevant filters.
+// Subsequent checks for relevance of a block should use the intersects
+// method.
+type BlockPropertiesFilterer struct {
+	filters []BlockPropertyFilter
+	// Maps shortID => index in filters. This can be sparse, and shortIDs for
+	// which there is no filter are represented with an index of -1. The
+	// length of this can be shorter than the shortIDs allocated in the
+	// sstable. e.g. if the sstable used shortIDs 0, 1, 2, 3, and the iterator
+	// has two filters, corresponding to shortIDs 2, 0, this would be:
+	// len(shortIDToFiltersIndex)==3, 0=>1, 1=>-1, 2=>0.
+	shortIDToFiltersIndex []int
+
+	// boundLimitedFilter, if non-nil, holds a single block-property filter with
+	// additional constraints on its filtering. A boundLimitedFilter may only
+	// filter blocks that are wholly contained within its bounds. During forward
+	// iteration the lower bound (and during backward iteration the upper bound)
+	// must be externally guaranteed, with Intersects only returning false if
+	// that bound is met. The opposite bound is verified during iteration by the
+	// sstable iterator.
+	//
+	// boundLimitedFilter is permitted to be defined on a property (`Name()`)
+	// for which another filter exists in filters. In this case both filters
+	// will be consulted, and either filter may exclude block(s). Only a single
+	// bound-limited block-property filter may be set.
+	//
+	// The boundLimitedShortID field contains the shortID of the filter's
+	// property within the sstable. It's set to -1 if the property was not
+	// collected when the table was built.
+	boundLimitedFilter  BoundLimitedBlockPropertyFilter
+	boundLimitedShortID int
+}
+
+var blockPropertiesFiltererPool = sync.Pool{
+	New: func() interface{} {
+		return &BlockPropertiesFilterer{}
+	},
+}
+
+// newBlockPropertiesFilterer returns a partially initialized filterer. To complete
+// initialization, call IntersectsUserPropsAndFinishInit.
+func newBlockPropertiesFilterer(
+	filters []BlockPropertyFilter, limited BoundLimitedBlockPropertyFilter,
+) *BlockPropertiesFilterer {
+	filterer := blockPropertiesFiltererPool.Get().(*BlockPropertiesFilterer)
+	*filterer = BlockPropertiesFilterer{
+		filters:               filters,
+		shortIDToFiltersIndex: filterer.shortIDToFiltersIndex[:0],
+		boundLimitedFilter:    limited,
+		boundLimitedShortID:   -1,
+	}
+	return filterer
+}
+
+func releaseBlockPropertiesFilterer(filterer *BlockPropertiesFilterer) {
+	*filterer = BlockPropertiesFilterer{
+		shortIDToFiltersIndex: filterer.shortIDToFiltersIndex[:0],
+	}
+	blockPropertiesFiltererPool.Put(filterer)
+}
+
+// IntersectsTable evaluates the provided block-property filter against the
+// provided set of table-level properties. If there is no intersection between
+// the filters and the table or an error is encountered, IntersectsTable returns
+// a nil filterer (and possibly an error). If there is an intersection,
+// IntersectsTable returns a non-nil filterer that may be used by an iterator
+// reading the table.
+func IntersectsTable(
+	filters []BlockPropertyFilter,
+	limited BoundLimitedBlockPropertyFilter,
+	userProperties map[string]string,
+) (*BlockPropertiesFilterer, error) {
+	f := newBlockPropertiesFilterer(filters, limited)
+	ok, err := f.intersectsUserPropsAndFinishInit(userProperties)
+	if !ok || err != nil {
+		releaseBlockPropertiesFilterer(f)
+		return nil, err
+	}
+	return f, nil
+}
+
+// intersectsUserPropsAndFinishInit is called with the user properties map for
+// the sstable and returns whether the sstable intersects the filters. It
+// additionally initializes the shortIDToFiltersIndex for the filters that are
+// relevant to this sstable.
+func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
+	userProperties map[string]string,
+) (bool, error) {
+	for i := range f.filters {
+		props, ok := userProperties[f.filters[i].Name()]
+		if !ok {
+			// Collector was not used when writing this file, so it is
+			// considered intersecting.
+			continue
+		}
+		if len(props) < 1 {
+			return false, base.CorruptionErrorf(
+				"block properties for %s is corrupted", f.filters[i].Name())
+		}
+		shortID := shortID(props[0])
+		{
+			// Use an unsafe conversion to avoid allocating. Intersects() is not
+			// supposed to modify the given slice.
+			// Note that unsafe.StringData only works if the string is not empty
+			// (which we already checked).
+			byteProps := unsafe.Slice(unsafe.StringData(props), len(props))
+			intersects, err := f.filters[i].Intersects(byteProps[1:])
+			if err != nil || !intersects {
+				return false, err
+			}
+		}
+		// Intersects the sstable, so need to use this filter when
+		// deciding whether to read blocks.
+		n := len(f.shortIDToFiltersIndex)
+		if n <= int(shortID) {
+			if cap(f.shortIDToFiltersIndex) <= int(shortID) {
+				index := make([]int, shortID+1, 2*(shortID+1))
+				copy(index, f.shortIDToFiltersIndex)
+				f.shortIDToFiltersIndex = index
+			} else {
+				f.shortIDToFiltersIndex = f.shortIDToFiltersIndex[:shortID+1]
+			}
+			for j := n; j < int(shortID); j++ {
+				f.shortIDToFiltersIndex[j] = -1
+			}
+		}
+		f.shortIDToFiltersIndex[shortID] = i
+	}
+	if f.boundLimitedFilter == nil {
+		return true, nil
+	}
+
+	// There's a bound-limited filter. Find its shortID. It's possible that
+	// there's an existing filter in f.filters on the same property. That's
+	// okay. Both filters will be consulted whenever a relevant prop is decoded.
+	props, ok := userProperties[f.boundLimitedFilter.Name()]
+	if !ok {
+		// The collector was not used when writing this file, so it's
+		// intersecting. We leave f.boundLimitedShortID=-1, so the filter will
+		// be unused within this file.
+		return true, nil
+	}
+	if len(props) < 1 {
+		return false, base.CorruptionErrorf(
+			"block properties for %s is corrupted", f.boundLimitedFilter.Name())
+	}
+	f.boundLimitedShortID = int(props[0])
+
+	// We don't check for table-level intersection for the bound-limited filter.
+	// The bound-limited filter is treated as vacuously intersecting.
+	//
+	// NB: If a block-property filter needs to be toggled inactive/active, it
+	// should be implemented within the Intersects implementation.
+	//
+	// TODO(jackson): We could filter at the table-level by threading the table
+	// smallest and largest bounds here.
+
+	// The bound-limited filter isn't included in shortIDToFiltersIndex.
+	//
+	// When determining intersection, we decode props only up to the shortID
+	// len(shortIDToFiltersIndex). If f.limitedShortID is greater than any of
+	// the existing filters' shortIDs, we need to grow shortIDToFiltersIndex.
+	// Growing the index with -1s ensures we're able to consult the index
+	// without length checks.
+	if n := len(f.shortIDToFiltersIndex); n <= f.boundLimitedShortID {
+		if cap(f.shortIDToFiltersIndex) <= f.boundLimitedShortID {
+			index := make([]int, f.boundLimitedShortID+1)
+			copy(index, f.shortIDToFiltersIndex)
+			f.shortIDToFiltersIndex = index
+		} else {
+			f.shortIDToFiltersIndex = f.shortIDToFiltersIndex[:f.boundLimitedShortID+1]
+		}
+		for j := n; j <= f.boundLimitedShortID; j++ {
+			f.shortIDToFiltersIndex[j] = -1
+		}
+	}
+	return true, nil
+}
+
+type intersectsResult int8
+
+const (
+	blockIntersects intersectsResult = iota
+	blockExcluded
+	// blockMaybeExcluded is returned by BlockPropertiesFilterer.intersects when
+	// no filters unconditionally exclude the block, but the bound-limited block
+	// property filter will exclude it if the block's bounds fall within the
+	// filter's current bounds. See the reader's
+	// {single,two}LevelIterator.resolveMaybeExcluded methods.
+	blockMaybeExcluded
+)
+
+func (f *BlockPropertiesFilterer) intersects(props []byte) (ret intersectsResult, err error) {
+	i := 0
+	decoder := blockPropertiesDecoder{props: props}
+	ret = blockIntersects
+	for i < len(f.shortIDToFiltersIndex) {
+		var id int
+		var prop []byte
+		if !decoder.done() {
+			var shortID shortID
+			var err error
+			shortID, prop, err = decoder.next()
+			if err != nil {
+				return ret, err
+			}
+			id = int(shortID)
+		} else {
+			id = math.MaxUint8 + 1
+		}
+		for i < len(f.shortIDToFiltersIndex) && id > i {
+			// The property for this id is not encoded for this block, but there
+			// may still be a filter for this id.
+			if intersects, err := f.intersectsFilter(i, nil); err != nil {
+				return ret, err
+			} else if intersects == blockExcluded {
+				return blockExcluded, nil
+			} else if intersects == blockMaybeExcluded {
+				ret = blockMaybeExcluded
+			}
+			i++
+		}
+		if i >= len(f.shortIDToFiltersIndex) {
+			return ret, nil
+		}
+		// INVARIANT: id <= i. And since i is always incremented by 1, id==i.
+		if id != i {
+			panic(fmt.Sprintf("%d != %d", id, i))
+		}
+		if intersects, err := f.intersectsFilter(i, prop); err != nil {
+			return ret, err
+		} else if intersects == blockExcluded {
+			return blockExcluded, nil
+		} else if intersects == blockMaybeExcluded {
+			ret = blockMaybeExcluded
+		}
+		i++
+	}
+	// ret == blockIntersects || ret == blockMaybeExcluded
+	return ret, nil
+}
+
+func (f *BlockPropertiesFilterer) intersectsFilter(i int, prop []byte) (intersectsResult, error) {
+	if f.shortIDToFiltersIndex[i] >= 0 {
+		intersects, err := f.filters[f.shortIDToFiltersIndex[i]].Intersects(prop)
+		if err != nil {
+			return blockIntersects, err
+		}
+		if !intersects {
+			return blockExcluded, nil
+		}
+	}
+	if i == f.boundLimitedShortID {
+		// The bound-limited filter uses this id.
+		//
+		// The bound-limited filter only applies within a keyspan interval. We
+		// expect the Intersects call to be cheaper than bounds checks. If
+		// Intersects determines that there is no intersection, we return
+		// `blockMaybeExcluded` if no other bpf unconditionally excludes the
+		// block.
+		intersects, err := f.boundLimitedFilter.Intersects(prop)
+		if err != nil {
+			return blockIntersects, err
+		} else if !intersects {
+			return blockMaybeExcluded, nil
+		}
+	}
+	return blockIntersects, nil
+}

vendor/github.com/cockroachdb/pebble/sstable/block_property_test_utils.go

@@ -0,0 +1,117 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"math"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/testkeys"
+)
+
+// Code in this file contains utils for testing. It implements interval block
+// property collectors and filters on the suffixes of keys in the format used
+// by the testkeys package (eg, 'key@5').
+
+const testKeysBlockPropertyName = `pebble.internal.testkeys.suffixes`
+
+// NewTestKeysBlockPropertyCollector constructs a sstable property collector
+// over testkey suffixes.
+func NewTestKeysBlockPropertyCollector() BlockPropertyCollector {
+	return NewBlockIntervalCollector(
+		testKeysBlockPropertyName,
+		&testKeysSuffixIntervalCollector{},
+		nil)
+}
+
+// NewTestKeysBlockPropertyFilter constructs a new block-property filter that excludes
+// blocks containing exclusively suffixed keys where all the suffixes fall
+// outside of the range [filterMin, filterMax).
+//
+// The filter only filters based on data derived from the key. The iteration
+// results of this block property filter are deterministic for unsuffixed keys
+// and keys with suffixes within the range [filterMin, filterMax). For keys with
+// suffixes outside the range, iteration is nondeterministic.
+func NewTestKeysBlockPropertyFilter(filterMin, filterMax uint64) *BlockIntervalFilter {
+	return NewBlockIntervalFilter(testKeysBlockPropertyName, filterMin, filterMax)
+}
+
+// NewTestKeysMaskingFilter constructs a TestKeysMaskingFilter that implements
+// pebble.BlockPropertyFilterMask for efficient range-key masking using the
+// testkeys block property filter. The masking filter wraps a block interval
+// filter, and modifies the configured interval when Pebble requests it.
+func NewTestKeysMaskingFilter() TestKeysMaskingFilter {
+	return TestKeysMaskingFilter{BlockIntervalFilter: NewTestKeysBlockPropertyFilter(0, math.MaxUint64)}
+}
+
+// TestKeysMaskingFilter implements BlockPropertyFilterMask and may be used to mask
+// point keys with the testkeys-style suffixes (eg, @4) that are masked by range
+// keys with testkeys-style suffixes.
+type TestKeysMaskingFilter struct {
+	*BlockIntervalFilter
+}
+
+// SetSuffix implements pebble.BlockPropertyFilterMask.
+func (f TestKeysMaskingFilter) SetSuffix(suffix []byte) error {
+	ts, err := testkeys.ParseSuffix(suffix)
+	if err != nil {
+		return err
+	}
+	f.BlockIntervalFilter.SetInterval(uint64(ts), math.MaxUint64)
+	return nil
+}
+
+// Intersects implements the BlockPropertyFilter interface.
+func (f TestKeysMaskingFilter) Intersects(prop []byte) (bool, error) {
+	return f.BlockIntervalFilter.Intersects(prop)
+}
+
+var _ DataBlockIntervalCollector = (*testKeysSuffixIntervalCollector)(nil)
+
+// testKeysSuffixIntervalCollector maintains an interval over the timestamps in
+// MVCC-like suffixes for keys (e.g. foo@123).
+type testKeysSuffixIntervalCollector struct {
+	initialized  bool
+	lower, upper uint64
+}
+
+// Add implements DataBlockIntervalCollector by adding the timestamp(s) in the
+// suffix(es) of this record to the current interval.
+//
+// Note that range sets and unsets may have multiple suffixes. Range key deletes
+// do not have a suffix. All other point keys have a single suffix.
+func (c *testKeysSuffixIntervalCollector) Add(key base.InternalKey, value []byte) error {
+	i := testkeys.Comparer.Split(key.UserKey)
+	if i == len(key.UserKey) {
+		c.initialized = true
+		c.lower, c.upper = 0, math.MaxUint64
+		return nil
+	}
+	ts, err := testkeys.ParseSuffix(key.UserKey[i:])
+	if err != nil {
+		return err
+	}
+	uts := uint64(ts)
+	if !c.initialized {
+		c.lower, c.upper = uts, uts+1
+		c.initialized = true
+		return nil
+	}
+	if uts < c.lower {
+		c.lower = uts
+	}
+	if uts >= c.upper {
+		c.upper = uts + 1
+	}
+	return nil
+}
+
+// FinishDataBlock implements DataBlockIntervalCollector.
+func (c *testKeysSuffixIntervalCollector) FinishDataBlock() (lower, upper uint64, err error) {
+	l, u := c.lower, c.upper
+	c.lower, c.upper = 0, 0
+	c.initialized = false
+	return l, u, nil
+}

vendor/github.com/cockroachdb/pebble/sstable/buffer_pool.go

@@ -0,0 +1,148 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/cache"
+)
+
+// A bufferHandle is a handle to manually-managed memory. The handle may point
+// to a block in the block cache (h.Get() != nil), or a buffer that exists
+// outside the block cache allocated from a BufferPool (b.Valid()).
+type bufferHandle struct {
+	h cache.Handle
+	b Buf
+}
+
+// Get retrieves the underlying buffer referenced by the handle.
+func (bh bufferHandle) Get() []byte {
+	if v := bh.h.Get(); v != nil {
+		return v
+	} else if bh.b.p != nil {
+		return bh.b.p.pool[bh.b.i].b
+	}
+	return nil
+}
+
+// Release releases the buffer, either back to the block cache or BufferPool.
+func (bh bufferHandle) Release() {
+	bh.h.Release()
+	bh.b.Release()
+}
+
+// A BufferPool holds a pool of buffers for holding sstable blocks. An initial
+// size of the pool is provided on Init, but a BufferPool will grow to meet the
+// largest working set size. It'll never shrink. When a buffer is released, the
+// BufferPool recycles the buffer for future allocations.
+//
+// A BufferPool should only be used for short-lived allocations with
+// well-understood working set sizes to avoid excessive memory consumption.
+//
+// BufferPool is not thread-safe.
+type BufferPool struct {
+	// pool contains all the buffers held by the pool, including buffers that
+	// are in-use. For every i < len(pool): pool[i].v is non-nil.
+	pool []allocedBuffer
+}
+
+type allocedBuffer struct {
+	v *cache.Value
+	// b holds the current byte slice. It's backed by v, but may be a subslice
+	// of v's memory while the buffer is in-use [ len(b) ≤ len(v.Buf()) ].
+	//
+	// If the buffer is not currently in-use, b is nil. When being recycled, the
+	// BufferPool.Alloc will reset b to be a subslice of v.Buf().
+	b []byte
+}
+
+// Init initializes the pool with an initial working set buffer size of
+// `initialSize`.
+func (p *BufferPool) Init(initialSize int) {
+	*p = BufferPool{
+		pool: make([]allocedBuffer, 0, initialSize),
+	}
+}
+
+// initPreallocated is like Init but for internal sstable package use in
+// instances where a pre-allocated slice of []allocedBuffer already exists. It's
+// used to avoid an extra allocation initializing BufferPool.pool.
+func (p *BufferPool) initPreallocated(pool []allocedBuffer) {
+	*p = BufferPool{
+		pool: pool[:0],
+	}
+}
+
+// Release releases all buffers held by the pool and resets the pool to an
+// uninitialized state.
+func (p *BufferPool) Release() {
+	for i := range p.pool {
+		if p.pool[i].b != nil {
+			panic(errors.AssertionFailedf("Release called on a BufferPool with in-use buffers"))
+		}
+		cache.Free(p.pool[i].v)
+	}
+	*p = BufferPool{}
+}
+
+// Alloc allocates a new buffer of size n. If the pool already holds a buffer at
+// least as large as n, the pooled buffer is used instead.
+//
+// Alloc is O(MAX(N,M)) where N is the largest number of concurrently in-use
+// buffers allocated and M is the initialSize passed to Init.
+func (p *BufferPool) Alloc(n int) Buf {
+	unusableBufferIdx := -1
+	for i := 0; i < len(p.pool); i++ {
+		if p.pool[i].b == nil {
+			if len(p.pool[i].v.Buf()) >= n {
+				p.pool[i].b = p.pool[i].v.Buf()[:n]
+				return Buf{p: p, i: i}
+			}
+			unusableBufferIdx = i
+		}
+	}
+
+	// If we would need to grow the size of the pool to allocate another buffer,
+	// but there was a slot available occupied by a buffer that's just too
+	// small, replace the too-small buffer.
+	if len(p.pool) == cap(p.pool) && unusableBufferIdx >= 0 {
+		i := unusableBufferIdx
+		cache.Free(p.pool[i].v)
+		p.pool[i].v = cache.Alloc(n)
+		p.pool[i].b = p.pool[i].v.Buf()
+		return Buf{p: p, i: i}
+	}
+
+	// Allocate a new buffer.
+	v := cache.Alloc(n)
+	p.pool = append(p.pool, allocedBuffer{v: v, b: v.Buf()[:n]})
+	return Buf{p: p, i: len(p.pool) - 1}
+}
+
+// A Buf holds a reference to a manually-managed, pooled byte buffer.
+type Buf struct {
+	p *BufferPool
+	// i holds the index into p.pool where the buffer may be found. This scheme
+	// avoids needing to allocate the handle to the buffer on the heap at the
+	// cost of copying two words instead of one.
+	i int
+}
+
+// Valid returns true if the buf holds a valid buffer.
+func (b Buf) Valid() bool {
+	return b.p != nil
+}
+
+// Release releases the buffer back to the pool.
+func (b *Buf) Release() {
+	if b.p == nil {
+		return
+	}
+	// Clear the allocedBuffer's byte slice. This signals the allocated buffer
+	// is no longer in use and a future call to BufferPool.Alloc may reuse this
+	// buffer.
+	b.p.pool[b.i].b = nil
+	b.p = nil
+}

vendor/github.com/cockroachdb/pebble/sstable/comparer.go

@@ -0,0 +1,34 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Compare exports the base.Compare type.
+type Compare = base.Compare
+
+// Equal exports the base.Equal type.
+type Equal = base.Equal
+
+// AbbreviatedKey exports the base.AbbreviatedKey type.
+type AbbreviatedKey = base.AbbreviatedKey
+
+// Separator exports the base.Separator type.
+type Separator = base.Separator
+
+// Successor exports the base.Successor type.
+type Successor = base.Successor
+
+// Split exports the base.Split type.
+type Split = base.Split
+
+// Comparer exports the base.Comparer type.
+type Comparer = base.Comparer
+
+// DefaultComparer exports the base.DefaultComparer variable.
+var DefaultComparer = base.DefaultComparer
+
+// Merger exports the base.Merger type.
+type Merger = base.Merger

vendor/github.com/cockroachdb/pebble/sstable/compression.go

@@ -0,0 +1,99 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"encoding/binary"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/cache"
+	"github.com/golang/snappy"
+)
+
+func decompressedLen(blockType blockType, b []byte) (int, int, error) {
+	switch blockType {
+	case noCompressionBlockType:
+		return 0, 0, nil
+	case snappyCompressionBlockType:
+		l, err := snappy.DecodedLen(b)
+		return l, 0, err
+	case zstdCompressionBlockType:
+		// This will also be used by zlib, bzip2 and lz4 to retrieve the decodedLen
+		// if we implement these algorithms in the future.
+		decodedLenU64, varIntLen := binary.Uvarint(b)
+		if varIntLen <= 0 {
+			return 0, 0, base.CorruptionErrorf("pebble/table: compression block has invalid length")
+		}
+		return int(decodedLenU64), varIntLen, nil
+	default:
+		return 0, 0, base.CorruptionErrorf("pebble/table: unknown block compression: %d", errors.Safe(blockType))
+	}
+}
+
+func decompressInto(blockType blockType, compressed []byte, buf []byte) ([]byte, error) {
+	var result []byte
+	var err error
+	switch blockType {
+	case snappyCompressionBlockType:
+		result, err = snappy.Decode(buf, compressed)
+	case zstdCompressionBlockType:
+		result, err = decodeZstd(buf, compressed)
+	}
+	if err != nil {
+		return nil, base.MarkCorruptionError(err)
+	}
+	if len(result) != 0 && (len(result) != len(buf) || &result[0] != &buf[0]) {
+		return nil, base.CorruptionErrorf("pebble/table: decompressed into unexpected buffer: %p != %p",
+			errors.Safe(result), errors.Safe(buf))
+	}
+	return result, nil
+}
+
+// decompressBlock decompresses an SST block, with manually-allocated space.
+// NB: If decompressBlock returns (nil, nil), no decompression was necessary and
+// the caller may use `b` directly.
+func decompressBlock(blockType blockType, b []byte) (*cache.Value, error) {
+	if blockType == noCompressionBlockType {
+		return nil, nil
+	}
+	// first obtain the decoded length.
+	decodedLen, prefixLen, err := decompressedLen(blockType, b)
+	if err != nil {
+		return nil, err
+	}
+	b = b[prefixLen:]
+	// Allocate sufficient space from the cache.
+	decoded := cache.Alloc(decodedLen)
+	decodedBuf := decoded.Buf()
+	if _, err := decompressInto(blockType, b, decodedBuf); err != nil {
+		cache.Free(decoded)
+		return nil, err
+	}
+	return decoded, nil
+}
+
+// compressBlock compresses an SST block, using compressBuf as the desired destination.
+func compressBlock(
+	compression Compression, b []byte, compressedBuf []byte,
+) (blockType blockType, compressed []byte) {
+	switch compression {
+	case SnappyCompression:
+		return snappyCompressionBlockType, snappy.Encode(compressedBuf, b)
+	case NoCompression:
+		return noCompressionBlockType, b
+	}
+
+	if len(compressedBuf) < binary.MaxVarintLen64 {
+		compressedBuf = append(compressedBuf, make([]byte, binary.MaxVarintLen64-len(compressedBuf))...)
+	}
+	varIntLen := binary.PutUvarint(compressedBuf, uint64(len(b)))
+	switch compression {
+	case ZstdCompression:
+		return zstdCompressionBlockType, encodeZstd(compressedBuf, varIntLen, b)
+	default:
+		return noCompressionBlockType, b
+	}
+}

vendor/github.com/cockroachdb/pebble/sstable/compression_cgo.go

@@ -0,0 +1,34 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build cgo
+// +build cgo
+
+package sstable
+
+import (
+	"bytes"
+
+	"github.com/DataDog/zstd"
+)
+
+// decodeZstd decompresses b with the Zstandard algorithm.
+// It reuses the preallocated capacity of decodedBuf if it is sufficient.
+// On success, it returns the decoded byte slice.
+func decodeZstd(decodedBuf, b []byte) ([]byte, error) {
+	return zstd.Decompress(decodedBuf, b)
+}
+
+// encodeZstd compresses b with the Zstandard algorithm at default compression
+// level (level 3). It reuses the preallocated capacity of compressedBuf if it
+// is sufficient. The subslice `compressedBuf[:varIntLen]` should already encode
+// the length of `b` before calling encodeZstd. It returns the encoded byte
+// slice, including the `compressedBuf[:varIntLen]` prefix.
+func encodeZstd(compressedBuf []byte, varIntLen int, b []byte) []byte {
+	buf := bytes.NewBuffer(compressedBuf[:varIntLen])
+	writer := zstd.NewWriterLevel(buf, 3)
+	writer.Write(b)
+	writer.Close()
+	return buf.Bytes()
+}

vendor/github.com/cockroachdb/pebble/sstable/compression_nocgo.go

@@ -0,0 +1,30 @@
+// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !cgo
+// +build !cgo
+
+package sstable
+
+import "github.com/klauspost/compress/zstd"
+
+// decodeZstd decompresses b with the Zstandard algorithm.
+// It reuses the preallocated capacity of decodedBuf if it is sufficient.
+// On success, it returns the decoded byte slice.
+func decodeZstd(decodedBuf, b []byte) ([]byte, error) {
+	decoder, _ := zstd.NewReader(nil)
+	defer decoder.Close()
+	return decoder.DecodeAll(b, decodedBuf[:0])
+}
+
+// encodeZstd compresses b with the Zstandard algorithm at default compression
+// level (level 3). It reuses the preallocated capacity of compressedBuf if it
+// is sufficient. The subslice `compressedBuf[:varIntLen]` should already encode
+// the length of `b` before calling encodeZstd. It returns the encoded byte
+// slice, including the `compressedBuf[:varIntLen]` prefix.
+func encodeZstd(compressedBuf []byte, varIntLen int, b []byte) []byte {
+	encoder, _ := zstd.NewWriter(nil)
+	defer encoder.Close()
+	return encoder.EncodeAll(b, compressedBuf[:varIntLen])
+}

vendor/github.com/cockroachdb/pebble/sstable/filter.go

@@ -0,0 +1,122 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import "sync/atomic"
+
+// FilterMetrics holds metrics for the filter policy.
+type FilterMetrics struct {
+	// The number of hits for the filter policy. This is the
+	// number of times the filter policy was successfully used to avoid access
+	// of a data block.
+	Hits int64
+	// The number of misses for the filter policy. This is the number of times
+	// the filter policy was checked but was unable to filter an access of a data
+	// block.
+	Misses int64
+}
+
+// FilterMetricsTracker is used to keep track of filter metrics. It contains the
+// same metrics as FilterMetrics, but they can be updated atomically. An
+// instance of FilterMetricsTracker can be passed to a Reader as a ReaderOption.
+type FilterMetricsTracker struct {
+	// See FilterMetrics.Hits.
+	hits atomic.Int64
+	// See FilterMetrics.Misses.
+	misses atomic.Int64
+}
+
+var _ ReaderOption = (*FilterMetricsTracker)(nil)
+
+func (m *FilterMetricsTracker) readerApply(r *Reader) {
+	if r.tableFilter != nil {
+		r.tableFilter.metrics = m
+	}
+}
+
+// Load returns the current values as FilterMetrics.
+func (m *FilterMetricsTracker) Load() FilterMetrics {
+	return FilterMetrics{
+		Hits:   m.hits.Load(),
+		Misses: m.misses.Load(),
+	}
+}
+
+// BlockHandle is the file offset and length of a block.
+type BlockHandle struct {
+	Offset, Length uint64
+}
+
+// BlockHandleWithProperties is used for data blocks and first/lower level
+// index blocks, since they can be annotated using BlockPropertyCollectors.
+type BlockHandleWithProperties struct {
+	BlockHandle
+	Props []byte
+}
+
+type filterWriter interface {
+	addKey(key []byte)
+	finish() ([]byte, error)
+	metaName() string
+	policyName() string
+}
+
+type tableFilterReader struct {
+	policy  FilterPolicy
+	metrics *FilterMetricsTracker
+}
+
+func newTableFilterReader(policy FilterPolicy) *tableFilterReader {
+	return &tableFilterReader{
+		policy:  policy,
+		metrics: nil,
+	}
+}
+
+func (f *tableFilterReader) mayContain(data, key []byte) bool {
+	mayContain := f.policy.MayContain(TableFilter, data, key)
+	if f.metrics != nil {
+		if mayContain {
+			f.metrics.misses.Add(1)
+		} else {
+			f.metrics.hits.Add(1)
+		}
+	}
+	return mayContain
+}
+
+type tableFilterWriter struct {
+	policy FilterPolicy
+	writer FilterWriter
+	// count is the count of the number of keys added to the filter.
+	count int
+}
+
+func newTableFilterWriter(policy FilterPolicy) *tableFilterWriter {
+	return &tableFilterWriter{
+		policy: policy,
+		writer: policy.NewWriter(TableFilter),
+	}
+}
+
+func (f *tableFilterWriter) addKey(key []byte) {
+	f.count++
+	f.writer.AddKey(key)
+}
+
+func (f *tableFilterWriter) finish() ([]byte, error) {
+	if f.count == 0 {
+		return nil, nil
+	}
+	return f.writer.Finish(nil), nil
+}
+
+func (f *tableFilterWriter) metaName() string {
+	return "fullfilter." + f.policy.Name()
+}
+
+func (f *tableFilterWriter) policyName() string {
+	return f.policy.Name()
+}

vendor/github.com/cockroachdb/pebble/sstable/format.go

@@ -0,0 +1,257 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+// TableFormat specifies the format version for sstables. The legacy LevelDB
+// format is format version 1.
+type TableFormat uint32
+
+// The available table formats, representing the tuple (magic number, version
+// number). Note that these values are not (and should not) be serialized to
+// disk. The ordering should follow the order the versions were introduced to
+// Pebble (i.e. the history is linear).
+const (
+	TableFormatUnspecified TableFormat = iota
+	TableFormatLevelDB
+	TableFormatRocksDBv2
+	TableFormatPebblev1 // Block properties.
+	TableFormatPebblev2 // Range keys.
+	TableFormatPebblev3 // Value blocks.
+	TableFormatPebblev4 // DELSIZED tombstones.
+	NumTableFormats
+
+	TableFormatMax = NumTableFormats - 1
+)
+
+// TableFormatPebblev4, in addition to DELSIZED, introduces the use of
+// InternalKeyKindSSTableInternalObsoleteBit.
+//
+// 1. Motivation
+//
+// We have various related problems caused by Pebble snapshots:
+//
+// - P1: RANGEDELs that delete points in the same sstable, but the points
+//   happen to not get deleted during compactions because of an open snapshot.
+//   This causes very expensive iteration, that has been observed in
+//   production deployments
+//
+// - P2: When iterating over a foreign sstable (in disaggregated storage), we
+//   need to do (a) point collapsing to expose at most one point per user key,
+//   (b) apply RANGEDELs in the sstable to hide deleted points in the same
+//   sstable. This per-sstable point collapsing iteration needs to be very
+//   efficient (ideally as efficient from a CPU perspective as iteration over
+//   regular sstables) since foreign sstables can be very long-lived -- one of
+//   the goals of disaggregated storage is to scale compute and disk bandwidth
+//   resources as a function of the hot (from a write perspective) data and
+//   not the whole data, so we don't want to have to rewrite foreign sstables
+//   solely to improve read performance.
+//
+// The ideal solution for P2 would allow user-facing reads to utilize the
+// existing SST iterators (with slight modifications) and with no loss of
+// efficiency. And for P1 and P2 we would like to skip whole blocks of
+// overwritten/deleted points. Even when we can't skip whole blocks, avoiding
+// key comparisons at iteration time to discover what points are deleted is
+// very desirable, since keys can be long.
+//
+// We observe that:
+//
+// - Reads:
+//   - All user-facing reads in CockroachDB use iterators over the DB, hence
+//     have a higher read seqnum than all sstables (there are some rare cases
+//     that can violate this, but those are not important from a performance
+//     optimization perspective).
+//
+//   - Certain internal-facing reads in CockroachDB use snapshots, but the
+//     snapshots are shortlived enough that most L5 and L6 sstables will have
+//     all seqnums lower than the snapshot seqnum.
+//
+// - Writes:
+//   - We already do key comparisons between points when writing the sstable
+//     to ensure that the sstable invariant (monotonically increasing internal
+//     keys) is not violated. So we know which points share the same userkey,
+//     and thereby which points are obsolete because there is a more recent
+//     point in the same sstable.
+//
+//   - The compactionIter knows which point id deleted by a RANGEDEL even if
+//     the point does need to be written because of a snapshot.
+//
+//   So this known information can be encoded in the sstable at write time and
+//   utilized for optimized reading.
+//
+// 2. Solution
+//
+// We primarily scope the solution to the following point kinds: SET,
+// SETWITHDEL, DEL, DELSIZED, SINGLEDEL. These are the ones marked locally
+// obsolete, i.e., obsolete within the sstable, and we can guarantee that at
+// most one point will be exposed per user key. MERGE keys create more
+// complexity: MERGE followed by MERGE causes multiple keys to not be
+// obsolete. Same applies for MERGE followed by SET/SETWITHDEL/DEL*. Note
+// that:
+//
+// - For regular sst iteration, the obsolete marking is a performance
+//   optimization, and multiple keys for the same userkey can be handled by
+//   higher layers in the iterator tree (specifically pebble.Iterator).
+//
+// - For foreign sst iteration, we disallow MERGEs to be written to such
+//   shared ssts (details below).
+//
+// The key kinds are marked with an obsolete bit
+// (InternalKeyKindSSTableInternalObsoleteBit) when the key-value pair is
+// obsolete. This marking is done within blockWriter, based on information
+// passed to it by Writer. In turn, Writer uses a combination of key
+// comparisons, and information provided by compactionIter to decide whether a
+// key-value pair is obsolete. Additionally, a Pebble-internal
+// BlockPropertyCollector (obsoleteKeyBlockPropertyCollector) is used to mark
+// blocks where all key-value pairs are obsolete. Since the common case is
+// non-obsolete blocks, this block property collector uses the empty byte
+// slice to represent a non-obsolete block, which consumes no space in
+// BlockHandleWithProperties.Props.
+//
+// At read time, the obsolete bit is only visible to the blockIter, which can
+// be optionally configured to hide obsolete points. This hiding is only
+// configured for data block iterators for sstables being read by user-facing
+// iterators at a seqnum greater than the max seqnum in the sstable.
+// Additionally, when this hiding is configured, a Pebble-internal block
+// property filter (obsoleteKeyBlockPropertyFilter), is used to skip whole
+// blocks that are obsolete.
+//
+// 2.1 Correctness
+//
+// Due to the level invariant, the sequence of seqnums for a user key in a
+// sstable represents a contiguous subsequence of the seqnums for the userkey
+// across the whole LSM, and is more recent than the seqnums in a sstable in a
+// lower level. So exposing exactly one point from a sstable for a userkey
+// will also mask the points for the userkey in lower levels. If we expose no
+// point, because of RANGEDELs, that RANGEDEL will also mask the points in
+// lower levels.
+//
+// Note that we do not need to do anything special at write time for
+// SETWITHDEL and SINGLEDEL. This is because these key kinds are treated
+// specially only by compactions, which do not hide obsolete points. For
+// regular reads, SETWITHDEL behaves the same as SET and SINGLEDEL behaves the
+// same as DEL.
+//
+// 2.2 Strictness and MERGE
+//
+// Setting the obsolete bit on point keys is advanced usage, so we support two
+// modes, both of which must be truthful when setting the obsolete bit, but
+// vary in when they don't set the obsolete bit.
+//
+// - Non-strict: In this mode, the bit does not need to be set for keys that
+//   are obsolete. Additionally, any sstable containing MERGE keys can only
+//   use this mode. An iterator over such an sstable, when configured to
+//   hideObsoletePoints, can expose multiple internal keys per user key, and
+//   can expose keys that are deleted by rangedels in the same sstable. This
+//   is the mode that non-advanced users should use. Pebble without
+//   disaggregated storage will also use this mode and will best-effort set
+//   the obsolete bit, to optimize iteration when snapshots have retained many
+//   obsolete keys.
+//
+// - Strict: In this mode, every obsolete key must have the obsolete bit set,
+//   and no MERGE keys are permitted. An iterator over such an sstable, when
+//   configured to hideObsoletePoints satisfies two properties:
+//   - S1: will expose at most one internal key per user key, which is the
+//     most recent one.
+//   - S2: will never expose keys that are deleted by rangedels in the same
+//     sstable.
+//
+//   This is the mode for two use cases in disaggregated storage (which will
+//   exclude parts of the key space that has MERGEs), for levels that contain
+//   sstables that can become foreign sstables:
+//   - Pebble compaction output to these levels that can become foreign
+//     sstables.
+//
+//   - CockroachDB ingest operations that can ingest into the levels that can
+//     become foreign sstables. Note, these are not sstables corresponding to
+//     copied data for CockroachDB range snapshots. This case occurs for
+//     operations like index backfills: these trivially satisfy the strictness
+//     criteria since they only write one key per userkey.
+//
+//     TODO(sumeer): this latter case is not currently supported, since only
+//     Writer.AddWithForceObsolete calls are permitted for writing strict
+//     obsolete sstables. This is done to reduce the likelihood of bugs. One
+//     simple way to lift this limitation would be to disallow adding any
+//     RANGEDELs when a Pebble-external writer is trying to construct a strict
+//     obsolete sstable.
+
+// ParseTableFormat parses the given magic bytes and version into its
+// corresponding internal TableFormat.
+func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) {
+	switch string(magic) {
+	case levelDBMagic:
+		return TableFormatLevelDB, nil
+	case rocksDBMagic:
+		if version != rocksDBFormatVersion2 {
+			return TableFormatUnspecified, base.CorruptionErrorf(
+				"pebble/table: unsupported rocksdb format version %d", errors.Safe(version),
+			)
+		}
+		return TableFormatRocksDBv2, nil
+	case pebbleDBMagic:
+		switch version {
+		case 1:
+			return TableFormatPebblev1, nil
+		case 2:
+			return TableFormatPebblev2, nil
+		case 3:
+			return TableFormatPebblev3, nil
+		case 4:
+			return TableFormatPebblev4, nil
+		default:
+			return TableFormatUnspecified, base.CorruptionErrorf(
+				"pebble/table: unsupported pebble format version %d", errors.Safe(version),
+			)
+		}
+	default:
+		return TableFormatUnspecified, base.CorruptionErrorf(
+			"pebble/table: invalid table (bad magic number: 0x%x)", magic,
+		)
+	}
+}
+
+// AsTuple returns the TableFormat's (Magic String, Version) tuple.
+func (f TableFormat) AsTuple() (string, uint32) {
+	switch f {
+	case TableFormatLevelDB:
+		return levelDBMagic, 0
+	case TableFormatRocksDBv2:
+		return rocksDBMagic, 2
+	case TableFormatPebblev1:
+		return pebbleDBMagic, 1
+	case TableFormatPebblev2:
+		return pebbleDBMagic, 2
+	case TableFormatPebblev3:
+		return pebbleDBMagic, 3
+	case TableFormatPebblev4:
+		return pebbleDBMagic, 4
+	default:
+		panic("sstable: unknown table format version tuple")
+	}
+}
+
+// String returns the TableFormat (Magic String,Version) tuple.
+func (f TableFormat) String() string {
+	switch f {
+	case TableFormatLevelDB:
+		return "(LevelDB)"
+	case TableFormatRocksDBv2:
+		return "(RocksDB,v2)"
+	case TableFormatPebblev1:
+		return "(Pebble,v1)"
+	case TableFormatPebblev2:
+		return "(Pebble,v2)"
+	case TableFormatPebblev3:
+		return "(Pebble,v3)"
+	case TableFormatPebblev4:
+		return "(Pebble,v4)"
+	default:
+		panic("sstable: unknown table format version tuple")
+	}
+}

vendor/github.com/cockroachdb/pebble/sstable/internal.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// InternalKeyKind exports the base.InternalKeyKind type.
+type InternalKeyKind = base.InternalKeyKind
+
+// SeekGEFlags exports base.SeekGEFlags.
+type SeekGEFlags = base.SeekGEFlags
+
+// SeekLTFlags exports base.SeekLTFlags.
+type SeekLTFlags = base.SeekLTFlags
+
+// These constants are part of the file format, and should not be changed.
+const (
+	InternalKeyKindDelete          = base.InternalKeyKindDelete
+	InternalKeyKindSet             = base.InternalKeyKindSet
+	InternalKeyKindMerge           = base.InternalKeyKindMerge
+	InternalKeyKindLogData         = base.InternalKeyKindLogData
+	InternalKeyKindSingleDelete    = base.InternalKeyKindSingleDelete
+	InternalKeyKindRangeDelete     = base.InternalKeyKindRangeDelete
+	InternalKeyKindSetWithDelete   = base.InternalKeyKindSetWithDelete
+	InternalKeyKindDeleteSized     = base.InternalKeyKindDeleteSized
+	InternalKeyKindMax             = base.InternalKeyKindMax
+	InternalKeyKindInvalid         = base.InternalKeyKindInvalid
+	InternalKeySeqNumBatch         = base.InternalKeySeqNumBatch
+	InternalKeySeqNumMax           = base.InternalKeySeqNumMax
+	InternalKeyRangeDeleteSentinel = base.InternalKeyRangeDeleteSentinel
+)
+
+// InternalKey exports the base.InternalKey type.
+type InternalKey = base.InternalKey

vendor/github.com/cockroachdb/pebble/sstable/layout.go

@@ -0,0 +1,306 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"sort"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+// Layout describes the block organization of an sstable.
+type Layout struct {
+	// NOTE: changes to fields in this struct should also be reflected in
+	// ValidateBlockChecksums, which validates a static list of BlockHandles
+	// referenced in this struct.
+
+	Data       []BlockHandleWithProperties
+	Index      []BlockHandle
+	TopIndex   BlockHandle
+	Filter     BlockHandle
+	RangeDel   BlockHandle
+	RangeKey   BlockHandle
+	ValueBlock []BlockHandle
+	ValueIndex BlockHandle
+	Properties BlockHandle
+	MetaIndex  BlockHandle
+	Footer     BlockHandle
+	Format     TableFormat
+}
+
+// Describe returns a description of the layout. If the verbose parameter is
+// true, details of the structure of each block are returned as well.
+func (l *Layout) Describe(
+	w io.Writer, verbose bool, r *Reader, fmtRecord func(key *base.InternalKey, value []byte),
+) {
+	ctx := context.TODO()
+	type block struct {
+		BlockHandle
+		name string
+	}
+	var blocks []block
+
+	for i := range l.Data {
+		blocks = append(blocks, block{l.Data[i].BlockHandle, "data"})
+	}
+	for i := range l.Index {
+		blocks = append(blocks, block{l.Index[i], "index"})
+	}
+	if l.TopIndex.Length != 0 {
+		blocks = append(blocks, block{l.TopIndex, "top-index"})
+	}
+	if l.Filter.Length != 0 {
+		blocks = append(blocks, block{l.Filter, "filter"})
+	}
+	if l.RangeDel.Length != 0 {
+		blocks = append(blocks, block{l.RangeDel, "range-del"})
+	}
+	if l.RangeKey.Length != 0 {
+		blocks = append(blocks, block{l.RangeKey, "range-key"})
+	}
+	for i := range l.ValueBlock {
+		blocks = append(blocks, block{l.ValueBlock[i], "value-block"})
+	}
+	if l.ValueIndex.Length != 0 {
+		blocks = append(blocks, block{l.ValueIndex, "value-index"})
+	}
+	if l.Properties.Length != 0 {
+		blocks = append(blocks, block{l.Properties, "properties"})
+	}
+	if l.MetaIndex.Length != 0 {
+		blocks = append(blocks, block{l.MetaIndex, "meta-index"})
+	}
+	if l.Footer.Length != 0 {
+		if l.Footer.Length == levelDBFooterLen {
+			blocks = append(blocks, block{l.Footer, "leveldb-footer"})
+		} else {
+			blocks = append(blocks, block{l.Footer, "footer"})
+		}
+	}
+
+	sort.Slice(blocks, func(i, j int) bool {
+		return blocks[i].Offset < blocks[j].Offset
+	})
+
+	for i := range blocks {
+		b := &blocks[i]
+		fmt.Fprintf(w, "%10d  %s (%d)\n", b.Offset, b.name, b.Length)
+
+		if !verbose {
+			continue
+		}
+		if b.name == "filter" {
+			continue
+		}
+
+		if b.name == "footer" || b.name == "leveldb-footer" {
+			trailer, offset := make([]byte, b.Length), b.Offset
+			_ = r.readable.ReadAt(ctx, trailer, int64(offset))
+
+			if b.name == "footer" {
+				checksumType := ChecksumType(trailer[0])
+				fmt.Fprintf(w, "%10d    checksum type: %s\n", offset, checksumType)
+				trailer, offset = trailer[1:], offset+1
+			}
+
+			metaHandle, n := binary.Uvarint(trailer)
+			metaLen, m := binary.Uvarint(trailer[n:])
+			fmt.Fprintf(w, "%10d    meta: offset=%d, length=%d\n", offset, metaHandle, metaLen)
+			trailer, offset = trailer[n+m:], offset+uint64(n+m)
+
+			indexHandle, n := binary.Uvarint(trailer)
+			indexLen, m := binary.Uvarint(trailer[n:])
+			fmt.Fprintf(w, "%10d    index: offset=%d, length=%d\n", offset, indexHandle, indexLen)
+			trailer, offset = trailer[n+m:], offset+uint64(n+m)
+
+			fmt.Fprintf(w, "%10d    [padding]\n", offset)
+
+			trailing := 12
+			if b.name == "leveldb-footer" {
+				trailing = 8
+			}
+
+			offset += uint64(len(trailer) - trailing)
+			trailer = trailer[len(trailer)-trailing:]
+
+			if b.name == "footer" {
+				version := trailer[:4]
+				fmt.Fprintf(w, "%10d    version: %d\n", offset, binary.LittleEndian.Uint32(version))
+				trailer, offset = trailer[4:], offset+4
+			}
+
+			magicNumber := trailer
+			fmt.Fprintf(w, "%10d    magic number: 0x%x\n", offset, magicNumber)
+
+			continue
+		}
+
+		h, err := r.readBlock(
+			context.Background(), b.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */, nil /* buffer pool */)
+		if err != nil {
+			fmt.Fprintf(w, "  [err: %s]\n", err)
+			continue
+		}
+
+		getRestart := func(data []byte, restarts, i int32) int32 {
+			return decodeRestart(data[restarts+4*i:])
+		}
+
+		formatIsRestart := func(data []byte, restarts, numRestarts, offset int32) {
+			i := sort.Search(int(numRestarts), func(i int) bool {
+				return getRestart(data, restarts, int32(i)) >= offset
+			})
+			if i < int(numRestarts) && getRestart(data, restarts, int32(i)) == offset {
+				fmt.Fprintf(w, " [restart]\n")
+			} else {
+				fmt.Fprintf(w, "\n")
+			}
+		}
+
+		formatRestarts := func(data []byte, restarts, numRestarts int32) {
+			for i := int32(0); i < numRestarts; i++ {
+				offset := getRestart(data, restarts, i)
+				fmt.Fprintf(w, "%10d    [restart %d]\n",
+					b.Offset+uint64(restarts+4*i), b.Offset+uint64(offset))
+			}
+		}
+
+		formatTrailer := func() {
+			trailer := make([]byte, blockTrailerLen)
+			offset := int64(b.Offset + b.Length)
+			_ = r.readable.ReadAt(ctx, trailer, offset)
+			bt := blockType(trailer[0])
+			checksum := binary.LittleEndian.Uint32(trailer[1:])
+			fmt.Fprintf(w, "%10d    [trailer compression=%s checksum=0x%04x]\n", offset, bt, checksum)
+		}
+
+		var lastKey InternalKey
+		switch b.name {
+		case "data", "range-del", "range-key":
+			iter, _ := newBlockIter(r.Compare, h.Get())
+			for key, value := iter.First(); key != nil; key, value = iter.Next() {
+				ptr := unsafe.Pointer(uintptr(iter.ptr) + uintptr(iter.offset))
+				shared, ptr := decodeVarint(ptr)
+				unshared, ptr := decodeVarint(ptr)
+				value2, _ := decodeVarint(ptr)
+
+				total := iter.nextOffset - iter.offset
+				// The format of the numbers in the record line is:
+				//
+				//   (<total> = <length> [<shared>] + <unshared> + <value>)
+				//
+				// <total>    is the total number of bytes for the record.
+				// <length>   is the size of the 3 varint encoded integers for <shared>,
+				//            <unshared>, and <value>.
+				// <shared>   is the number of key bytes shared with the previous key.
+				// <unshared> is the number of unshared key bytes.
+				// <value>    is the number of value bytes.
+				fmt.Fprintf(w, "%10d    record (%d = %d [%d] + %d + %d)",
+					b.Offset+uint64(iter.offset), total,
+					total-int32(unshared+value2), shared, unshared, value2)
+				formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
+				if fmtRecord != nil {
+					fmt.Fprintf(w, "              ")
+					if l.Format < TableFormatPebblev3 {
+						fmtRecord(key, value.InPlaceValue())
+					} else {
+						// InPlaceValue() will succeed even for data blocks where the
+						// actual value is in a different location, since this value was
+						// fetched from a blockIter which does not know about value
+						// blocks.
+						v := value.InPlaceValue()
+						if base.TrailerKind(key.Trailer) != InternalKeyKindSet {
+							fmtRecord(key, v)
+						} else if !isValueHandle(valuePrefix(v[0])) {
+							fmtRecord(key, v[1:])
+						} else {
+							vh := decodeValueHandle(v[1:])
+							fmtRecord(key, []byte(fmt.Sprintf("value handle %+v", vh)))
+						}
+					}
+				}
+
+				if base.InternalCompare(r.Compare, lastKey, *key) >= 0 {
+					fmt.Fprintf(w, "              WARNING: OUT OF ORDER KEYS!\n")
+				}
+				lastKey.Trailer = key.Trailer
+				lastKey.UserKey = append(lastKey.UserKey[:0], key.UserKey...)
+			}
+			formatRestarts(iter.data, iter.restarts, iter.numRestarts)
+			formatTrailer()
+		case "index", "top-index":
+			iter, _ := newBlockIter(r.Compare, h.Get())
+			for key, value := iter.First(); key != nil; key, value = iter.Next() {
+				bh, err := decodeBlockHandleWithProperties(value.InPlaceValue())
+				if err != nil {
+					fmt.Fprintf(w, "%10d    [err: %s]\n", b.Offset+uint64(iter.offset), err)
+					continue
+				}
+				fmt.Fprintf(w, "%10d    block:%d/%d",
+					b.Offset+uint64(iter.offset), bh.Offset, bh.Length)
+				formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
+			}
+			formatRestarts(iter.data, iter.restarts, iter.numRestarts)
+			formatTrailer()
+		case "properties":
+			iter, _ := newRawBlockIter(r.Compare, h.Get())
+			for valid := iter.First(); valid; valid = iter.Next() {
+				fmt.Fprintf(w, "%10d    %s (%d)",
+					b.Offset+uint64(iter.offset), iter.Key().UserKey, iter.nextOffset-iter.offset)
+				formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
+			}
+			formatRestarts(iter.data, iter.restarts, iter.numRestarts)
+			formatTrailer()
+		case "meta-index":
+			iter, _ := newRawBlockIter(r.Compare, h.Get())
+			for valid := iter.First(); valid; valid = iter.Next() {
+				value := iter.Value()
+				var bh BlockHandle
+				var n int
+				var vbih valueBlocksIndexHandle
+				isValueBlocksIndexHandle := false
+				if bytes.Equal(iter.Key().UserKey, []byte(metaValueIndexName)) {
+					vbih, n, err = decodeValueBlocksIndexHandle(value)
+					bh = vbih.h
+					isValueBlocksIndexHandle = true
+				} else {
+					bh, n = decodeBlockHandle(value)
+				}
+				if n == 0 || n != len(value) {
+					fmt.Fprintf(w, "%10d    [err: %s]\n", b.Offset+uint64(iter.offset), err)
+					continue
+				}
+				var vbihStr string
+				if isValueBlocksIndexHandle {
+					vbihStr = fmt.Sprintf(" value-blocks-index-lengths: %d(num), %d(offset), %d(length)",
+						vbih.blockNumByteLength, vbih.blockOffsetByteLength, vbih.blockLengthByteLength)
+				}
+				fmt.Fprintf(w, "%10d    %s block:%d/%d%s",
+					b.Offset+uint64(iter.offset), iter.Key().UserKey,
+					bh.Offset, bh.Length, vbihStr)
+				formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset)
+			}
+			formatRestarts(iter.data, iter.restarts, iter.numRestarts)
+			formatTrailer()
+		case "value-block":
+			// We don't peer into the value-block since it can't be interpreted
+			// without the valueHandles.
+		case "value-index":
+			// We have already read the value-index to construct the list of
+			// value-blocks, so no need to do it again.
+		}
+
+		h.Release()
+	}
+
+	last := blocks[len(blocks)-1]
+	fmt.Fprintf(w, "%10d  EOF\n", last.Offset+last.Length)
+}

vendor/github.com/cockroachdb/pebble/sstable/options.go

@@ -0,0 +1,311 @@
+// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"github.com/cockroachdb/fifo"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/cache"
+)
+
+// Compression is the per-block compression algorithm to use.
+type Compression int
+
+// The available compression types.
+const (
+	DefaultCompression Compression = iota
+	NoCompression
+	SnappyCompression
+	ZstdCompression
+	NCompression
+)
+
+var ignoredInternalProperties = map[string]struct{}{
+	"rocksdb.column.family.id":             {},
+	"rocksdb.fixed.key.length":             {},
+	"rocksdb.index.key.is.user.key":        {},
+	"rocksdb.index.value.is.delta.encoded": {},
+	"rocksdb.oldest.key.time":              {},
+	"rocksdb.creation.time":                {},
+	"rocksdb.file.creation.time":           {},
+	"rocksdb.format.version":               {},
+}
+
+func (c Compression) String() string {
+	switch c {
+	case DefaultCompression:
+		return "Default"
+	case NoCompression:
+		return "NoCompression"
+	case SnappyCompression:
+		return "Snappy"
+	case ZstdCompression:
+		return "ZSTD"
+	default:
+		return "Unknown"
+	}
+}
+
+// FilterType exports the base.FilterType type.
+type FilterType = base.FilterType
+
+// Exported TableFilter constants.
+const (
+	TableFilter = base.TableFilter
+)
+
+// FilterWriter exports the base.FilterWriter type.
+type FilterWriter = base.FilterWriter
+
+// FilterPolicy exports the base.FilterPolicy type.
+type FilterPolicy = base.FilterPolicy
+
+// TablePropertyCollector provides a hook for collecting user-defined
+// properties based on the keys and values stored in an sstable. A new
+// TablePropertyCollector is created for an sstable when the sstable is being
+// written.
+type TablePropertyCollector interface {
+	// Add is called with each new entry added to the sstable. While the sstable
+	// is itself sorted by key, do not assume that the entries are added in any
+	// order. In particular, the ordering of point entries and range tombstones
+	// is unspecified.
+	Add(key InternalKey, value []byte) error
+
+	// Finish is called when all entries have been added to the sstable. The
+	// collected properties (if any) should be added to the specified map. Note
+	// that in case of an error during sstable construction, Finish may not be
+	// called.
+	Finish(userProps map[string]string) error
+
+	// The name of the property collector.
+	Name() string
+}
+
+// SuffixReplaceableTableCollector is an extension to the TablePropertyCollector
+// interface that allows a table property collector to indicate that it supports
+// being *updated* during suffix replacement, i.e. when an existing SST in which
+// all keys have the same key suffix is updated to have a new suffix.
+//
+// A collector which supports being updated in such cases must be able to derive
+// its updated value from its old value and the change being made to the suffix,
+// without needing to be passed each updated K/V.
+//
+// For example, a collector that only inspects values can simply copy its
+// previously computed property as-is, since key-suffix replacement does not
+// change values, while a collector that depends only on key suffixes, like one
+// which collected mvcc-timestamp bounds from timestamp-suffixed keys, can just
+// set its new bounds from the new suffix, as it is common to all keys, without
+// needing to recompute it from every key.
+type SuffixReplaceableTableCollector interface {
+	// UpdateKeySuffixes is called when a table is updated to change the suffix of
+	// all keys in the table, and is passed the old value for that prop, if any,
+	// for that table as well as the old and new suffix.
+	UpdateKeySuffixes(oldProps map[string]string, oldSuffix, newSuffix []byte) error
+}
+
+// ReaderOptions holds the parameters needed for reading an sstable.
+type ReaderOptions struct {
+	// Cache is used to cache uncompressed blocks from sstables.
+	//
+	// The default cache size is a zero-size cache.
+	Cache *cache.Cache
+
+	// LoadBlockSema, if set, is used to limit the number of blocks that can be
+	// loaded (i.e. read from the filesystem) in parallel. Each load acquires one
+	// unit from the semaphore for the duration of the read.
+	LoadBlockSema *fifo.Semaphore
+
+	// User properties specified in this map will not be added to sst.Properties.UserProperties.
+	DeniedUserProperties map[string]struct{}
+
+	// Comparer defines a total ordering over the space of []byte keys: a 'less
+	// than' relationship. The same comparison algorithm must be used for reads
+	// and writes over the lifetime of the DB.
+	//
+	// The default value uses the same ordering as bytes.Compare.
+	Comparer *Comparer
+
+	// Merge defines the Merge function in use for this keyspace.
+	Merge base.Merge
+
+	// Filters is a map from filter policy name to filter policy. It is used for
+	// debugging tools which may be used on multiple databases configured with
+	// different filter policies. It is not necessary to populate this filters
+	// map during normal usage of a DB.
+	Filters map[string]FilterPolicy
+
+	// Merger defines the associative merge operation to use for merging values
+	// written with {Batch,DB}.Merge. The MergerName is checked for consistency
+	// with the value stored in the sstable when it was written.
+	MergerName string
+
+	// Logger is an optional logger and tracer.
+	LoggerAndTracer base.LoggerAndTracer
+}
+
+func (o ReaderOptions) ensureDefaults() ReaderOptions {
+	if o.Comparer == nil {
+		o.Comparer = base.DefaultComparer
+	}
+	if o.Merge == nil {
+		o.Merge = base.DefaultMerger.Merge
+	}
+	if o.MergerName == "" {
+		o.MergerName = base.DefaultMerger.Name
+	}
+	if o.LoggerAndTracer == nil {
+		o.LoggerAndTracer = base.NoopLoggerAndTracer{}
+	}
+	if o.DeniedUserProperties == nil {
+		o.DeniedUserProperties = ignoredInternalProperties
+	}
+	return o
+}
+
+// WriterOptions holds the parameters used to control building an sstable.
+type WriterOptions struct {
+	// BlockRestartInterval is the number of keys between restart points
+	// for delta encoding of keys.
+	//
+	// The default value is 16.
+	BlockRestartInterval int
+
+	// BlockSize is the target uncompressed size in bytes of each table block.
+	//
+	// The default value is 4096.
+	BlockSize int
+
+	// BlockSizeThreshold finishes a block if the block size is larger than the
+	// specified percentage of the target block size and adding the next entry
+	// would cause the block to be larger than the target block size.
+	//
+	// The default value is 90
+	BlockSizeThreshold int
+
+	// Cache is used to cache uncompressed blocks from sstables.
+	//
+	// The default is a nil cache.
+	Cache *cache.Cache
+
+	// Comparer defines a total ordering over the space of []byte keys: a 'less
+	// than' relationship. The same comparison algorithm must be used for reads
+	// and writes over the lifetime of the DB.
+	//
+	// The default value uses the same ordering as bytes.Compare.
+	Comparer *Comparer
+
+	// Compression defines the per-block compression to use.
+	//
+	// The default value (DefaultCompression) uses snappy compression.
+	Compression Compression
+
+	// FilterPolicy defines a filter algorithm (such as a Bloom filter) that can
+	// reduce disk reads for Get calls.
+	//
+	// One such implementation is bloom.FilterPolicy(10) from the pebble/bloom
+	// package.
+	//
+	// The default value means to use no filter.
+	FilterPolicy FilterPolicy
+
+	// FilterType defines whether an existing filter policy is applied at a
+	// block-level or table-level. Block-level filters use less memory to create,
+	// but are slower to access as a check for the key in the index must first be
+	// performed to locate the filter block. A table-level filter will require
+	// memory proportional to the number of keys in an sstable to create, but
+	// avoids the index lookup when determining if a key is present. Table-level
+	// filters should be preferred except under constrained memory situations.
+	FilterType FilterType
+
+	// IndexBlockSize is the target uncompressed size in bytes of each index
+	// block. When the index block size is larger than this target, two-level
+	// indexes are automatically enabled. Setting this option to a large value
+	// (such as math.MaxInt32) disables the automatic creation of two-level
+	// indexes.
+	//
+	// The default value is the value of BlockSize.
+	IndexBlockSize int
+
+	// Merger defines the associative merge operation to use for merging values
+	// written with {Batch,DB}.Merge. The MergerName is checked for consistency
+	// with the value stored in the sstable when it was written.
+	MergerName string
+
+	// TableFormat specifies the format version for writing sstables. The default
+	// is TableFormatRocksDBv2 which creates RocksDB compatible sstables. Use
+	// TableFormatLevelDB to create LevelDB compatible sstable which can be used
+	// by a wider range of tools and libraries.
+	TableFormat TableFormat
+
+	// IsStrictObsolete is only relevant for >= TableFormatPebblev4. See comment
+	// in format.go. Must be false if format < TableFormatPebblev4.
+	//
+	// TODO(bilal): set this when writing shared ssts.
+	IsStrictObsolete bool
+
+	// WritingToLowestLevel is only relevant for >= TableFormatPebblev4. It is
+	// used to set the obsolete bit on DEL/DELSIZED/SINGLEDEL if they are the
+	// youngest for a userkey.
+	WritingToLowestLevel bool
+
+	// TablePropertyCollectors is a list of TablePropertyCollector creation
+	// functions. A new TablePropertyCollector is created for each sstable built
+	// and lives for the lifetime of the table.
+	TablePropertyCollectors []func() TablePropertyCollector
+
+	// BlockPropertyCollectors is a list of BlockPropertyCollector creation
+	// functions. A new BlockPropertyCollector is created for each sstable
+	// built and lives for the lifetime of writing that table.
+	BlockPropertyCollectors []func() BlockPropertyCollector
+
+	// Checksum specifies which checksum to use.
+	Checksum ChecksumType
+
+	// Parallelism is used to indicate that the sstable Writer is allowed to
+	// compress data blocks and write datablocks to disk in parallel with the
+	// Writer client goroutine.
+	Parallelism bool
+
+	// ShortAttributeExtractor mirrors
+	// Options.Experimental.ShortAttributeExtractor.
+	ShortAttributeExtractor base.ShortAttributeExtractor
+
+	// RequiredInPlaceValueBound mirrors
+	// Options.Experimental.RequiredInPlaceValueBound.
+	RequiredInPlaceValueBound UserKeyPrefixBound
+}
+
+func (o WriterOptions) ensureDefaults() WriterOptions {
+	if o.BlockRestartInterval <= 0 {
+		o.BlockRestartInterval = base.DefaultBlockRestartInterval
+	}
+	if o.BlockSize <= 0 {
+		o.BlockSize = base.DefaultBlockSize
+	}
+	if o.BlockSizeThreshold <= 0 {
+		o.BlockSizeThreshold = base.DefaultBlockSizeThreshold
+	}
+	if o.Comparer == nil {
+		o.Comparer = base.DefaultComparer
+	}
+	if o.Compression <= DefaultCompression || o.Compression >= NCompression {
+		o.Compression = SnappyCompression
+	}
+	if o.IndexBlockSize <= 0 {
+		o.IndexBlockSize = o.BlockSize
+	}
+	if o.MergerName == "" {
+		o.MergerName = base.DefaultMerger.Name
+	}
+	if o.Checksum == ChecksumTypeNone {
+		o.Checksum = ChecksumTypeCRC32c
+	}
+	// By default, if the table format is not specified, fall back to using the
+	// most compatible format.
+	if o.TableFormat == TableFormatUnspecified {
+		o.TableFormat = TableFormatRocksDBv2
+	}
+	return o
+}

vendor/github.com/cockroachdb/pebble/sstable/properties.go

@@ -0,0 +1,450 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"reflect"
+	"sort"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/intern"
+)
+
+const propertiesBlockRestartInterval = math.MaxInt32
+const propGlobalSeqnumName = "rocksdb.external_sst_file.global_seqno"
+
+var propTagMap = make(map[string]reflect.StructField)
+var propBoolTrue = []byte{'1'}
+var propBoolFalse = []byte{'0'}
+
+var propOffsetTagMap = make(map[uintptr]string)
+
+func generateTagMaps(t reflect.Type, indexPrefix []int) {
+	for i := 0; i < t.NumField(); i++ {
+		f := t.Field(i)
+		if f.Type.Kind() == reflect.Struct {
+			if tag := f.Tag.Get("prop"); i == 0 && tag == "pebble.embbeded_common_properties" {
+				// CommonProperties struct embedded in Properties. Note that since
+				// CommonProperties is placed at the top of properties we can use
+				// the offsets of the fields within CommonProperties to determine
+				// the offsets of those fields within Properties.
+				generateTagMaps(f.Type, []int{i})
+				continue
+			}
+			panic("pebble: unknown struct type in Properties")
+		}
+		if tag := f.Tag.Get("prop"); tag != "" {
+			switch f.Type.Kind() {
+			case reflect.Bool:
+			case reflect.Uint32:
+			case reflect.Uint64:
+			case reflect.String:
+			default:
+				panic(fmt.Sprintf("unsupported property field type: %s %s", f.Name, f.Type))
+			}
+			if len(indexPrefix) > 0 {
+				// Prepend the index prefix so that we can use FieldByIndex on the top-level struct.
+				f.Index = append(indexPrefix[:len(indexPrefix):len(indexPrefix)], f.Index...)
+			}
+			propTagMap[tag] = f
+			propOffsetTagMap[f.Offset] = tag
+		}
+	}
+}
+
+func init() {
+	generateTagMaps(reflect.TypeOf(Properties{}), nil)
+}
+
+// CommonProperties holds properties for either a virtual or a physical sstable. This
+// can be used by code which doesn't care to make the distinction between physical
+// and virtual sstables properties.
+//
+// For virtual sstables, fields are constructed through extrapolation upon virtual
+// reader construction. See MakeVirtualReader for implementation details.
+//
+// NB: The values of these properties can affect correctness. For example,
+// if NumRangeKeySets == 0, but the sstable actually contains range keys, then
+// the iterators will behave incorrectly.
+type CommonProperties struct {
+	// The number of entries in this table.
+	NumEntries uint64 `prop:"rocksdb.num.entries"`
+	// Total raw key size.
+	RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
+	// Total raw value size.
+	RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
+	// Total raw key size of point deletion tombstones. This value is comparable
+	// to RawKeySize.
+	RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
+	// Sum of the raw value sizes carried by point deletion tombstones
+	// containing size estimates. See the DeleteSized key kind. This value is
+	// comparable to Raw{Key,Value}Size.
+	RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
+	// The number of point deletion entries ("tombstones") in this table that
+	// carry a size hint indicating the size of the value the tombstone deletes.
+	NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
+	// The number of deletion entries in this table, including both point and
+	// range deletions.
+	NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
+	// The number of range deletions in this table.
+	NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"`
+	// The number of RANGEKEYDELs in this table.
+	NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"`
+	// The number of RANGEKEYSETs in this table.
+	NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
+	// Total size of value blocks and value index block. Only serialized if > 0.
+	ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
+}
+
+// String is only used for testing purposes.
+func (c *CommonProperties) String() string {
+	var buf bytes.Buffer
+	v := reflect.ValueOf(*c)
+	loaded := make(map[uintptr]struct{})
+	writeProperties(loaded, v, &buf)
+	return buf.String()
+}
+
+// NumPointDeletions is the number of point deletions in the sstable. For virtual
+// sstables, this is an estimate.
+func (c *CommonProperties) NumPointDeletions() uint64 {
+	return c.NumDeletions - c.NumRangeDeletions
+}
+
+// Properties holds the sstable property values. The properties are
+// automatically populated during sstable creation and load from the properties
+// meta block when an sstable is opened.
+type Properties struct {
+	// CommonProperties needs to be at the top of the Properties struct so that the
+	// offsets of the fields in CommonProperties match the offsets of the embedded
+	// fields of CommonProperties in Properties.
+	CommonProperties `prop:"pebble.embbeded_common_properties"`
+
+	// The name of the comparer used in this table.
+	ComparerName string `prop:"rocksdb.comparator"`
+	// The compression algorithm used to compress blocks.
+	CompressionName string `prop:"rocksdb.compression"`
+	// The compression options used to compress blocks.
+	CompressionOptions string `prop:"rocksdb.compression_options"`
+	// The total size of all data blocks.
+	DataSize uint64 `prop:"rocksdb.data.size"`
+	// The external sstable version format. Version 2 is the one RocksDB has been
+	// using since 5.13. RocksDB only uses the global sequence number for an
+	// sstable if this property has been set.
+	ExternalFormatVersion uint32 `prop:"rocksdb.external_sst_file.version"`
+	// The name of the filter policy used in this table. Empty if no filter
+	// policy is used.
+	FilterPolicyName string `prop:"rocksdb.filter.policy"`
+	// The size of filter block.
+	FilterSize uint64 `prop:"rocksdb.filter.size"`
+	// The global sequence number to use for all entries in the table. Present if
+	// the table was created externally and ingested whole.
+	GlobalSeqNum uint64 `prop:"rocksdb.external_sst_file.global_seqno"`
+	// Total number of index partitions if kTwoLevelIndexSearch is used.
+	IndexPartitions uint64 `prop:"rocksdb.index.partitions"`
+	// The size of index block.
+	IndexSize uint64 `prop:"rocksdb.index.size"`
+	// The index type. TODO(peter): add a more detailed description.
+	IndexType uint32 `prop:"rocksdb.block.based.table.index.type"`
+	// For formats >= TableFormatPebblev4, this is set to true if the obsolete
+	// bit is strict for all the point keys.
+	IsStrictObsolete bool `prop:"pebble.obsolete.is_strict"`
+	// The name of the merger used in this table. Empty if no merger is used.
+	MergerName string `prop:"rocksdb.merge.operator"`
+	// The number of blocks in this table.
+	NumDataBlocks uint64 `prop:"rocksdb.num.data.blocks"`
+	// The number of merge operands in the table.
+	NumMergeOperands uint64 `prop:"rocksdb.merge.operands"`
+	// The number of RANGEKEYUNSETs in this table.
+	NumRangeKeyUnsets uint64 `prop:"pebble.num.range-key-unsets"`
+	// The number of value blocks in this table. Only serialized if > 0.
+	NumValueBlocks uint64 `prop:"pebble.num.value-blocks"`
+	// The number of values stored in value blocks. Only serialized if > 0.
+	NumValuesInValueBlocks uint64 `prop:"pebble.num.values.in.value-blocks"`
+	// The name of the prefix extractor used in this table. Empty if no prefix
+	// extractor is used.
+	PrefixExtractorName string `prop:"rocksdb.prefix.extractor.name"`
+	// If filtering is enabled, was the filter created on the key prefix.
+	PrefixFiltering bool `prop:"rocksdb.block.based.table.prefix.filtering"`
+	// A comma separated list of names of the property collectors used in this
+	// table.
+	PropertyCollectorNames string `prop:"rocksdb.property.collectors"`
+	// Total raw rangekey key size.
+	RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"`
+	// Total raw rangekey value size.
+	RawRangeKeyValueSize uint64 `prop:"pebble.raw.range-key.value.size"`
+	// The total number of keys in this table that were pinned by open snapshots.
+	SnapshotPinnedKeys uint64 `prop:"pebble.num.snapshot-pinned-keys"`
+	// The cumulative bytes of keys in this table that were pinned by
+	// open snapshots. This value is comparable to RawKeySize.
+	SnapshotPinnedKeySize uint64 `prop:"pebble.raw.snapshot-pinned-keys.size"`
+	// The cumulative bytes of values in this table that were pinned by
+	// open snapshots. This value is comparable to RawValueSize.
+	SnapshotPinnedValueSize uint64 `prop:"pebble.raw.snapshot-pinned-values.size"`
+	// Size of the top-level index if kTwoLevelIndexSearch is used.
+	TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"`
+	// User collected properties.
+	UserProperties map[string]string
+	// If filtering is enabled, was the filter created on the whole key.
+	WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"`
+
+	// Loaded set indicating which fields have been loaded from disk. Indexed by
+	// the field's byte offset within the struct
+	// (reflect.StructField.Offset). Only set if the properties have been loaded
+	// from a file. Only exported for testing purposes.
+	Loaded map[uintptr]struct{}
+}
+
+// NumPointDeletions returns the number of point deletions in this table.
+func (p *Properties) NumPointDeletions() uint64 {
+	return p.NumDeletions - p.NumRangeDeletions
+}
+
+// NumRangeKeys returns a count of the number of range keys in this table.
+func (p *Properties) NumRangeKeys() uint64 {
+	return p.NumRangeKeyDels + p.NumRangeKeySets + p.NumRangeKeyUnsets
+}
+
+func writeProperties(loaded map[uintptr]struct{}, v reflect.Value, buf *bytes.Buffer) {
+	vt := v.Type()
+	for i := 0; i < v.NumField(); i++ {
+		ft := vt.Field(i)
+		if ft.Type.Kind() == reflect.Struct {
+			// Embedded struct within the properties.
+			writeProperties(loaded, v.Field(i), buf)
+			continue
+		}
+		tag := ft.Tag.Get("prop")
+		if tag == "" {
+			continue
+		}
+
+		f := v.Field(i)
+		// TODO(peter): Use f.IsZero() when we can rely on go1.13.
+		if zero := reflect.Zero(f.Type()); zero.Interface() == f.Interface() {
+			// Skip printing of zero values which were not loaded from disk.
+			if _, ok := loaded[ft.Offset]; !ok {
+				continue
+			}
+		}
+
+		fmt.Fprintf(buf, "%s: ", tag)
+		switch ft.Type.Kind() {
+		case reflect.Bool:
+			fmt.Fprintf(buf, "%t\n", f.Bool())
+		case reflect.Uint32:
+			fmt.Fprintf(buf, "%d\n", f.Uint())
+		case reflect.Uint64:
+			fmt.Fprintf(buf, "%d\n", f.Uint())
+		case reflect.String:
+			fmt.Fprintf(buf, "%s\n", f.String())
+		default:
+			panic("not reached")
+		}
+	}
+}
+
+func (p *Properties) String() string {
+	var buf bytes.Buffer
+	v := reflect.ValueOf(*p)
+	writeProperties(p.Loaded, v, &buf)
+
+	// Write the UserProperties.
+	keys := make([]string, 0, len(p.UserProperties))
+	for key := range p.UserProperties {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		fmt.Fprintf(&buf, "%s: %s\n", key, p.UserProperties[key])
+	}
+	return buf.String()
+}
+
+func (p *Properties) load(
+	b block, blockOffset uint64, deniedUserProperties map[string]struct{},
+) error {
+	i, err := newRawBlockIter(bytes.Compare, b)
+	if err != nil {
+		return err
+	}
+	p.Loaded = make(map[uintptr]struct{})
+	v := reflect.ValueOf(p).Elem()
+
+	for valid := i.First(); valid; valid = i.Next() {
+		if f, ok := propTagMap[string(i.Key().UserKey)]; ok {
+			p.Loaded[f.Offset] = struct{}{}
+			field := v.FieldByIndex(f.Index)
+			switch f.Type.Kind() {
+			case reflect.Bool:
+				field.SetBool(bytes.Equal(i.Value(), propBoolTrue))
+			case reflect.Uint32:
+				field.SetUint(uint64(binary.LittleEndian.Uint32(i.Value())))
+			case reflect.Uint64:
+				var n uint64
+				if string(i.Key().UserKey) == propGlobalSeqnumName {
+					n = binary.LittleEndian.Uint64(i.Value())
+				} else {
+					n, _ = binary.Uvarint(i.Value())
+				}
+				field.SetUint(n)
+			case reflect.String:
+				field.SetString(intern.Bytes(i.Value()))
+			default:
+				panic("not reached")
+			}
+			continue
+		}
+		if p.UserProperties == nil {
+			p.UserProperties = make(map[string]string)
+		}
+
+		if _, denied := deniedUserProperties[string(i.Key().UserKey)]; !denied {
+			p.UserProperties[intern.Bytes(i.Key().UserKey)] = string(i.Value())
+		}
+	}
+	return nil
+}
+
+func (p *Properties) saveBool(m map[string][]byte, offset uintptr, value bool) {
+	tag := propOffsetTagMap[offset]
+	if value {
+		m[tag] = propBoolTrue
+	} else {
+		m[tag] = propBoolFalse
+	}
+}
+
+func (p *Properties) saveUint32(m map[string][]byte, offset uintptr, value uint32) {
+	var buf [4]byte
+	binary.LittleEndian.PutUint32(buf[:], value)
+	m[propOffsetTagMap[offset]] = buf[:]
+}
+
+func (p *Properties) saveUint64(m map[string][]byte, offset uintptr, value uint64) {
+	var buf [8]byte
+	binary.LittleEndian.PutUint64(buf[:], value)
+	m[propOffsetTagMap[offset]] = buf[:]
+}
+
+func (p *Properties) saveUvarint(m map[string][]byte, offset uintptr, value uint64) {
+	var buf [10]byte
+	n := binary.PutUvarint(buf[:], value)
+	m[propOffsetTagMap[offset]] = buf[:n]
+}
+
+func (p *Properties) saveString(m map[string][]byte, offset uintptr, value string) {
+	m[propOffsetTagMap[offset]] = []byte(value)
+}
+
+func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) {
+	m := make(map[string][]byte)
+	for k, v := range p.UserProperties {
+		m[k] = []byte(v)
+	}
+
+	if p.ComparerName != "" {
+		p.saveString(m, unsafe.Offsetof(p.ComparerName), p.ComparerName)
+	}
+	if p.CompressionName != "" {
+		p.saveString(m, unsafe.Offsetof(p.CompressionName), p.CompressionName)
+	}
+	if p.CompressionOptions != "" {
+		p.saveString(m, unsafe.Offsetof(p.CompressionOptions), p.CompressionOptions)
+	}
+	p.saveUvarint(m, unsafe.Offsetof(p.DataSize), p.DataSize)
+	if p.ExternalFormatVersion != 0 {
+		p.saveUint32(m, unsafe.Offsetof(p.ExternalFormatVersion), p.ExternalFormatVersion)
+		p.saveUint64(m, unsafe.Offsetof(p.GlobalSeqNum), p.GlobalSeqNum)
+	}
+	if p.FilterPolicyName != "" {
+		p.saveString(m, unsafe.Offsetof(p.FilterPolicyName), p.FilterPolicyName)
+	}
+	p.saveUvarint(m, unsafe.Offsetof(p.FilterSize), p.FilterSize)
+	if p.IndexPartitions != 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.IndexPartitions), p.IndexPartitions)
+		p.saveUvarint(m, unsafe.Offsetof(p.TopLevelIndexSize), p.TopLevelIndexSize)
+	}
+	p.saveUvarint(m, unsafe.Offsetof(p.IndexSize), p.IndexSize)
+	p.saveUint32(m, unsafe.Offsetof(p.IndexType), p.IndexType)
+	if p.IsStrictObsolete {
+		p.saveBool(m, unsafe.Offsetof(p.IsStrictObsolete), p.IsStrictObsolete)
+	}
+	if p.MergerName != "" {
+		p.saveString(m, unsafe.Offsetof(p.MergerName), p.MergerName)
+	}
+	p.saveUvarint(m, unsafe.Offsetof(p.NumDataBlocks), p.NumDataBlocks)
+	p.saveUvarint(m, unsafe.Offsetof(p.NumEntries), p.NumEntries)
+	p.saveUvarint(m, unsafe.Offsetof(p.NumDeletions), p.NumDeletions)
+	if p.NumSizedDeletions > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.NumSizedDeletions), p.NumSizedDeletions)
+	}
+	p.saveUvarint(m, unsafe.Offsetof(p.NumMergeOperands), p.NumMergeOperands)
+	p.saveUvarint(m, unsafe.Offsetof(p.NumRangeDeletions), p.NumRangeDeletions)
+	// NB: We only write out some properties for Pebble formats. This isn't
+	// strictly necessary because unrecognized properties are interpreted as
+	// user-defined properties, however writing them prevents byte-for-byte
+	// equivalence with RocksDB files that some of our testing requires.
+	if p.RawPointTombstoneKeySize > 0 && tblFormat >= TableFormatPebblev1 {
+		p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneKeySize), p.RawPointTombstoneKeySize)
+	}
+	if p.RawPointTombstoneValueSize > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneValueSize), p.RawPointTombstoneValueSize)
+	}
+	if p.NumRangeKeys() > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeyDels), p.NumRangeKeyDels)
+		p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeySets), p.NumRangeKeySets)
+		p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeyUnsets), p.NumRangeKeyUnsets)
+		p.saveUvarint(m, unsafe.Offsetof(p.RawRangeKeyKeySize), p.RawRangeKeyKeySize)
+		p.saveUvarint(m, unsafe.Offsetof(p.RawRangeKeyValueSize), p.RawRangeKeyValueSize)
+	}
+	if p.NumValueBlocks > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.NumValueBlocks), p.NumValueBlocks)
+	}
+	if p.NumValuesInValueBlocks > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.NumValuesInValueBlocks), p.NumValuesInValueBlocks)
+	}
+	if p.PrefixExtractorName != "" {
+		p.saveString(m, unsafe.Offsetof(p.PrefixExtractorName), p.PrefixExtractorName)
+	}
+	p.saveBool(m, unsafe.Offsetof(p.PrefixFiltering), p.PrefixFiltering)
+	if p.PropertyCollectorNames != "" {
+		p.saveString(m, unsafe.Offsetof(p.PropertyCollectorNames), p.PropertyCollectorNames)
+	}
+	if p.SnapshotPinnedKeys > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedKeys), p.SnapshotPinnedKeys)
+		p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedKeySize), p.SnapshotPinnedKeySize)
+		p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedValueSize), p.SnapshotPinnedValueSize)
+	}
+	p.saveUvarint(m, unsafe.Offsetof(p.RawKeySize), p.RawKeySize)
+	p.saveUvarint(m, unsafe.Offsetof(p.RawValueSize), p.RawValueSize)
+	if p.ValueBlocksSize > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.ValueBlocksSize), p.ValueBlocksSize)
+	}
+	p.saveBool(m, unsafe.Offsetof(p.WholeKeyFiltering), p.WholeKeyFiltering)
+
+	if tblFormat < TableFormatPebblev1 {
+		m["rocksdb.column.family.id"] = binary.AppendUvarint([]byte(nil), math.MaxInt32)
+		m["rocksdb.fixed.key.length"] = []byte{0x00}
+		m["rocksdb.index.key.is.user.key"] = []byte{0x00}
+		m["rocksdb.index.value.is.delta.encoded"] = []byte{0x00}
+		m["rocksdb.oldest.key.time"] = []byte{0x00}
+		m["rocksdb.creation.time"] = []byte{0x00}
+		m["rocksdb.format.version"] = []byte{0x00}
+	}
+
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		w.add(InternalKey{UserKey: []byte(key)}, m[key])
+	}
+}

vendor/github.com/cockroachdb/pebble/sstable/raw_block.go

@@ -0,0 +1,262 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"encoding/binary"
+	"sort"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+type rawBlockWriter struct {
+	blockWriter
+}
+
+func (w *rawBlockWriter) add(key InternalKey, value []byte) {
+	w.curKey, w.prevKey = w.prevKey, w.curKey
+
+	size := len(key.UserKey)
+	if cap(w.curKey) < size {
+		w.curKey = make([]byte, 0, size*2)
+	}
+	w.curKey = w.curKey[:size]
+	copy(w.curKey, key.UserKey)
+
+	w.storeWithOptionalValuePrefix(
+		size, value, len(key.UserKey), false, 0, false)
+}
+
+// rawBlockIter is an iterator over a single block of data. Unlike blockIter,
+// keys are stored in "raw" format (i.e. not as internal keys). Note that there
+// is significant similarity between this code and the code in blockIter. Yet
+// reducing duplication is difficult due to the blockIter being performance
+// critical. rawBlockIter must only be used for blocks where the value is
+// stored together with the key.
+type rawBlockIter struct {
+	cmp         Compare
+	offset      int32
+	nextOffset  int32
+	restarts    int32
+	numRestarts int32
+	ptr         unsafe.Pointer
+	data        []byte
+	key, val    []byte
+	ikey        InternalKey
+	cached      []blockEntry
+	cachedBuf   []byte
+}
+
+func newRawBlockIter(cmp Compare, block block) (*rawBlockIter, error) {
+	i := &rawBlockIter{}
+	return i, i.init(cmp, block)
+}
+
+func (i *rawBlockIter) init(cmp Compare, block block) error {
+	numRestarts := int32(binary.LittleEndian.Uint32(block[len(block)-4:]))
+	if numRestarts == 0 {
+		return base.CorruptionErrorf("pebble/table: invalid table (block has no restart points)")
+	}
+	i.cmp = cmp
+	i.restarts = int32(len(block)) - 4*(1+numRestarts)
+	i.numRestarts = numRestarts
+	i.ptr = unsafe.Pointer(&block[0])
+	i.data = block
+	if i.key == nil {
+		i.key = make([]byte, 0, 256)
+	} else {
+		i.key = i.key[:0]
+	}
+	i.val = nil
+	i.clearCache()
+	return nil
+}
+
+func (i *rawBlockIter) readEntry() {
+	ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(i.offset))
+	shared, ptr := decodeVarint(ptr)
+	unshared, ptr := decodeVarint(ptr)
+	value, ptr := decodeVarint(ptr)
+	i.key = append(i.key[:shared], getBytes(ptr, int(unshared))...)
+	i.key = i.key[:len(i.key):len(i.key)]
+	ptr = unsafe.Pointer(uintptr(ptr) + uintptr(unshared))
+	i.val = getBytes(ptr, int(value))
+	i.nextOffset = int32(uintptr(ptr)-uintptr(i.ptr)) + int32(value)
+}
+
+func (i *rawBlockIter) loadEntry() {
+	i.readEntry()
+	i.ikey.UserKey = i.key
+}
+
+func (i *rawBlockIter) clearCache() {
+	i.cached = i.cached[:0]
+	i.cachedBuf = i.cachedBuf[:0]
+}
+
+func (i *rawBlockIter) cacheEntry() {
+	var valStart int32
+	valSize := int32(len(i.val))
+	if valSize > 0 {
+		valStart = int32(uintptr(unsafe.Pointer(&i.val[0])) - uintptr(i.ptr))
+	}
+
+	i.cached = append(i.cached, blockEntry{
+		offset:   i.offset,
+		keyStart: int32(len(i.cachedBuf)),
+		keyEnd:   int32(len(i.cachedBuf) + len(i.key)),
+		valStart: valStart,
+		valSize:  valSize,
+	})
+	i.cachedBuf = append(i.cachedBuf, i.key...)
+}
+
+// SeekGE implements internalIterator.SeekGE, as documented in the pebble
+// package.
+func (i *rawBlockIter) SeekGE(key []byte) bool {
+	// Find the index of the smallest restart point whose key is > the key
+	// sought; index will be numRestarts if there is no such restart point.
+	i.offset = 0
+	index := sort.Search(int(i.numRestarts), func(j int) bool {
+		offset := int32(binary.LittleEndian.Uint32(i.data[int(i.restarts)+4*j:]))
+		// For a restart point, there are 0 bytes shared with the previous key.
+		// The varint encoding of 0 occupies 1 byte.
+		ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(offset+1))
+		// Decode the key at that restart point, and compare it to the key sought.
+		v1, ptr := decodeVarint(ptr)
+		_, ptr = decodeVarint(ptr)
+		s := getBytes(ptr, int(v1))
+		return i.cmp(key, s) < 0
+	})
+
+	// Since keys are strictly increasing, if index > 0 then the restart point at
+	// index-1 will be the largest whose key is <= the key sought.  If index ==
+	// 0, then all keys in this block are larger than the key sought, and offset
+	// remains at zero.
+	if index > 0 {
+		i.offset = int32(binary.LittleEndian.Uint32(i.data[int(i.restarts)+4*(index-1):]))
+	}
+	i.loadEntry()
+
+	// Iterate from that restart point to somewhere >= the key sought.
+	for valid := i.Valid(); valid; valid = i.Next() {
+		if i.cmp(key, i.key) <= 0 {
+			break
+		}
+	}
+	return i.Valid()
+}
+
+// First implements internalIterator.First, as documented in the pebble
+// package.
+func (i *rawBlockIter) First() bool {
+	i.offset = 0
+	i.loadEntry()
+	return i.Valid()
+}
+
+// Last implements internalIterator.Last, as documented in the pebble package.
+func (i *rawBlockIter) Last() bool {
+	// Seek forward from the last restart point.
+	i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(i.numRestarts-1):]))
+
+	i.readEntry()
+	i.clearCache()
+	i.cacheEntry()
+
+	for i.nextOffset < i.restarts {
+		i.offset = i.nextOffset
+		i.readEntry()
+		i.cacheEntry()
+	}
+
+	i.ikey.UserKey = i.key
+	return i.Valid()
+}
+
+// Next implements internalIterator.Next, as documented in the pebble
+// package.
+func (i *rawBlockIter) Next() bool {
+	i.offset = i.nextOffset
+	if !i.Valid() {
+		return false
+	}
+	i.loadEntry()
+	return true
+}
+
+// Prev implements internalIterator.Prev, as documented in the pebble
+// package.
+func (i *rawBlockIter) Prev() bool {
+	if n := len(i.cached) - 1; n > 0 && i.cached[n].offset == i.offset {
+		i.nextOffset = i.offset
+		e := &i.cached[n-1]
+		i.offset = e.offset
+		i.val = getBytes(unsafe.Pointer(uintptr(i.ptr)+uintptr(e.valStart)), int(e.valSize))
+		i.ikey.UserKey = i.cachedBuf[e.keyStart:e.keyEnd]
+		i.cached = i.cached[:n]
+		return true
+	}
+
+	if i.offset == 0 {
+		i.offset = -1
+		i.nextOffset = 0
+		return false
+	}
+
+	targetOffset := i.offset
+	index := sort.Search(int(i.numRestarts), func(j int) bool {
+		offset := int32(binary.LittleEndian.Uint32(i.data[int(i.restarts)+4*j:]))
+		return offset >= targetOffset
+	})
+	i.offset = 0
+	if index > 0 {
+		i.offset = int32(binary.LittleEndian.Uint32(i.data[int(i.restarts)+4*(index-1):]))
+	}
+
+	i.readEntry()
+	i.clearCache()
+	i.cacheEntry()
+
+	for i.nextOffset < targetOffset {
+		i.offset = i.nextOffset
+		i.readEntry()
+		i.cacheEntry()
+	}
+
+	i.ikey.UserKey = i.key
+	return true
+}
+
+// Key implements internalIterator.Key, as documented in the pebble package.
+func (i *rawBlockIter) Key() InternalKey {
+	return i.ikey
+}
+
+// Value implements internalIterator.Value, as documented in the pebble
+// package.
+func (i *rawBlockIter) Value() []byte {
+	return i.val
+}
+
+// Valid implements internalIterator.Valid, as documented in the pebble
+// package.
+func (i *rawBlockIter) Valid() bool {
+	return i.offset >= 0 && i.offset < i.restarts
+}
+
+// Error implements internalIterator.Error, as documented in the pebble
+// package.
+func (i *rawBlockIter) Error() error {
+	return nil
+}
+
+// Close implements internalIterator.Close, as documented in the pebble
+// package.
+func (i *rawBlockIter) Close() error {
+	i.val = nil
+	return nil
+}

vendor/github.com/cockroachdb/pebble/sstable/reader_iter.go

@@ -0,0 +1,291 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// Iterator iterates over an entire table of data.
+type Iterator interface {
+	base.InternalIterator
+
+	// NextPrefix implements (base.InternalIterator).NextPrefix.
+	NextPrefix(succKey []byte) (*InternalKey, base.LazyValue)
+
+	// MaybeFilteredKeys may be called when an iterator is exhausted to indicate
+	// whether or not the last positioning method may have skipped any keys due
+	// to block-property filters. This is used by the Pebble levelIter to
+	// control when an iterator steps to the next sstable.
+	//
+	// MaybeFilteredKeys may always return false positives, that is it may
+	// return true when no keys were filtered. It should only be called when the
+	// iterator is exhausted. It must never return false negatives when the
+	// iterator is exhausted.
+	MaybeFilteredKeys() bool
+
+	SetCloseHook(fn func(i Iterator) error)
+}
+
+// Iterator positioning optimizations and singleLevelIterator and
+// twoLevelIterator:
+//
+// An iterator is absolute positioned using one of the Seek or First or Last
+// calls. After absolute positioning, there can be relative positioning done
+// by stepping using Prev or Next.
+//
+// We implement optimizations below where an absolute positioning call can in
+// some cases use the current position to do less work. To understand these,
+// we first define some terms. An iterator is bounds-exhausted if the bounds
+// (upper of lower) have been reached. An iterator is data-exhausted if it has
+// the reached the end of the data (forward or reverse) in the sstable. A
+// singleLevelIterator only knows a local-data-exhausted property since when
+// it is used as part of a twoLevelIterator, the twoLevelIterator can step to
+// the next lower-level index block.
+//
+// The bounds-exhausted property is tracked by
+// singleLevelIterator.exhaustedBounds being +1 (upper bound reached) or -1
+// (lower bound reached). The same field is reused by twoLevelIterator. Either
+// may notice the exhaustion of the bound and set it. Note that if
+// singleLevelIterator sets this property, it is not a local property (since
+// the bound has been reached regardless of whether this is in the context of
+// the twoLevelIterator or not).
+//
+// The data-exhausted property is tracked in a more subtle manner. We define
+// two predicates:
+// - partial-local-data-exhausted (PLDE):
+//   i.data.isDataInvalidated() || !i.data.valid()
+// - partial-global-data-exhausted (PGDE):
+//   i.index.isDataInvalidated() || !i.index.valid() || i.data.isDataInvalidated() ||
+//   !i.data.valid()
+//
+// PLDE is defined for a singleLevelIterator. PGDE is defined for a
+// twoLevelIterator. Oddly, in our code below the singleLevelIterator does not
+// know when it is part of a twoLevelIterator so it does not know when its
+// property is local or global.
+//
+// Now to define data-exhausted:
+// - Prerequisite: we must know that the iterator has been positioned and
+//   i.err is nil.
+// - bounds-exhausted must not be true:
+//   If bounds-exhausted is true, we have incomplete knowledge of
+//   data-exhausted since PLDE or PGDE could be true because we could have
+//   chosen not to load index block or data block and figured out that the
+//   bound is exhausted (due to block property filters filtering out index and
+//   data blocks and going past the bound on the top level index block). Note
+//   that if we tried to separate out the BPF case from others we could
+//   develop more knowledge here.
+// - PGDE is true for twoLevelIterator. PLDE is true if it is a standalone
+//   singleLevelIterator. !PLDE or !PGDE of course imply that data-exhausted
+//   is not true.
+//
+// An implication of the above is that if we are going to somehow utilize
+// knowledge of data-exhausted in an optimization, we must not forget the
+// existing value of bounds-exhausted since by forgetting the latter we can
+// erroneously think that data-exhausted is true. Bug #2036 was due to this
+// forgetting.
+//
+// Now to the two categories of optimizations we currently have:
+// - Monotonic bounds optimization that reuse prior iterator position when
+//   doing seek: These only work with !data-exhausted. We could choose to make
+//   these work with data-exhausted but have not bothered because in the
+//   context of a DB if data-exhausted were true, the DB would move to the
+//   next file in the level. Note that this behavior of moving to the next
+//   file is not necessarily true for L0 files, so there could be some benefit
+//   in the future in this optimization. See the WARNING-data-exhausted
+//   comments if trying to optimize this in the future.
+// - TrySeekUsingNext optimizations: these work regardless of exhaustion
+//   state.
+//
+// Implementation detail: In the code PLDE only checks that
+// i.data.isDataInvalidated(). This narrower check is safe, since this is a
+// subset of the set expressed by the OR expression. Also, it is not a
+// de-optimization since whenever we exhaust the iterator we explicitly call
+// i.data.invalidate(). PGDE checks i.index.isDataInvalidated() &&
+// i.data.isDataInvalidated(). Again, this narrower check is safe, and not a
+// de-optimization since whenever we exhaust the iterator we explicitly call
+// i.index.invalidate() and i.data.invalidate(). The && is questionable -- for
+// now this is a bit of defensive code. We should seriously consider removing
+// it, since defensive code suggests we are not confident about our invariants
+// (and if we are not confident, we need more invariant assertions, not
+// defensive code).
+//
+// TODO(sumeer): remove the aforementioned defensive code.
+
+var singleLevelIterPool = sync.Pool{
+	New: func() interface{} {
+		i := &singleLevelIterator{}
+		// Note: this is a no-op if invariants are disabled or race is enabled.
+		invariants.SetFinalizer(i, checkSingleLevelIterator)
+		return i
+	},
+}
+
+var twoLevelIterPool = sync.Pool{
+	New: func() interface{} {
+		i := &twoLevelIterator{}
+		// Note: this is a no-op if invariants are disabled or race is enabled.
+		invariants.SetFinalizer(i, checkTwoLevelIterator)
+		return i
+	},
+}
+
+// TODO(jackson): rangedel fragmentBlockIters can't be pooled because of some
+// code paths that double Close the iters. Fix the double close and pool the
+// *fragmentBlockIter type directly.
+
+var rangeKeyFragmentBlockIterPool = sync.Pool{
+	New: func() interface{} {
+		i := &rangeKeyFragmentBlockIter{}
+		// Note: this is a no-op if invariants are disabled or race is enabled.
+		invariants.SetFinalizer(i, checkRangeKeyFragmentBlockIterator)
+		return i
+	},
+}
+
+func checkSingleLevelIterator(obj interface{}) {
+	i := obj.(*singleLevelIterator)
+	if p := i.data.handle.Get(); p != nil {
+		fmt.Fprintf(os.Stderr, "singleLevelIterator.data.handle is not nil: %p\n", p)
+		os.Exit(1)
+	}
+	if p := i.index.handle.Get(); p != nil {
+		fmt.Fprintf(os.Stderr, "singleLevelIterator.index.handle is not nil: %p\n", p)
+		os.Exit(1)
+	}
+}
+
+func checkTwoLevelIterator(obj interface{}) {
+	i := obj.(*twoLevelIterator)
+	if p := i.data.handle.Get(); p != nil {
+		fmt.Fprintf(os.Stderr, "singleLevelIterator.data.handle is not nil: %p\n", p)
+		os.Exit(1)
+	}
+	if p := i.index.handle.Get(); p != nil {
+		fmt.Fprintf(os.Stderr, "singleLevelIterator.index.handle is not nil: %p\n", p)
+		os.Exit(1)
+	}
+}
+
+func checkRangeKeyFragmentBlockIterator(obj interface{}) {
+	i := obj.(*rangeKeyFragmentBlockIter)
+	if p := i.blockIter.handle.Get(); p != nil {
+		fmt.Fprintf(os.Stderr, "fragmentBlockIter.blockIter.handle is not nil: %p\n", p)
+		os.Exit(1)
+	}
+}
+
+// compactionIterator is similar to Iterator but it increments the number of
+// bytes that have been iterated through.
+type compactionIterator struct {
+	*singleLevelIterator
+	bytesIterated *uint64
+	prevOffset    uint64
+}
+
+// compactionIterator implements the base.InternalIterator interface.
+var _ base.InternalIterator = (*compactionIterator)(nil)
+
+func (i *compactionIterator) String() string {
+	if i.vState != nil {
+		return i.vState.fileNum.String()
+	}
+	return i.reader.fileNum.String()
+}
+
+func (i *compactionIterator) SeekGE(
+	key []byte, flags base.SeekGEFlags,
+) (*InternalKey, base.LazyValue) {
+	panic("pebble: SeekGE unimplemented")
+}
+
+func (i *compactionIterator) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	panic("pebble: SeekPrefixGE unimplemented")
+}
+
+func (i *compactionIterator) SeekLT(
+	key []byte, flags base.SeekLTFlags,
+) (*InternalKey, base.LazyValue) {
+	panic("pebble: SeekLT unimplemented")
+}
+
+func (i *compactionIterator) First() (*InternalKey, base.LazyValue) {
+	i.err = nil // clear cached iteration error
+	return i.skipForward(i.singleLevelIterator.First())
+}
+
+func (i *compactionIterator) Last() (*InternalKey, base.LazyValue) {
+	panic("pebble: Last unimplemented")
+}
+
+// Note: compactionIterator.Next mirrors the implementation of Iterator.Next
+// due to performance. Keep the two in sync.
+func (i *compactionIterator) Next() (*InternalKey, base.LazyValue) {
+	if i.err != nil {
+		return nil, base.LazyValue{}
+	}
+	return i.skipForward(i.data.Next())
+}
+
+func (i *compactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
+	panic("pebble: NextPrefix unimplemented")
+}
+
+func (i *compactionIterator) Prev() (*InternalKey, base.LazyValue) {
+	panic("pebble: Prev unimplemented")
+}
+
+func (i *compactionIterator) skipForward(
+	key *InternalKey, val base.LazyValue,
+) (*InternalKey, base.LazyValue) {
+	if key == nil {
+		for {
+			if key, _ := i.index.Next(); key == nil {
+				break
+			}
+			result := i.loadBlock(+1)
+			if result != loadBlockOK {
+				if i.err != nil {
+					break
+				}
+				switch result {
+				case loadBlockFailed:
+					// We checked that i.index was at a valid entry, so
+					// loadBlockFailed could not have happened due to to i.index
+					// being exhausted, and must be due to an error.
+					panic("loadBlock should not have failed with no error")
+				case loadBlockIrrelevant:
+					panic("compactionIter should not be using block intervals for skipping")
+				default:
+					panic(fmt.Sprintf("unexpected case %d", result))
+				}
+			}
+			// result == loadBlockOK
+			if key, val = i.data.First(); key != nil {
+				break
+			}
+		}
+	}
+
+	curOffset := i.recordOffset()
+	*i.bytesIterated += uint64(curOffset - i.prevOffset)
+	i.prevOffset = curOffset
+
+	if i.vState != nil && key != nil {
+		cmp := i.cmp(key.UserKey, i.vState.upper.UserKey)
+		if cmp > 0 || (i.vState.upper.IsExclusiveSentinel() && cmp == 0) {
+			return nil, base.LazyValue{}
+		}
+	}
+
+	return key, val
+}

vendor/github.com/cockroachdb/pebble/sstable/reader_virtual.go

@@ -0,0 +1,206 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"context"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manifest"
+)
+
+// VirtualReader wraps Reader. Its purpose is to restrict functionality of the
+// Reader which should be inaccessible to virtual sstables, and enforce bounds
+// invariants associated with virtual sstables. All reads on virtual sstables
+// should go through a VirtualReader.
+//
+// INVARIANT: Any iterators created through a virtual reader will guarantee that
+// they don't expose keys outside the virtual sstable bounds.
+type VirtualReader struct {
+	vState     virtualState
+	reader     *Reader
+	Properties CommonProperties
+}
+
+// Lightweight virtual sstable state which can be passed to sstable iterators.
+type virtualState struct {
+	lower     InternalKey
+	upper     InternalKey
+	fileNum   base.FileNum
+	Compare   Compare
+	isForeign bool
+}
+
+func ceilDiv(a, b uint64) uint64 {
+	return (a + b - 1) / b
+}
+
+// MakeVirtualReader is used to contruct a reader which can read from virtual
+// sstables.
+func MakeVirtualReader(
+	reader *Reader, meta manifest.VirtualFileMeta, isForeign bool,
+) VirtualReader {
+	if reader.fileNum != meta.FileBacking.DiskFileNum {
+		panic("pebble: invalid call to MakeVirtualReader")
+	}
+
+	vState := virtualState{
+		lower:     meta.Smallest,
+		upper:     meta.Largest,
+		fileNum:   meta.FileNum,
+		Compare:   reader.Compare,
+		isForeign: isForeign,
+	}
+	v := VirtualReader{
+		vState: vState,
+		reader: reader,
+	}
+
+	v.Properties.RawKeySize = ceilDiv(reader.Properties.RawKeySize*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawValueSize = ceilDiv(reader.Properties.RawValueSize*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumEntries = ceilDiv(reader.Properties.NumEntries*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumDeletions = ceilDiv(reader.Properties.NumDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumRangeDeletions = ceilDiv(reader.Properties.NumRangeDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumRangeKeyDels = ceilDiv(reader.Properties.NumRangeKeyDels*meta.Size, meta.FileBacking.Size)
+
+	// Note that we rely on NumRangeKeySets for correctness. If the sstable may
+	// contain range keys, then NumRangeKeySets must be > 0. ceilDiv works because
+	// meta.Size will not be 0 for virtual sstables.
+	v.Properties.NumRangeKeySets = ceilDiv(reader.Properties.NumRangeKeySets*meta.Size, meta.FileBacking.Size)
+	v.Properties.ValueBlocksSize = ceilDiv(reader.Properties.ValueBlocksSize*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumSizedDeletions = ceilDiv(reader.Properties.NumSizedDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawPointTombstoneKeySize = ceilDiv(reader.Properties.RawPointTombstoneKeySize*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawPointTombstoneValueSize = ceilDiv(reader.Properties.RawPointTombstoneValueSize*meta.Size, meta.FileBacking.Size)
+	return v
+}
+
+// NewCompactionIter is the compaction iterator function for virtual readers.
+func (v *VirtualReader) NewCompactionIter(
+	bytesIterated *uint64, rp ReaderProvider, bufferPool *BufferPool,
+) (Iterator, error) {
+	return v.reader.newCompactionIter(bytesIterated, rp, &v.vState, bufferPool)
+}
+
+// NewIterWithBlockPropertyFiltersAndContextEtc wraps
+// Reader.NewIterWithBlockPropertyFiltersAndContext. We assume that the passed
+// in [lower, upper) bounds will have at least some overlap with the virtual
+// sstable bounds. No overlap is not currently supported in the iterator.
+func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContextEtc(
+	ctx context.Context,
+	lower, upper []byte,
+	filterer *BlockPropertiesFilterer,
+	hideObsoletePoints, useFilterBlock bool,
+	stats *base.InternalIteratorStats,
+	rp ReaderProvider,
+) (Iterator, error) {
+	return v.reader.newIterWithBlockPropertyFiltersAndContext(
+		ctx, lower, upper, filterer, hideObsoletePoints, useFilterBlock, stats, rp, &v.vState,
+	)
+}
+
+// ValidateBlockChecksumsOnBacking will call ValidateBlockChecksumsOnBacking on the underlying reader.
+// Note that block checksum validation is NOT restricted to virtual sstable bounds.
+func (v *VirtualReader) ValidateBlockChecksumsOnBacking() error {
+	return v.reader.ValidateBlockChecksums()
+}
+
+// NewRawRangeDelIter wraps Reader.NewRawRangeDelIter.
+func (v *VirtualReader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) {
+	iter, err := v.reader.NewRawRangeDelIter()
+	if err != nil {
+		return nil, err
+	}
+	if iter == nil {
+		return nil, nil
+	}
+
+	// Truncation of spans isn't allowed at a user key that also contains points
+	// in the same virtual sstable, as it would lead to covered points getting
+	// uncovered. Set panicOnUpperTruncate to true if the file's upper bound
+	// is not an exclusive sentinel.
+	//
+	// As an example, if an sstable contains a rangedel a-c and point keys at
+	// a.SET.2 and b.SET.3, the file bounds [a#2,SET-b#RANGEDELSENTINEL] are
+	// allowed (as they exclude b.SET.3), or [a#2,SET-c#RANGEDELSENTINEL] (as it
+	// includes both point keys), but not [a#2,SET-b#3,SET] (as it would truncate
+	// the rangedel at b and lead to the point being uncovered).
+	return keyspan.Truncate(
+		v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey,
+		&v.vState.lower, &v.vState.upper, !v.vState.upper.IsExclusiveSentinel(), /* panicOnUpperTruncate */
+	), nil
+}
+
+// NewRawRangeKeyIter wraps Reader.NewRawRangeKeyIter.
+func (v *VirtualReader) NewRawRangeKeyIter() (keyspan.FragmentIterator, error) {
+	iter, err := v.reader.NewRawRangeKeyIter()
+	if err != nil {
+		return nil, err
+	}
+	if iter == nil {
+		return nil, nil
+	}
+
+	// Truncation of spans isn't allowed at a user key that also contains points
+	// in the same virtual sstable, as it would lead to covered points getting
+	// uncovered. Set panicOnUpperTruncate to true if the file's upper bound
+	// is not an exclusive sentinel.
+	//
+	// As an example, if an sstable contains a range key a-c and point keys at
+	// a.SET.2 and b.SET.3, the file bounds [a#2,SET-b#RANGEKEYSENTINEL] are
+	// allowed (as they exclude b.SET.3), or [a#2,SET-c#RANGEKEYSENTINEL] (as it
+	// includes both point keys), but not [a#2,SET-b#3,SET] (as it would truncate
+	// the range key at b and lead to the point being uncovered).
+	return keyspan.Truncate(
+		v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey,
+		&v.vState.lower, &v.vState.upper, !v.vState.upper.IsExclusiveSentinel(), /* panicOnUpperTruncate */
+	), nil
+}
+
+// Constrain bounds will narrow the start, end bounds if they do not fit within
+// the virtual sstable. The function will return if the new end key is
+// inclusive.
+func (v *virtualState) constrainBounds(
+	start, end []byte, endInclusive bool,
+) (lastKeyInclusive bool, first []byte, last []byte) {
+	first = start
+	if start == nil || v.Compare(start, v.lower.UserKey) < 0 {
+		first = v.lower.UserKey
+	}
+
+	// Note that we assume that start, end has some overlap with the virtual
+	// sstable bounds.
+	last = v.upper.UserKey
+	lastKeyInclusive = !v.upper.IsExclusiveSentinel()
+	if end != nil {
+		cmp := v.Compare(end, v.upper.UserKey)
+		switch {
+		case cmp == 0:
+			lastKeyInclusive = !v.upper.IsExclusiveSentinel() && endInclusive
+			last = v.upper.UserKey
+		case cmp > 0:
+			lastKeyInclusive = !v.upper.IsExclusiveSentinel()
+			last = v.upper.UserKey
+		default:
+			lastKeyInclusive = endInclusive
+			last = end
+		}
+	}
+	// TODO(bananabrick): What if someone passes in bounds completely outside of
+	// virtual sstable bounds?
+	return lastKeyInclusive, first, last
+}
+
+// EstimateDiskUsage just calls VirtualReader.reader.EstimateDiskUsage after
+// enforcing the virtual sstable bounds.
+func (v *VirtualReader) EstimateDiskUsage(start, end []byte) (uint64, error) {
+	_, f, l := v.vState.constrainBounds(start, end, true /* endInclusive */)
+	return v.reader.EstimateDiskUsage(f, l)
+}
+
+// CommonProperties implements the CommonReader interface.
+func (v *VirtualReader) CommonProperties() *CommonProperties {
+	return &v.Properties
+}

vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go

@@ -0,0 +1,589 @@
+package sstable
+
+import (
+	"bytes"
+	"context"
+	"math"
+	"sync"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/bytealloc"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/rangekey"
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// RewriteKeySuffixes is deprecated.
+//
+// TODO(sumeer): remove after switching CockroachDB to RewriteKeySuffixesAndReturnFormat.
+func RewriteKeySuffixes(
+	sst []byte,
+	rOpts ReaderOptions,
+	out objstorage.Writable,
+	o WriterOptions,
+	from, to []byte,
+	concurrency int,
+) (*WriterMetadata, error) {
+	meta, _, err := RewriteKeySuffixesAndReturnFormat(sst, rOpts, out, o, from, to, concurrency)
+	return meta, err
+}
+
+// RewriteKeySuffixesAndReturnFormat copies the content of the passed SSTable
+// bytes to a new sstable, written to `out`, in which the suffix `from` has is
+// replaced with `to` in every key. The input sstable must consist of only
+// Sets or RangeKeySets and every key must have `from` as its suffix as
+// determined by the Split function of the Comparer in the passed
+// WriterOptions. Range deletes must not exist in this sstable, as they will
+// be ignored.
+//
+// Data blocks are rewritten in parallel by `concurrency` workers and then
+// assembled into a final SST. Filters are copied from the original SST without
+// modification as they are not affected by the suffix, while block and table
+// properties are only minimally recomputed.
+//
+// TODO(sumeer): document limitations, if any, due to this limited
+// re-computation of properties (is there any loss of fidelity?).
+//
+// Any block and table property collectors configured in the WriterOptions must
+// implement SuffixReplaceableTableCollector/SuffixReplaceableBlockCollector.
+//
+// The WriterOptions.TableFormat is ignored, and the output sstable has the
+// same TableFormat as the input, which is returned in case the caller wants
+// to do some error checking. Suffix rewriting is meant to be efficient, and
+// allowing changes in the TableFormat detracts from that efficiency.
+//
+// Any obsolete bits that key-value pairs may be annotated with are ignored
+// and lost during the rewrite. Additionally, the output sstable has the
+// pebble.obsolete.is_strict property set to false. These limitations could be
+// removed if needed. The current use case for
+// RewriteKeySuffixesAndReturnFormat in CockroachDB is for MVCC-compliant file
+// ingestion, where these files do not contain RANGEDELs and have one
+// key-value pair per userkey -- so they trivially satisfy the strict
+// criteria, and we don't need the obsolete bit as a performance optimization.
+// For disaggregated storage, strict obsolete sstables are needed for L5 and
+// L6, but at the time of writing, we expect such MVCC-compliant file
+// ingestion to only ingest into levels L4 and higher. If this changes, we can
+// do one of two things to get rid of this limitation:
+//   - Validate that there are no duplicate userkeys and no RANGEDELs/MERGEs
+//     in the sstable to be rewritten. Validating no duplicate userkeys is
+//     non-trivial when rewriting blocks in parallel, so we could encode the
+//     pre-existing condition in the (existing) SnapshotPinnedKeys property --
+//     we need to update the external sst writer to calculate and encode this
+//     property.
+//   - Preserve the obsolete bit (with changes to the blockIter).
+func RewriteKeySuffixesAndReturnFormat(
+	sst []byte,
+	rOpts ReaderOptions,
+	out objstorage.Writable,
+	o WriterOptions,
+	from, to []byte,
+	concurrency int,
+) (*WriterMetadata, TableFormat, error) {
+	r, err := NewMemReader(sst, rOpts)
+	if err != nil {
+		return nil, TableFormatUnspecified, err
+	}
+	defer r.Close()
+	return rewriteKeySuffixesInBlocks(r, out, o, from, to, concurrency)
+}
+
+func rewriteKeySuffixesInBlocks(
+	r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte, concurrency int,
+) (*WriterMetadata, TableFormat, error) {
+	if o.Comparer == nil || o.Comparer.Split == nil {
+		return nil, TableFormatUnspecified,
+			errors.New("a valid splitter is required to rewrite suffixes")
+	}
+	if concurrency < 1 {
+		return nil, TableFormatUnspecified, errors.New("concurrency must be >= 1")
+	}
+	// Even though NumValueBlocks = 0 => NumValuesInValueBlocks = 0, check both
+	// as a defensive measure.
+	if r.Properties.NumValueBlocks > 0 || r.Properties.NumValuesInValueBlocks > 0 {
+		return nil, TableFormatUnspecified,
+			errors.New("sstable with a single suffix should not have value blocks")
+	}
+
+	tableFormat := r.tableFormat
+	o.TableFormat = tableFormat
+	w := NewWriter(out, o)
+	defer func() {
+		if w != nil {
+			w.Close()
+		}
+	}()
+
+	for _, c := range w.propCollectors {
+		if _, ok := c.(SuffixReplaceableTableCollector); !ok {
+			return nil, TableFormatUnspecified,
+				errors.Errorf("property collector %s does not support suffix replacement", c.Name())
+		}
+	}
+	for _, c := range w.blockPropCollectors {
+		if _, ok := c.(SuffixReplaceableBlockCollector); !ok {
+			return nil, TableFormatUnspecified,
+				errors.Errorf("block property collector %s does not support suffix replacement", c.Name())
+		}
+	}
+
+	l, err := r.Layout()
+	if err != nil {
+		return nil, TableFormatUnspecified, errors.Wrap(err, "reading layout")
+	}
+
+	if err := rewriteDataBlocksToWriter(r, w, l.Data, from, to, w.split, concurrency); err != nil {
+		return nil, TableFormatUnspecified, errors.Wrap(err, "rewriting data blocks")
+	}
+
+	// Copy over the range key block and replace suffixes in it if it exists.
+	if err := rewriteRangeKeyBlockToWriter(r, w, from, to); err != nil {
+		return nil, TableFormatUnspecified, errors.Wrap(err, "rewriting range key blocks")
+	}
+
+	// Copy over the filter block if it exists (rewriteDataBlocksToWriter will
+	// already have ensured this is valid if it exists).
+	if w.filter != nil && l.Filter.Length > 0 {
+		filterBlock, _, err := readBlockBuf(r, l.Filter, nil)
+		if err != nil {
+			return nil, TableFormatUnspecified, errors.Wrap(err, "reading filter")
+		}
+		w.filter = copyFilterWriter{
+			origPolicyName: w.filter.policyName(), origMetaName: w.filter.metaName(), data: filterBlock,
+		}
+	}
+
+	if err := w.Close(); err != nil {
+		w = nil
+		return nil, TableFormatUnspecified, err
+	}
+	writerMeta, err := w.Metadata()
+	w = nil
+	return writerMeta, tableFormat, err
+}
+
+var errBadKind = errors.New("key does not have expected kind (set)")
+
+type blockWithSpan struct {
+	start, end InternalKey
+	data       []byte
+}
+
+func rewriteBlocks(
+	r *Reader,
+	restartInterval int,
+	checksumType ChecksumType,
+	compression Compression,
+	input []BlockHandleWithProperties,
+	output []blockWithSpan,
+	totalWorkers, worker int,
+	from, to []byte,
+	split Split,
+) error {
+	bw := blockWriter{
+		restartInterval: restartInterval,
+	}
+	buf := blockBuf{checksummer: checksummer{checksumType: checksumType}}
+	if checksumType == ChecksumTypeXXHash {
+		buf.checksummer.xxHasher = xxhash.New()
+	}
+
+	var blockAlloc bytealloc.A
+	var keyAlloc bytealloc.A
+	var scratch InternalKey
+
+	var inputBlock, inputBlockBuf []byte
+
+	iter := &blockIter{}
+
+	// We'll assume all blocks are _roughly_ equal so round-robin static partition
+	// of each worker doing every ith block is probably enough.
+	for i := worker; i < len(input); i += totalWorkers {
+		bh := input[i]
+
+		var err error
+		inputBlock, inputBlockBuf, err = readBlockBuf(r, bh.BlockHandle, inputBlockBuf)
+		if err != nil {
+			return err
+		}
+		if err := iter.init(r.Compare, inputBlock, r.Properties.GlobalSeqNum, false); err != nil {
+			return err
+		}
+
+		if cap(bw.restarts) < int(iter.restarts) {
+			bw.restarts = make([]uint32, 0, iter.restarts)
+		}
+		if cap(bw.buf) == 0 {
+			bw.buf = make([]byte, 0, len(inputBlock))
+		}
+		if cap(bw.restarts) < int(iter.numRestarts) {
+			bw.restarts = make([]uint32, 0, iter.numRestarts)
+		}
+
+		for key, val := iter.First(); key != nil; key, val = iter.Next() {
+			if key.Kind() != InternalKeyKindSet {
+				return errBadKind
+			}
+			si := split(key.UserKey)
+			oldSuffix := key.UserKey[si:]
+			if !bytes.Equal(oldSuffix, from) {
+				err := errors.Errorf("key has suffix %q, expected %q", oldSuffix, from)
+				return err
+			}
+			newLen := si + len(to)
+			if cap(scratch.UserKey) < newLen {
+				scratch.UserKey = make([]byte, 0, len(key.UserKey)*2+len(to)-len(from))
+			}
+
+			scratch.Trailer = key.Trailer
+			scratch.UserKey = scratch.UserKey[:newLen]
+			copy(scratch.UserKey, key.UserKey[:si])
+			copy(scratch.UserKey[si:], to)
+
+			// NB: for TableFormatPebblev3 and higher, since
+			// !iter.lazyValueHandling.hasValuePrefix, it will return the raw value
+			// in the block, which includes the 1-byte prefix. This is fine since bw
+			// also does not know about the prefix and will preserve it in bw.add.
+			v := val.InPlaceValue()
+			if invariants.Enabled && r.tableFormat >= TableFormatPebblev3 &&
+				key.Kind() == InternalKeyKindSet {
+				if len(v) < 1 {
+					return errors.Errorf("value has no prefix")
+				}
+				prefix := valuePrefix(v[0])
+				if isValueHandle(prefix) {
+					return errors.Errorf("value prefix is incorrect")
+				}
+				if setHasSamePrefix(prefix) {
+					return errors.Errorf("multiple keys with same key prefix")
+				}
+			}
+			bw.add(scratch, v)
+			if output[i].start.UserKey == nil {
+				keyAlloc, output[i].start = cloneKeyWithBuf(scratch, keyAlloc)
+			}
+		}
+		*iter = iter.resetForReuse()
+
+		keyAlloc, output[i].end = cloneKeyWithBuf(scratch, keyAlloc)
+
+		finished := compressAndChecksum(bw.finish(), compression, &buf)
+
+		// copy our finished block into the output buffer.
+		blockAlloc, output[i].data = blockAlloc.Alloc(len(finished) + blockTrailerLen)
+		copy(output[i].data, finished)
+		copy(output[i].data[len(finished):], buf.tmp[:blockTrailerLen])
+	}
+	return nil
+}
+
+func rewriteDataBlocksToWriter(
+	r *Reader,
+	w *Writer,
+	data []BlockHandleWithProperties,
+	from, to []byte,
+	split Split,
+	concurrency int,
+) error {
+	if r.Properties.NumEntries == 0 {
+		// No point keys.
+		return nil
+	}
+	blocks := make([]blockWithSpan, len(data))
+
+	if w.filter != nil {
+		if r.Properties.FilterPolicyName != w.filter.policyName() {
+			return errors.New("mismatched filters")
+		}
+		if was, is := r.Properties.ComparerName, w.props.ComparerName; was != is {
+			return errors.Errorf("mismatched Comparer %s vs %s, replacement requires same splitter to copy filters", was, is)
+		}
+	}
+
+	g := &sync.WaitGroup{}
+	g.Add(concurrency)
+	errCh := make(chan error, concurrency)
+	for i := 0; i < concurrency; i++ {
+		worker := i
+		go func() {
+			defer g.Done()
+			err := rewriteBlocks(
+				r,
+				w.dataBlockBuf.dataBlock.restartInterval,
+				w.blockBuf.checksummer.checksumType,
+				w.compression,
+				data,
+				blocks,
+				concurrency,
+				worker,
+				from, to,
+				split,
+			)
+			if err != nil {
+				errCh <- err
+			}
+		}()
+	}
+	g.Wait()
+	close(errCh)
+	if err, ok := <-errCh; ok {
+		return err
+	}
+
+	for _, p := range w.propCollectors {
+		if err := p.(SuffixReplaceableTableCollector).UpdateKeySuffixes(r.Properties.UserProperties, from, to); err != nil {
+			return err
+		}
+	}
+
+	var decoder blockPropertiesDecoder
+	var oldShortIDs []shortID
+	var oldProps [][]byte
+	if len(w.blockPropCollectors) > 0 {
+		oldProps = make([][]byte, len(w.blockPropCollectors))
+		oldShortIDs = make([]shortID, math.MaxUint8)
+		for i, p := range w.blockPropCollectors {
+			if prop, ok := r.Properties.UserProperties[p.Name()]; ok {
+				was, is := shortID(byte(prop[0])), shortID(i)
+				oldShortIDs[was] = is
+			}
+		}
+	}
+
+	for i := range blocks {
+		// Write the rewritten block to the file.
+		if err := w.writable.Write(blocks[i].data); err != nil {
+			return err
+		}
+
+		n := len(blocks[i].data)
+		bh := BlockHandle{Offset: w.meta.Size, Length: uint64(n) - blockTrailerLen}
+		// Update the overall size.
+		w.meta.Size += uint64(n)
+
+		// Load any previous values for our prop collectors into oldProps.
+		for i := range oldProps {
+			oldProps[i] = nil
+		}
+		decoder.props = data[i].Props
+		for !decoder.done() {
+			id, val, err := decoder.next()
+			if err != nil {
+				return err
+			}
+			oldProps[oldShortIDs[id]] = val
+		}
+
+		for i, p := range w.blockPropCollectors {
+			if err := p.(SuffixReplaceableBlockCollector).UpdateKeySuffixes(oldProps[i], from, to); err != nil {
+				return err
+			}
+		}
+
+		bhp, err := w.maybeAddBlockPropertiesToBlockHandle(bh)
+		if err != nil {
+			return err
+		}
+		var nextKey InternalKey
+		if i+1 < len(blocks) {
+			nextKey = blocks[i+1].start
+		}
+		if err = w.addIndexEntrySync(blocks[i].end, nextKey, bhp, w.dataBlockBuf.tmp[:]); err != nil {
+			return err
+		}
+	}
+
+	w.meta.updateSeqNum(blocks[0].start.SeqNum())
+	w.props.NumEntries = r.Properties.NumEntries
+	w.props.RawKeySize = r.Properties.RawKeySize
+	w.props.RawValueSize = r.Properties.RawValueSize
+	w.meta.SetSmallestPointKey(blocks[0].start)
+	w.meta.SetLargestPointKey(blocks[len(blocks)-1].end)
+	return nil
+}
+
+func rewriteRangeKeyBlockToWriter(r *Reader, w *Writer, from, to []byte) error {
+	iter, err := r.NewRawRangeKeyIter()
+	if err != nil {
+		return err
+	}
+	if iter == nil {
+		// No range keys.
+		return nil
+	}
+	defer iter.Close()
+
+	for s := iter.First(); s != nil; s = iter.Next() {
+		if !s.Valid() {
+			break
+		}
+		for i := range s.Keys {
+			if s.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
+				return errBadKind
+			}
+			if !bytes.Equal(s.Keys[i].Suffix, from) {
+				return errors.Errorf("key has suffix %q, expected %q", s.Keys[i].Suffix, from)
+			}
+			s.Keys[i].Suffix = to
+		}
+
+		err := rangekey.Encode(s, func(k base.InternalKey, v []byte) error {
+			// Calling AddRangeKey instead of addRangeKeySpan bypasses the fragmenter.
+			// This is okay because the raw fragments off of `iter` are already
+			// fragmented, and suffix replacement should not affect fragmentation.
+			return w.AddRangeKey(k, v)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type copyFilterWriter struct {
+	origMetaName   string
+	origPolicyName string
+	data           []byte
+}
+
+func (copyFilterWriter) addKey(key []byte)         { panic("unimplemented") }
+func (c copyFilterWriter) finish() ([]byte, error) { return c.data, nil }
+func (c copyFilterWriter) metaName() string        { return c.origMetaName }
+func (c copyFilterWriter) policyName() string      { return c.origPolicyName }
+
+// RewriteKeySuffixesViaWriter is similar to RewriteKeySuffixes but uses just a
+// single loop over the Reader that writes each key to the Writer with the new
+// suffix. The is significantly slower than the parallelized rewriter, and does
+// more work to rederive filters, props, etc.
+//
+// Any obsolete bits that key-value pairs may be annotated with are ignored
+// and lost during the rewrite. Some of the obsolete bits may be recreated --
+// specifically when there are multiple keys with the same user key.
+// Additionally, the output sstable has the pebble.obsolete.is_strict property
+// set to false. See the longer comment at RewriteKeySuffixesAndReturnFormat.
+func RewriteKeySuffixesViaWriter(
+	r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte,
+) (*WriterMetadata, error) {
+	if o.Comparer == nil || o.Comparer.Split == nil {
+		return nil, errors.New("a valid splitter is required to rewrite suffixes")
+	}
+
+	o.IsStrictObsolete = false
+	w := NewWriter(out, o)
+	defer func() {
+		if w != nil {
+			w.Close()
+		}
+	}()
+	i, err := r.NewIter(nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer i.Close()
+
+	k, v := i.First()
+	var scratch InternalKey
+	for k != nil {
+		if k.Kind() != InternalKeyKindSet {
+			return nil, errors.New("invalid key type")
+		}
+		oldSuffix := k.UserKey[r.Split(k.UserKey):]
+		if !bytes.Equal(oldSuffix, from) {
+			return nil, errors.Errorf("key has suffix %q, expected %q", oldSuffix, from)
+		}
+		scratch.UserKey = append(scratch.UserKey[:0], k.UserKey[:len(k.UserKey)-len(from)]...)
+		scratch.UserKey = append(scratch.UserKey, to...)
+		scratch.Trailer = k.Trailer
+
+		val, _, err := v.Value(nil)
+		if err != nil {
+			return nil, err
+		}
+		if w.addPoint(scratch, val, false); err != nil {
+			return nil, err
+		}
+		k, v = i.Next()
+	}
+	if err := rewriteRangeKeyBlockToWriter(r, w, from, to); err != nil {
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		w = nil
+		return nil, err
+	}
+	writerMeta, err := w.Metadata()
+	w = nil
+	return writerMeta, err
+}
+
+// NewMemReader opens a reader over the SST stored in the passed []byte.
+func NewMemReader(sst []byte, o ReaderOptions) (*Reader, error) {
+	return NewReader(newMemReader(sst), o)
+}
+
+func readBlockBuf(r *Reader, bh BlockHandle, buf []byte) ([]byte, []byte, error) {
+	raw := r.readable.(*memReader).b[bh.Offset : bh.Offset+bh.Length+blockTrailerLen]
+	if err := checkChecksum(r.checksumType, raw, bh, 0); err != nil {
+		return nil, buf, err
+	}
+	typ := blockType(raw[bh.Length])
+	raw = raw[:bh.Length]
+	if typ == noCompressionBlockType {
+		return raw, buf, nil
+	}
+	decompressedLen, prefix, err := decompressedLen(typ, raw)
+	if err != nil {
+		return nil, buf, err
+	}
+	if cap(buf) < decompressedLen {
+		buf = make([]byte, decompressedLen)
+	}
+	res, err := decompressInto(typ, raw[prefix:], buf[:decompressedLen])
+	return res, buf, err
+}
+
+// memReader is a thin wrapper around a []byte such that it can be passed to
+// sstable.Reader. It supports concurrent use, and does so without locking in
+// contrast to the heavier read/write vfs.MemFile.
+type memReader struct {
+	b  []byte
+	r  *bytes.Reader
+	rh objstorage.NoopReadHandle
+}
+
+var _ objstorage.Readable = (*memReader)(nil)
+
+func newMemReader(b []byte) *memReader {
+	r := &memReader{
+		b: b,
+		r: bytes.NewReader(b),
+	}
+	r.rh = objstorage.MakeNoopReadHandle(r)
+	return r
+}
+
+// ReadAt is part of objstorage.Readable.
+func (m *memReader) ReadAt(_ context.Context, p []byte, off int64) error {
+	n, err := m.r.ReadAt(p, off)
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
+}
+
+// Close is part of objstorage.Readable.
+func (*memReader) Close() error {
+	return nil
+}
+
+// Stat is part of objstorage.Readable.
+func (m *memReader) Size() int64 {
+	return int64(len(m.b))
+}
+
+// NewReadHandle is part of objstorage.Readable.
+func (m *memReader) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+	return &m.rh
+}

vendor/github.com/cockroachdb/pebble/sstable/table.go

@@ -0,0 +1,455 @@
+// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+// Package sstable implements readers and writers of pebble tables.
+//
+// Tables are either opened for reading or created for writing but not both.
+//
+// A reader can create iterators, which allow seeking and next/prev
+// iteration. There may be multiple key/value pairs that have the same key and
+// different sequence numbers.
+//
+// A reader can be used concurrently. Multiple goroutines can call NewIter
+// concurrently, and each iterator can run concurrently with other iterators.
+// However, any particular iterator should not be used concurrently, and iterators
+// should not be used once a reader is closed.
+//
+// A writer writes key/value pairs in increasing key order, and cannot be used
+// concurrently. A table cannot be read until the writer has finished.
+//
+// Readers and writers can be created with various options. Passing a nil
+// Options pointer is valid and means to use the default values.
+//
+// One such option is to define the 'less than' ordering for keys. The default
+// Comparer uses the natural ordering consistent with bytes.Compare. The same
+// ordering should be used for reading and writing a table.
+//
+// To return the value for a key:
+//
+//	r := table.NewReader(file, options)
+//	defer r.Close()
+//	i := r.NewIter(nil, nil)
+//	defer i.Close()
+//	ikey, value := r.SeekGE(key)
+//	if options.Comparer.Compare(ikey.UserKey, key) != 0 {
+//	  // not found
+//	} else {
+//	  // value is the first record containing key
+//	}
+//
+// To count the number of entries in a table:
+//
+//	i, n := r.NewIter(nil, nil), 0
+//	for key, value := i.First(); key != nil; key, value = i.Next() {
+//		n++
+//	}
+//	if err := i.Close(); err != nil {
+//		return 0, err
+//	}
+//	return n, nil
+//
+// To write a table with three entries:
+//
+//	w := table.NewWriter(file, options)
+//	if err := w.Set([]byte("apple"), []byte("red")); err != nil {
+//		w.Close()
+//		return err
+//	}
+//	if err := w.Set([]byte("banana"), []byte("yellow")); err != nil {
+//		w.Close()
+//		return err
+//	}
+//	if err := w.Set([]byte("cherry"), []byte("red")); err != nil {
+//		w.Close()
+//		return err
+//	}
+//	return w.Close()
+package sstable // import "github.com/cockroachdb/pebble/sstable"
+
+import (
+	"context"
+	"encoding/binary"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+/*
+The table file format looks like:
+
+<start_of_file>
+[data block 0]
+[data block 1]
+...
+[data block N-1]
+[meta filter block] (optional)
+[index block] (for single level index)
+[meta rangedel block] (optional)
+[meta range key block] (optional)
+[value block 0] (optional)
+[value block M-1] (optional)
+[meta value index block] (optional)
+[meta properties block]
+[metaindex block]
+[footer]
+<end_of_file>
+
+A Reader eagerly loads the footer, metaindex block and meta properties block,
+because the data contained in those blocks is needed on every read, and even
+before reading. For example, the meta properties block is used to verify the
+comparer and merger are compatible, and the metaindex block contains the
+location of the meta properties (and other meta blocks). In situations where
+file system locality matters, or one wants to minimize number of read
+requests when eagerly loading these blocks, having these three as a suffix
+of the file is convenient.
+
+The interleaving of the index block(s) between the meta blocks is done to
+match RocksDB/LevelDB behavior.
+
+Each block consists of some data and a 5 byte trailer: a 1 byte block type and a
+4 byte checksum. The checksum is computed over the compressed data and the first
+byte of the trailer (i.e. the block type), and is serialized as little-endian.
+The block type gives the per-block compression used; each block is compressed
+independently. The checksum algorithm is described in the pebble/crc package.
+
+Most blocks, other than the meta filter block, value blocks and meta value
+index block, contain key/value pairs. The remainder of this comment refers to
+the decompressed block, containing key/value pairs, which has its 5 byte
+trailer stripped. The decompressed block data consists of a sequence of such
+key/value entries followed by a block suffix. Each key is encoded as a shared
+prefix length and a remainder string. For example, if two adjacent keys are
+"tweedledee" and "tweedledum", then the second key would be encoded as {8,
+"um"}. The shared prefix length is varint encoded. The remainder string and the
+value are encoded as a varint-encoded length followed by the literal contents.
+To continue the example, suppose that the key "tweedledum" mapped to the value
+"socks". The encoded key/value entry would be: "\x08\x02\x05umsocks".
+
+Every block has a restart interval I. Every I'th key/value entry in that block
+is called a restart point, and shares no key prefix with the previous entry.
+Continuing the example above, if the key after "tweedledum" was "two", but was
+part of a restart point, then that key would be encoded as {0, "two"} instead
+of {2, "o"}. If a block has P restart points, then the block suffix consists
+of (P+1)*4 bytes: (P+1) little-endian uint32 values. The first P of these
+uint32 values are the block offsets of each restart point. The final uint32
+value is P itself. Thus, when seeking for a particular key, one can use binary
+search to find the largest restart point whose key is <= the key sought.
+
+An index block is a block with N key/value entries. The i'th value is the
+encoded block handle of the i'th data block. The i'th key is a separator for
+i < N-1, and a successor for i == N-1. The separator between blocks i and i+1
+is a key that is >= every key in block i and is < every key i block i+1. The
+successor for the final block is a key that is >= every key in block N-1. The
+index block restart interval is 1: every entry is a restart point.
+
+A block handle is an offset, a length, and optional block properties (for data
+blocks and first/lower level index blocks); the length does not include the 5
+byte trailer. All numbers are varint-encoded, with no padding between the two
+values. The maximum size of an encoded block handle without properties is 20
+bytes. It is not advised to have properties that accumulate to be longer than
+100 bytes.
+
+Instead of a single index block, the sstable can have a two-level index (this
+is used to prevent a single huge index block). A two-level index consists of a
+sequence of lower-level index blocks with block handles for data blocks
+followed by a single top-level index block with block handles for the
+lower-level index blocks.
+
+The metaindex block also contains block handles as values, with keys being
+the names of the meta blocks.
+
+For a description of value blocks and the meta value index block, see
+value_block.go.
+
+Data blocks have some additional features:
+- For TableFormatPebblev3 onwards:
+  - For SETs, the value has a 1 byte value prefix, which indicates whether the
+    value is inline, or in a separate value block, and indicates whether the
+    prefix of the userkey (as defined by split) has changed or not. See
+    value_block.go for details.
+  - The most significant bit of the restart points is used to indicate whether
+    userkey prefix has changed since the last restart point. See the detailed
+    comment in blockWriter.
+  - The maximum length of the "shared prefix" when encoding the key, is the
+    length of the prefix of the userkey (as defined by split) of the previous
+    key.
+
+- For TableFormatPebblev4 onwards:
+  - The key kinds may be altered to set the
+    InternalKeyKindSSTableInternalObsoleteBit if the key-value pair is obsolete
+    in the context of that sstable (for a reader that reads at a higher seqnum
+    than the highest seqnum in the sstable). For details, see the comment in
+    format.go.
+*/
+
+const (
+	blockTrailerLen                    = 5
+	blockHandleMaxLenWithoutProperties = 10 + 10
+	// blockHandleLikelyMaxLen can be used for pre-allocating buffers to
+	// reduce memory copies. It is not guaranteed that a block handle will not
+	// exceed this length.
+	blockHandleLikelyMaxLen = blockHandleMaxLenWithoutProperties + 100
+
+	levelDBFooterLen   = 48
+	levelDBMagic       = "\x57\xfb\x80\x8b\x24\x75\x47\xdb"
+	levelDBMagicOffset = levelDBFooterLen - len(levelDBMagic)
+
+	rocksDBFooterLen             = 1 + 2*blockHandleMaxLenWithoutProperties + 4 + 8
+	rocksDBMagic                 = "\xf7\xcf\xf4\x85\xb7\x41\xe2\x88"
+	rocksDBMagicOffset           = rocksDBFooterLen - len(rocksDBMagic)
+	rocksDBVersionOffset         = rocksDBMagicOffset - 4
+	rocksDBExternalFormatVersion = 2
+
+	pebbleDBMagic = "\xf0\x9f\xaa\xb3\xf0\x9f\xaa\xb3" // 🪳🪳
+
+	minFooterLen = levelDBFooterLen
+	maxFooterLen = rocksDBFooterLen
+
+	levelDBFormatVersion  = 0
+	rocksDBFormatVersion2 = 2
+
+	metaRangeKeyName   = "pebble.range_key"
+	metaValueIndexName = "pebble.value_index"
+	metaPropertiesName = "rocksdb.properties"
+	metaRangeDelName   = "rocksdb.range_del"
+	metaRangeDelV2Name = "rocksdb.range_del2"
+
+	// Index Types.
+	// A space efficient index block that is optimized for binary-search-based
+	// index.
+	binarySearchIndex = 0
+	// hashSearchIndex               = 1
+	// A two-level index implementation. Both levels are binary search indexes.
+	twoLevelIndex = 2
+	// binarySearchWithFirstKeyIndex = 3
+
+	// RocksDB always includes this in the properties block. Since Pebble
+	// doesn't use zstd compression, the string will always be the same.
+	// This should be removed if we ever decide to diverge from the RocksDB
+	// properties block.
+	rocksDBCompressionOptions = "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; "
+)
+
+// ChecksumType specifies the checksum used for blocks.
+type ChecksumType byte
+
+// The available checksum types.
+const (
+	ChecksumTypeNone     ChecksumType = 0
+	ChecksumTypeCRC32c   ChecksumType = 1
+	ChecksumTypeXXHash   ChecksumType = 2
+	ChecksumTypeXXHash64 ChecksumType = 3
+)
+
+// String implements fmt.Stringer.
+func (t ChecksumType) String() string {
+	switch t {
+	case ChecksumTypeCRC32c:
+		return "crc32c"
+	case ChecksumTypeNone:
+		return "none"
+	case ChecksumTypeXXHash:
+		return "xxhash"
+	case ChecksumTypeXXHash64:
+		return "xxhash64"
+	default:
+		panic(errors.Newf("sstable: unknown checksum type: %d", t))
+	}
+}
+
+type blockType byte
+
+const (
+	// The block type gives the per-block compression format.
+	// These constants are part of the file format and should not be changed.
+	// They are different from the Compression constants because the latter
+	// are designed so that the zero value of the Compression type means to
+	// use the default compression (which is snappy).
+	// Not all compression types listed here are supported.
+	noCompressionBlockType     blockType = 0
+	snappyCompressionBlockType blockType = 1
+	zlibCompressionBlockType   blockType = 2
+	bzip2CompressionBlockType  blockType = 3
+	lz4CompressionBlockType    blockType = 4
+	lz4hcCompressionBlockType  blockType = 5
+	xpressCompressionBlockType blockType = 6
+	zstdCompressionBlockType   blockType = 7
+)
+
+// String implements fmt.Stringer.
+func (t blockType) String() string {
+	switch t {
+	case 0:
+		return "none"
+	case 1:
+		return "snappy"
+	case 2:
+		return "zlib"
+	case 3:
+		return "bzip2"
+	case 4:
+		return "lz4"
+	case 5:
+		return "lz4hc"
+	case 6:
+		return "xpress"
+	case 7:
+		return "zstd"
+	default:
+		panic(errors.Newf("sstable: unknown block type: %d", t))
+	}
+}
+
+// legacy (LevelDB) footer format:
+//
+//	metaindex handle (varint64 offset, varint64 size)
+//	index handle     (varint64 offset, varint64 size)
+//	<padding> to make the total size 2 * BlockHandle::kMaxEncodedLength
+//	table_magic_number (8 bytes)
+//
+// new (RocksDB) footer format:
+//
+//	checksum type (char, 1 byte)
+//	metaindex handle (varint64 offset, varint64 size)
+//	index handle     (varint64 offset, varint64 size)
+//	<padding> to make the total size 2 * BlockHandle::kMaxEncodedLength + 1
+//	footer version (4 bytes)
+//	table_magic_number (8 bytes)
+type footer struct {
+	format      TableFormat
+	checksum    ChecksumType
+	metaindexBH BlockHandle
+	indexBH     BlockHandle
+	footerBH    BlockHandle
+}
+
+func readFooter(f objstorage.Readable) (footer, error) {
+	var footer footer
+	size := f.Size()
+	if size < minFooterLen {
+		return footer, base.CorruptionErrorf("pebble/table: invalid table (file size is too small)")
+	}
+
+	buf := make([]byte, maxFooterLen)
+	off := size - maxFooterLen
+	if off < 0 {
+		off = 0
+		buf = buf[:size]
+	}
+	if err := f.ReadAt(context.TODO(), buf, off); err != nil {
+		return footer, errors.Wrap(err, "pebble/table: invalid table (could not read footer)")
+	}
+
+	switch magic := buf[len(buf)-len(rocksDBMagic):]; string(magic) {
+	case levelDBMagic:
+		if len(buf) < levelDBFooterLen {
+			return footer, base.CorruptionErrorf(
+				"pebble/table: invalid table (footer too short): %d", errors.Safe(len(buf)))
+		}
+		footer.footerBH.Offset = uint64(off+int64(len(buf))) - levelDBFooterLen
+		buf = buf[len(buf)-levelDBFooterLen:]
+		footer.footerBH.Length = uint64(len(buf))
+		footer.format = TableFormatLevelDB
+		footer.checksum = ChecksumTypeCRC32c
+
+	case rocksDBMagic, pebbleDBMagic:
+		// NOTE: The Pebble magic string implies the same footer format as that used
+		// by the RocksDBv2 table format.
+		if len(buf) < rocksDBFooterLen {
+			return footer, base.CorruptionErrorf("pebble/table: invalid table (footer too short): %d", errors.Safe(len(buf)))
+		}
+		footer.footerBH.Offset = uint64(off+int64(len(buf))) - rocksDBFooterLen
+		buf = buf[len(buf)-rocksDBFooterLen:]
+		footer.footerBH.Length = uint64(len(buf))
+		version := binary.LittleEndian.Uint32(buf[rocksDBVersionOffset:rocksDBMagicOffset])
+
+		format, err := ParseTableFormat(magic, version)
+		if err != nil {
+			return footer, err
+		}
+		footer.format = format
+
+		switch ChecksumType(buf[0]) {
+		case ChecksumTypeCRC32c:
+			footer.checksum = ChecksumTypeCRC32c
+		case ChecksumTypeXXHash64:
+			footer.checksum = ChecksumTypeXXHash64
+		default:
+			return footer, base.CorruptionErrorf("pebble/table: unsupported checksum type %d", errors.Safe(footer.checksum))
+		}
+		buf = buf[1:]
+
+	default:
+		return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number: 0x%x)", magic)
+	}
+
+	{
+		end := uint64(size)
+		var n int
+		footer.metaindexBH, n = decodeBlockHandle(buf)
+		if n == 0 || footer.metaindexBH.Offset+footer.metaindexBH.Length > end {
+			return footer, base.CorruptionErrorf("pebble/table: invalid table (bad metaindex block handle)")
+		}
+		buf = buf[n:]
+
+		footer.indexBH, n = decodeBlockHandle(buf)
+		if n == 0 || footer.indexBH.Offset+footer.indexBH.Length > end {
+			return footer, base.CorruptionErrorf("pebble/table: invalid table (bad index block handle)")
+		}
+	}
+
+	return footer, nil
+}
+
+func (f footer) encode(buf []byte) []byte {
+	switch magic, version := f.format.AsTuple(); magic {
+	case levelDBMagic:
+		buf = buf[:levelDBFooterLen]
+		for i := range buf {
+			buf[i] = 0
+		}
+		n := encodeBlockHandle(buf[0:], f.metaindexBH)
+		encodeBlockHandle(buf[n:], f.indexBH)
+		copy(buf[len(buf)-len(levelDBMagic):], levelDBMagic)
+
+	case rocksDBMagic, pebbleDBMagic:
+		buf = buf[:rocksDBFooterLen]
+		for i := range buf {
+			buf[i] = 0
+		}
+		switch f.checksum {
+		case ChecksumTypeNone:
+			buf[0] = byte(ChecksumTypeNone)
+		case ChecksumTypeCRC32c:
+			buf[0] = byte(ChecksumTypeCRC32c)
+		case ChecksumTypeXXHash:
+			buf[0] = byte(ChecksumTypeXXHash)
+		case ChecksumTypeXXHash64:
+			buf[0] = byte(ChecksumTypeXXHash64)
+		default:
+			panic("unknown checksum type")
+		}
+		n := 1
+		n += encodeBlockHandle(buf[n:], f.metaindexBH)
+		encodeBlockHandle(buf[n:], f.indexBH)
+		binary.LittleEndian.PutUint32(buf[rocksDBVersionOffset:], version)
+		copy(buf[len(buf)-len(rocksDBMagic):], magic)
+
+	default:
+		panic("sstable: unspecified table format version")
+	}
+
+	return buf
+}
+
+func supportsTwoLevelIndex(format TableFormat) bool {
+	switch format {
+	case TableFormatLevelDB:
+		return false
+	case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2, TableFormatPebblev3, TableFormatPebblev4:
+		return true
+	default:
+		panic("sstable: unspecified table format version")
+	}
+}

vendor/github.com/cockroachdb/pebble/sstable/unsafe.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/manual"
+)
+
+func getBytes(ptr unsafe.Pointer, length int) []byte {
+	return (*[manual.MaxArrayLen]byte)(ptr)[:length:length]
+}
+
+func decodeVarint(ptr unsafe.Pointer) (uint32, unsafe.Pointer) {
+	if a := *((*uint8)(ptr)); a < 128 {
+		return uint32(a),
+			unsafe.Pointer(uintptr(ptr) + 1)
+	} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+		return uint32(b)<<7 | uint32(a),
+			unsafe.Pointer(uintptr(ptr) + 2)
+	} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+		return uint32(c)<<14 | uint32(b)<<7 | uint32(a),
+			unsafe.Pointer(uintptr(ptr) + 3)
+	} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+		return uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a),
+			unsafe.Pointer(uintptr(ptr) + 4)
+	} else {
+		d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+		return uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a),
+			unsafe.Pointer(uintptr(ptr) + 5)
+	}
+}

vendor/github.com/cockroachdb/pebble/sstable/value_block.go

@@ -0,0 +1,950 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"context"
+	"encoding/binary"
+	"io"
+	"sync"
+	"unsafe"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
+	"golang.org/x/exp/rand"
+)
+
+// Value blocks are supported in TableFormatPebblev3.
+//
+// 1. Motivation and overview
+//
+// Value blocks are a mechanism designed for sstables storing MVCC data, where
+// there can be many versions of a key that need to be kept, but only the
+// latest value is typically read (see the documentation for Comparer.Split
+// regarding MVCC keys). The goal is faster reads. Unlike Pebble versions,
+// which can be eagerly thrown away (except when there are snapshots), MVCC
+// versions are long-lived (e.g. default CockroachDB garbage collection
+// threshold for older versions is 24 hours) and can significantly slow down
+// reads. We have seen CockroachDB production workloads with very slow reads
+// due to:
+// - 100s of versions for each key in a table.
+//
+// - Tables with mostly MVCC garbage consisting of 2 versions per key -- a
+//   real key-value pair, followed by a key-value pair whose value (usually
+//   with zero byte length) indicates it is an MVCC tombstone.
+//
+// The value blocks mechanism attempts to improve read throughput in these
+// cases when the key size is smaller than the value sizes of older versions.
+// This is done by moving the value of an older version to a value block in a
+// different part of the sstable. This improves spatial locality of the data
+// being read by the workload, which increases caching effectiveness.
+//
+// Additionally, even when the key size is not smaller than the value of older
+// versions (e.g. secondary indexes in CockroachDB), TableFormatPebblev3
+// stores the result of key comparisons done at write time inside the sstable,
+// which makes stepping from one key prefix to the next prefix (i.e., skipping
+// over older versions of a MVCC key) more efficient by avoiding key
+// comparisons and key decoding. See the results in
+// https://github.com/cockroachdb/pebble/pull/2149 and more details in the
+// comment inside BenchmarkIteratorScanNextPrefix. These improvements are also
+// visible in end-to-end CockroachDB tests, as outlined in
+// https://github.com/cockroachdb/cockroach/pull/96652.
+//
+// In TableFormatPebblev3, each SET has a one byte value prefix that tells us
+// whether the value is in-place or in a value block. This 1 byte prefix
+// encodes additional information:
+//
+// - ShortAttribute: This is an attribute of the value. Currently, CockroachDB
+//   uses it to represent whether the value is a tombstone or not. This avoids
+//   the need to fetch a value from the value block if the caller only wants
+//   to figure out whether it is an MVCC tombstone. The length of the value is
+//   another attribute that the caller can be interested in, and it is also
+//   accessible without reading the value in the value block (see the value
+//   handle in the details section).
+//
+// - SET-same-prefix: this enables the aforementioned optimization when
+//   stepping from one key prefix to the next key prefix.
+//
+// We further optimize this iteration over prefixes by using the restart
+// points in a block to encode whether the SET at a restart point has the same
+// prefix since the last restart point. This allows us to skip over restart
+// points within the same block. See the comment in blockWriter, and how both
+// SET-same-prefix and the restart point information is used in
+// blockIter.nextPrefixV3.
+//
+// This flexibility of values that are in-place or in value blocks requires
+// flexibility in the iterator interface. The InternalIterator interface
+// returns a LazyValue instead of a byte slice. Additionally, pebble.Iterator
+// allows the caller to ask for a LazyValue. See lazy_value.go for details,
+// including the memory lifetime management.
+//
+// For historical discussions about this feature, see the issue
+// https://github.com/cockroachdb/pebble/issues/1170 and the prototype in
+// https://github.com/cockroachdb/pebble/pull/1443.
+//
+// The code in this file mainly covers value block and related encodings. We
+// discuss these in the next section.
+//
+// 2. Details
+//
+// Note that the notion of the latest value is local to the sstable. It is
+// possible that that latest value has been deleted by a sstable in a higher
+// level, and what is the latest value from the perspective of the whole LSM
+// is an older MVCC version. This only affects performance and not
+// correctness. This local knowledge is also why we continue to store these
+// older versions in the same sstable -- we need to be able to conveniently
+// read them. The code in this file is agnostic to the policy regarding what
+// should be stored in value blocks -- it allows even the latest MVCC version
+// to be stored in a value block. The policy decision in made in the
+// sstable.Writer. See Writer.makeAddPointDecisionV3.
+//
+// Data blocks contain two kinds of SET keys: those with in-place values and
+// those with a value handle. To distinguish these two cases we use a single
+// byte prefix (valuePrefix). This single byte prefix is split into multiple
+// parts, where nb represents information that is encoded in n bits.
+//
+// +---------------+--------------------+-----------+--------------------+
+// | value-kind 2b | SET-same-prefix 1b | unused 2b | short-attribute 3b |
+// +---------------+--------------------+-----------+--------------------+
+//
+// The 2 bit value-kind specifies whether this is an in-place value or a value
+// handle pointing to a value block. We use 2 bits here for future
+// representation of values that are in separate files. The 1 bit
+// SET-same-prefix is true if this key is a SET and is immediately preceded by
+// a SET that shares the same prefix. The 3 bit short-attribute is described
+// in base.ShortAttribute -- it stores user-defined attributes about the
+// value. It is unused for in-place values.
+//
+// Value Handle and Value Blocks:
+// valueHandles refer to values in value blocks. Value blocks are simpler than
+// normal data blocks (that contain key-value pairs, and allow for binary
+// search), which makes them cheap for value retrieval purposes. A valueHandle
+// is a tuple (valueLen, blockNum, offsetInBlock), where blockNum is the 0
+// indexed value block number and offsetInBlock is the byte offset in that
+// block containing the value. The valueHandle.valueLen is included since
+// there are multiple use cases in CockroachDB that need the value length but
+// not the value, for which we can avoid reading the value in the value block
+// (see
+// https://github.com/cockroachdb/pebble/issues/1170#issuecomment-958203245).
+//
+// A value block has a checksum like other blocks, and is optionally
+// compressed. An uncompressed value block is a sequence of values with no
+// separator or length (we rely on the valueHandle to demarcate). The
+// valueHandle.offsetInBlock points to the value, of length
+// valueHandle.valueLen. While writing a sstable, all the (possibly
+// compressed) value blocks need to be held in-memory until they can be
+// written. Value blocks are placed after the "meta rangedel" and "meta range
+// key" blocks since value blocks are considered less likely to be read.
+//
+// Meta Value Index Block:
+// Since the (key, valueHandle) pair are written before there is any knowledge
+// of the byte offset of the value block in the file, or its compressed
+// length, we need another lookup to map the valueHandle.blockNum to the
+// information needed to read it from the file. This information is provided
+// by the "value index block". The "value index block" is referred to by the
+// metaindex block. The design intentionally avoids making the "value index
+// block" a general purpose key-value block, since each caller wants to lookup
+// the information for a particular blockNum (there is no need for SeekGE
+// etc.). Instead, this index block stores a sequence of (blockNum,
+// blockOffset, blockLength) tuples, where the blockNums are consecutive
+// integers, and the tuples are encoded with a fixed width encoding. This
+// allows a reader to find the tuple for block K by looking at the offset
+// K*fixed-width. The fixed width for each field is decided by looking at the
+// maximum value of each of these fields. As a concrete example of a large
+// sstable with many value blocks, we constructed a 100MB sstable with many
+// versions and had 2475 value blocks (~32KB each). This sstable had this
+// tuple encoded using 2+4+2=8 bytes, which means the uncompressed value index
+// block was 2475*8=~19KB, which is modest. Therefore, we don't support more
+// than one value index block. Consider the example of 2 byte blockNum, 4 byte
+// blockOffset and 2 byte blockLen. The value index block will look like:
+//
+//   +---------------+------------------+---------------+
+//   | blockNum (2B) | blockOffset (4B) | blockLen (2B) |
+//   +---------------+------------------+---------------+
+//   |       0       |    7,123,456     |  30,000       |
+//   +---------------+------------------+---------------+
+//   |       1       |    7,153,456     |  20,000       |
+//   +---------------+------------------+---------------+
+//   |       2       |    7,173,456     |  25,567       |
+//   +---------------+------------------+---------------+
+//   |     ....      |      ...         |    ...        |
+//
+//
+// The metaindex block contains the valueBlocksIndexHandle which in addition
+// to the BlockHandle also specifies the widths of these tuple fields. In the
+// above example, the
+// valueBlockIndexHandle.{blockNumByteLength,blockOffsetByteLength,blockLengthByteLength}
+// will be (2,4,2).
+
+// valueHandle is stored with a key when the value is in a value block. This
+// handle is the pointer to that value.
+type valueHandle struct {
+	valueLen      uint32
+	blockNum      uint32
+	offsetInBlock uint32
+}
+
+// valuePrefix is the single byte prefix for either the in-place value or the
+// encoded valueHandle. It encoded multiple kinds of information.
+type valuePrefix byte
+
+const (
+	// 2 most-significant bits of valuePrefix encodes the value-kind.
+	valueKindMask           valuePrefix = '\xC0'
+	valueKindIsValueHandle  valuePrefix = '\x80'
+	valueKindIsInPlaceValue valuePrefix = '\x00'
+
+	// 1 bit indicates SET has same key prefix as immediately preceding key that
+	// is also a SET. If the immediately preceding key in the same block is a
+	// SET, AND this bit is 0, the prefix must have changed.
+	//
+	// Note that the current policy of only storing older MVCC versions in value
+	// blocks means that valueKindIsValueHandle => SET has same prefix. But no
+	// code should rely on this behavior. Also, SET has same prefix does *not*
+	// imply valueKindIsValueHandle.
+	setHasSameKeyPrefixMask valuePrefix = '\x20'
+
+	// 3 least-significant bits for the user-defined base.ShortAttribute.
+	// Undefined for valueKindIsInPlaceValue.
+	userDefinedShortAttributeMask valuePrefix = '\x07'
+)
+
+// valueHandle fields are varint encoded, so maximum 5 bytes each, plus 1 byte
+// for the valuePrefix. This could alternatively be group varint encoded, but
+// experiments were inconclusive
+// (https://github.com/cockroachdb/pebble/pull/1443#issuecomment-1270298802).
+const valueHandleMaxLen = 5*3 + 1
+
+// Assert blockHandleLikelyMaxLen >= valueHandleMaxLen.
+const _ = uint(blockHandleLikelyMaxLen - valueHandleMaxLen)
+
+func encodeValueHandle(dst []byte, v valueHandle) int {
+	n := 0
+	n += binary.PutUvarint(dst[n:], uint64(v.valueLen))
+	n += binary.PutUvarint(dst[n:], uint64(v.blockNum))
+	n += binary.PutUvarint(dst[n:], uint64(v.offsetInBlock))
+	return n
+}
+
+func makePrefixForValueHandle(setHasSameKeyPrefix bool, attribute base.ShortAttribute) valuePrefix {
+	prefix := valueKindIsValueHandle | valuePrefix(attribute)
+	if setHasSameKeyPrefix {
+		prefix = prefix | setHasSameKeyPrefixMask
+	}
+	return prefix
+}
+
+func makePrefixForInPlaceValue(setHasSameKeyPrefix bool) valuePrefix {
+	prefix := valueKindIsInPlaceValue
+	if setHasSameKeyPrefix {
+		prefix = prefix | setHasSameKeyPrefixMask
+	}
+	return prefix
+}
+
+func isValueHandle(b valuePrefix) bool {
+	return b&valueKindMask == valueKindIsValueHandle
+}
+
+// REQUIRES: isValueHandle(b)
+func getShortAttribute(b valuePrefix) base.ShortAttribute {
+	return base.ShortAttribute(b & userDefinedShortAttributeMask)
+}
+
+func setHasSamePrefix(b valuePrefix) bool {
+	return b&setHasSameKeyPrefixMask == setHasSameKeyPrefixMask
+}
+
+func decodeLenFromValueHandle(src []byte) (uint32, []byte) {
+	ptr := unsafe.Pointer(&src[0])
+	var v uint32
+	if a := *((*uint8)(ptr)); a < 128 {
+		v = uint32(a)
+		src = src[1:]
+	} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+		v = uint32(b)<<7 | uint32(a)
+		src = src[2:]
+	} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+		v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		src = src[3:]
+	} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+		v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		src = src[4:]
+	} else {
+		d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+		v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		src = src[5:]
+	}
+	return v, src
+}
+
+func decodeRemainingValueHandle(src []byte) valueHandle {
+	var vh valueHandle
+	ptr := unsafe.Pointer(&src[0])
+	// Manually inlined uvarint decoding. Saves ~25% in benchmarks. Unrolling
+	// a loop for i:=0; i<2; i++, saves ~6%.
+	var v uint32
+	if a := *((*uint8)(ptr)); a < 128 {
+		v = uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 1)
+	} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+		v = uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 2)
+	} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+		v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 3)
+	} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+		v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 4)
+	} else {
+		d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+		v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 5)
+	}
+	vh.blockNum = v
+
+	if a := *((*uint8)(ptr)); a < 128 {
+		v = uint32(a)
+	} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+		v = uint32(b)<<7 | uint32(a)
+	} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+		v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+	} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+		v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+	} else {
+		d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+		v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+	}
+	vh.offsetInBlock = v
+
+	return vh
+}
+
+func decodeValueHandle(src []byte) valueHandle {
+	valLen, src := decodeLenFromValueHandle(src)
+	vh := decodeRemainingValueHandle(src)
+	vh.valueLen = valLen
+	return vh
+}
+
+// valueBlocksIndexHandle is placed in the metaindex if there are any value
+// blocks. If there are no value blocks, there is no value blocks index, and
+// no entry in the metaindex. Note that the lack of entry in the metaindex
+// should not be used to ascertain whether the values are prefixed, since the
+// former is an emergent property of the data that was written and not known
+// until all the key-value pairs in the sstable are written.
+type valueBlocksIndexHandle struct {
+	h                     BlockHandle
+	blockNumByteLength    uint8
+	blockOffsetByteLength uint8
+	blockLengthByteLength uint8
+}
+
+const valueBlocksIndexHandleMaxLen = blockHandleMaxLenWithoutProperties + 3
+
+// Assert blockHandleLikelyMaxLen >= valueBlocksIndexHandleMaxLen.
+const _ = uint(blockHandleLikelyMaxLen - valueBlocksIndexHandleMaxLen)
+
+func encodeValueBlocksIndexHandle(dst []byte, v valueBlocksIndexHandle) int {
+	n := encodeBlockHandle(dst, v.h)
+	dst[n] = v.blockNumByteLength
+	n++
+	dst[n] = v.blockOffsetByteLength
+	n++
+	dst[n] = v.blockLengthByteLength
+	n++
+	return n
+}
+
+func decodeValueBlocksIndexHandle(src []byte) (valueBlocksIndexHandle, int, error) {
+	var vbih valueBlocksIndexHandle
+	var n int
+	vbih.h, n = decodeBlockHandle(src)
+	if n <= 0 {
+		return vbih, 0, errors.Errorf("bad BlockHandle %x", src)
+	}
+	if len(src) != n+3 {
+		return vbih, 0, errors.Errorf("bad BlockHandle %x", src)
+	}
+	vbih.blockNumByteLength = src[n]
+	vbih.blockOffsetByteLength = src[n+1]
+	vbih.blockLengthByteLength = src[n+2]
+	return vbih, n + 3, nil
+}
+
+type valueBlocksAndIndexStats struct {
+	numValueBlocks         uint64
+	numValuesInValueBlocks uint64
+	// Includes both value blocks and value index block.
+	valueBlocksAndIndexSize uint64
+}
+
+// valueBlockWriter writes a sequence of value blocks, and the value blocks
+// index, for a sstable.
+type valueBlockWriter struct {
+	// The configured uncompressed block size and size threshold
+	blockSize, blockSizeThreshold int
+	// Configured compression.
+	compression Compression
+	// checksummer with configured checksum type.
+	checksummer checksummer
+	// Block finished callback.
+	blockFinishedFunc func(compressedSize int)
+
+	// buf is the current block being written to (uncompressed).
+	buf *blockBuffer
+	// compressedBuf is used for compressing the block.
+	compressedBuf *blockBuffer
+	// Sequence of blocks that are finished.
+	blocks []blockAndHandle
+	// Cumulative value block bytes written so far.
+	totalBlockBytes uint64
+	numValues       uint64
+}
+
+type blockAndHandle struct {
+	block      *blockBuffer
+	handle     BlockHandle
+	compressed bool
+}
+
+type blockBuffer struct {
+	b []byte
+}
+
+// Pool of block buffers that should be roughly the blockSize.
+var uncompressedValueBlockBufPool = sync.Pool{
+	New: func() interface{} {
+		return &blockBuffer{}
+	},
+}
+
+// Pool of block buffers for compressed value blocks. These may widely vary in
+// size based on compression ratios.
+var compressedValueBlockBufPool = sync.Pool{
+	New: func() interface{} {
+		return &blockBuffer{}
+	},
+}
+
+func releaseToValueBlockBufPool(pool *sync.Pool, b *blockBuffer) {
+	// Don't pool buffers larger than 128KB, in case we had some rare large
+	// values.
+	if len(b.b) > 128*1024 {
+		return
+	}
+	if invariants.Enabled {
+		// Set the bytes to a random value. Cap the number of bytes being
+		// randomized to prevent test timeouts.
+		length := cap(b.b)
+		if length > 1000 {
+			length = 1000
+		}
+		b.b = b.b[:length:length]
+		rand.Read(b.b)
+	}
+	pool.Put(b)
+}
+
+var valueBlockWriterPool = sync.Pool{
+	New: func() interface{} {
+		return &valueBlockWriter{}
+	},
+}
+
+func newValueBlockWriter(
+	blockSize int,
+	blockSizeThreshold int,
+	compression Compression,
+	checksumType ChecksumType,
+	// compressedSize should exclude the block trailer.
+	blockFinishedFunc func(compressedSize int),
+) *valueBlockWriter {
+	w := valueBlockWriterPool.Get().(*valueBlockWriter)
+	*w = valueBlockWriter{
+		blockSize:          blockSize,
+		blockSizeThreshold: blockSizeThreshold,
+		compression:        compression,
+		checksummer: checksummer{
+			checksumType: checksumType,
+		},
+		blockFinishedFunc: blockFinishedFunc,
+		buf:               uncompressedValueBlockBufPool.Get().(*blockBuffer),
+		compressedBuf:     compressedValueBlockBufPool.Get().(*blockBuffer),
+		blocks:            w.blocks[:0],
+	}
+	w.buf.b = w.buf.b[:0]
+	w.compressedBuf.b = w.compressedBuf.b[:0]
+	return w
+}
+
+func releaseValueBlockWriter(w *valueBlockWriter) {
+	for i := range w.blocks {
+		if w.blocks[i].compressed {
+			releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.blocks[i].block)
+		} else {
+			releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.blocks[i].block)
+		}
+		w.blocks[i].block = nil
+	}
+	if w.buf != nil {
+		releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.buf)
+	}
+	if w.compressedBuf != nil {
+		releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.compressedBuf)
+	}
+	*w = valueBlockWriter{
+		blocks: w.blocks[:0],
+	}
+	valueBlockWriterPool.Put(w)
+}
+
+func (w *valueBlockWriter) addValue(v []byte) (valueHandle, error) {
+	if invariants.Enabled && len(v) == 0 {
+		return valueHandle{}, errors.Errorf("cannot write empty value to value block")
+	}
+	w.numValues++
+	blockLen := len(w.buf.b)
+	valueLen := len(v)
+	if blockLen >= w.blockSize ||
+		(blockLen > w.blockSizeThreshold && blockLen+valueLen > w.blockSize) {
+		// Block is not currently empty and adding this value will become too big,
+		// so finish this block.
+		w.compressAndFlush()
+		blockLen = len(w.buf.b)
+		if invariants.Enabled && blockLen != 0 {
+			panic("blockLen of new block should be 0")
+		}
+	}
+	vh := valueHandle{
+		valueLen:      uint32(valueLen),
+		blockNum:      uint32(len(w.blocks)),
+		offsetInBlock: uint32(blockLen),
+	}
+	blockLen = int(vh.offsetInBlock + vh.valueLen)
+	if cap(w.buf.b) < blockLen {
+		size := 2 * cap(w.buf.b)
+		if size < 1024 {
+			size = 1024
+		}
+		for size < blockLen {
+			size *= 2
+		}
+		buf := make([]byte, blockLen, size)
+		_ = copy(buf, w.buf.b)
+		w.buf.b = buf
+	} else {
+		w.buf.b = w.buf.b[:blockLen]
+	}
+	buf := w.buf.b[vh.offsetInBlock:]
+	n := copy(buf, v)
+	if n != len(buf) {
+		panic("incorrect length computation")
+	}
+	return vh, nil
+}
+
+func (w *valueBlockWriter) compressAndFlush() {
+	// Compress the buffer, discarding the result if the improvement isn't at
+	// least 12.5%.
+	blockType := noCompressionBlockType
+	b := w.buf
+	if w.compression != NoCompression {
+		blockType, w.compressedBuf.b =
+			compressBlock(w.compression, w.buf.b, w.compressedBuf.b[:cap(w.compressedBuf.b)])
+		if len(w.compressedBuf.b) < len(w.buf.b)-len(w.buf.b)/8 {
+			b = w.compressedBuf
+		} else {
+			blockType = noCompressionBlockType
+		}
+	}
+	n := len(b.b)
+	if n+blockTrailerLen > cap(b.b) {
+		block := make([]byte, n+blockTrailerLen)
+		copy(block, b.b)
+		b.b = block
+	} else {
+		b.b = b.b[:n+blockTrailerLen]
+	}
+	b.b[n] = byte(blockType)
+	w.computeChecksum(b.b)
+	bh := BlockHandle{Offset: w.totalBlockBytes, Length: uint64(n)}
+	w.totalBlockBytes += uint64(len(b.b))
+	// blockFinishedFunc length excludes the block trailer.
+	w.blockFinishedFunc(n)
+	compressed := blockType != noCompressionBlockType
+	w.blocks = append(w.blocks, blockAndHandle{
+		block:      b,
+		handle:     bh,
+		compressed: compressed,
+	})
+	// Handed off a buffer to w.blocks, so need get a new one.
+	if compressed {
+		w.compressedBuf = compressedValueBlockBufPool.Get().(*blockBuffer)
+	} else {
+		w.buf = uncompressedValueBlockBufPool.Get().(*blockBuffer)
+	}
+	w.buf.b = w.buf.b[:0]
+}
+
+func (w *valueBlockWriter) computeChecksum(block []byte) {
+	n := len(block) - blockTrailerLen
+	checksum := w.checksummer.checksum(block[:n], block[n:n+1])
+	binary.LittleEndian.PutUint32(block[n+1:], checksum)
+}
+
+func (w *valueBlockWriter) finish(
+	writer io.Writer, fileOffset uint64,
+) (valueBlocksIndexHandle, valueBlocksAndIndexStats, error) {
+	if len(w.buf.b) > 0 {
+		w.compressAndFlush()
+	}
+	n := len(w.blocks)
+	if n == 0 {
+		return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, nil
+	}
+	largestOffset := uint64(0)
+	largestLength := uint64(0)
+	for i := range w.blocks {
+		_, err := writer.Write(w.blocks[i].block.b)
+		if err != nil {
+			return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err
+		}
+		w.blocks[i].handle.Offset += fileOffset
+		largestOffset = w.blocks[i].handle.Offset
+		if largestLength < w.blocks[i].handle.Length {
+			largestLength = w.blocks[i].handle.Length
+		}
+	}
+	vbihOffset := fileOffset + w.totalBlockBytes
+
+	vbih := valueBlocksIndexHandle{
+		h: BlockHandle{
+			Offset: vbihOffset,
+		},
+		blockNumByteLength:    uint8(lenLittleEndian(uint64(n - 1))),
+		blockOffsetByteLength: uint8(lenLittleEndian(largestOffset)),
+		blockLengthByteLength: uint8(lenLittleEndian(largestLength)),
+	}
+	var err error
+	if vbih, err = w.writeValueBlocksIndex(writer, vbih); err != nil {
+		return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err
+	}
+	stats := valueBlocksAndIndexStats{
+		numValueBlocks:          uint64(n),
+		numValuesInValueBlocks:  w.numValues,
+		valueBlocksAndIndexSize: w.totalBlockBytes + vbih.h.Length + blockTrailerLen,
+	}
+	return vbih, stats, err
+}
+
+func (w *valueBlockWriter) writeValueBlocksIndex(
+	writer io.Writer, h valueBlocksIndexHandle,
+) (valueBlocksIndexHandle, error) {
+	blockLen :=
+		int(h.blockNumByteLength+h.blockOffsetByteLength+h.blockLengthByteLength) * len(w.blocks)
+	h.h.Length = uint64(blockLen)
+	blockLen += blockTrailerLen
+	var buf []byte
+	if cap(w.buf.b) < blockLen {
+		buf = make([]byte, blockLen)
+		w.buf.b = buf
+	} else {
+		buf = w.buf.b[:blockLen]
+	}
+	b := buf
+	for i := range w.blocks {
+		littleEndianPut(uint64(i), b, int(h.blockNumByteLength))
+		b = b[int(h.blockNumByteLength):]
+		littleEndianPut(w.blocks[i].handle.Offset, b, int(h.blockOffsetByteLength))
+		b = b[int(h.blockOffsetByteLength):]
+		littleEndianPut(w.blocks[i].handle.Length, b, int(h.blockLengthByteLength))
+		b = b[int(h.blockLengthByteLength):]
+	}
+	if len(b) != blockTrailerLen {
+		panic("incorrect length calculation")
+	}
+	b[0] = byte(noCompressionBlockType)
+	w.computeChecksum(buf)
+	if _, err := writer.Write(buf); err != nil {
+		return valueBlocksIndexHandle{}, err
+	}
+	return h, nil
+}
+
+// littleEndianPut writes v to b using little endian encoding, under the
+// assumption that v can be represented using n bytes.
+func littleEndianPut(v uint64, b []byte, n int) {
+	_ = b[n-1] // bounds check
+	for i := 0; i < n; i++ {
+		b[i] = byte(v)
+		v = v >> 8
+	}
+}
+
+// lenLittleEndian returns the minimum number of bytes needed to encode v
+// using little endian encoding.
+func lenLittleEndian(v uint64) int {
+	n := 0
+	for i := 0; i < 8; i++ {
+		n++
+		v = v >> 8
+		if v == 0 {
+			break
+		}
+	}
+	return n
+}
+
+func littleEndianGet(b []byte, n int) uint64 {
+	_ = b[n-1] // bounds check
+	v := uint64(b[0])
+	for i := 1; i < n; i++ {
+		v |= uint64(b[i]) << (8 * i)
+	}
+	return v
+}
+
+// UserKeyPrefixBound represents a [Lower,Upper) bound of user key prefixes.
+// If both are nil, there is no bound specified. Else, Compare(Lower,Upper)
+// must be < 0.
+type UserKeyPrefixBound struct {
+	// Lower is a lower bound user key prefix.
+	Lower []byte
+	// Upper is an upper bound user key prefix.
+	Upper []byte
+}
+
+// IsEmpty returns true iff the bound is empty.
+func (ukb *UserKeyPrefixBound) IsEmpty() bool {
+	return len(ukb.Lower) == 0 && len(ukb.Upper) == 0
+}
+
+type blockProviderWhenOpen interface {
+	readBlockForVBR(
+		ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats,
+	) (bufferHandle, error)
+}
+
+type blockProviderWhenClosed struct {
+	rp ReaderProvider
+	r  *Reader
+}
+
+func (bpwc *blockProviderWhenClosed) open() error {
+	var err error
+	bpwc.r, err = bpwc.rp.GetReader()
+	return err
+}
+
+func (bpwc *blockProviderWhenClosed) close() {
+	bpwc.rp.Close()
+	bpwc.r = nil
+}
+
+func (bpwc blockProviderWhenClosed) readBlockForVBR(
+	ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats,
+) (bufferHandle, error) {
+	ctx = objiotracing.WithBlockType(ctx, objiotracing.ValueBlock)
+	// TODO(jackson,sumeer): Consider whether to use a buffer pool in this case.
+	// The bpwc is not allowed to outlive the iterator tree, so it cannot
+	// outlive the buffer pool.
+	return bpwc.r.readBlock(ctx, h, nil, nil, stats, nil /* buffer pool */)
+}
+
+// ReaderProvider supports the implementation of blockProviderWhenClosed.
+// GetReader and Close can be called multiple times in pairs.
+type ReaderProvider interface {
+	GetReader() (r *Reader, err error)
+	Close()
+}
+
+// TrivialReaderProvider implements ReaderProvider for a Reader that will
+// outlive the top-level iterator in the iterator tree.
+type TrivialReaderProvider struct {
+	*Reader
+}
+
+var _ ReaderProvider = TrivialReaderProvider{}
+
+// GetReader implements ReaderProvider.
+func (trp TrivialReaderProvider) GetReader() (*Reader, error) {
+	return trp.Reader, nil
+}
+
+// Close implements ReaderProvider.
+func (trp TrivialReaderProvider) Close() {}
+
+// valueBlockReader is used to retrieve values in value
+// blocks. It is used when the sstable was written with
+// Properties.ValueBlocksAreEnabled.
+type valueBlockReader struct {
+	ctx    context.Context
+	bpOpen blockProviderWhenOpen
+	rp     ReaderProvider
+	vbih   valueBlocksIndexHandle
+	stats  *base.InternalIteratorStats
+
+	// The value blocks index is lazily retrieved the first time the reader
+	// needs to read a value that resides in a value block.
+	vbiBlock []byte
+	vbiCache bufferHandle
+	// When sequentially iterating through all key-value pairs, the cost of
+	// repeatedly getting a block that is already in the cache and releasing the
+	// bufferHandle can be ~40% of the cpu overhead. So the reader remembers the
+	// last value block it retrieved, in case there is locality of access, and
+	// this value block can be used for the next value retrieval.
+	valueBlockNum uint32
+	valueBlock    []byte
+	valueBlockPtr unsafe.Pointer
+	valueCache    bufferHandle
+	lazyFetcher   base.LazyFetcher
+	closed        bool
+	bufToMangle   []byte
+}
+
+func (r *valueBlockReader) getLazyValueForPrefixAndValueHandle(handle []byte) base.LazyValue {
+	fetcher := &r.lazyFetcher
+	valLen, h := decodeLenFromValueHandle(handle[1:])
+	*fetcher = base.LazyFetcher{
+		Fetcher: r,
+		Attribute: base.AttributeAndLen{
+			ValueLen:       int32(valLen),
+			ShortAttribute: getShortAttribute(valuePrefix(handle[0])),
+		},
+	}
+	if r.stats != nil {
+		r.stats.SeparatedPointValue.Count++
+		r.stats.SeparatedPointValue.ValueBytes += uint64(valLen)
+	}
+	return base.LazyValue{
+		ValueOrHandle: h,
+		Fetcher:       fetcher,
+	}
+}
+
+func (r *valueBlockReader) close() {
+	r.bpOpen = nil
+	r.vbiBlock = nil
+	r.vbiCache.Release()
+	// Set the handle to empty since Release does not nil the Handle.value. If
+	// we were to reopen this valueBlockReader and retrieve the same
+	// Handle.value from the cache, we don't want to accidentally unref it when
+	// attempting to unref the old handle.
+	r.vbiCache = bufferHandle{}
+	r.valueBlock = nil
+	r.valueBlockPtr = nil
+	r.valueCache.Release()
+	// See comment above.
+	r.valueCache = bufferHandle{}
+	r.closed = true
+	// rp, vbih, stats remain valid, so that LazyFetcher.ValueFetcher can be
+	// implemented.
+}
+
+// Fetch implements base.ValueFetcher.
+func (r *valueBlockReader) Fetch(
+	handle []byte, valLen int32, buf []byte,
+) (val []byte, callerOwned bool, err error) {
+	if !r.closed {
+		val, err := r.getValueInternal(handle, valLen)
+		if invariants.Enabled {
+			val = r.doValueMangling(val)
+		}
+		return val, false, err
+	}
+
+	bp := blockProviderWhenClosed{rp: r.rp}
+	err = bp.open()
+	if err != nil {
+		return nil, false, err
+	}
+	defer bp.close()
+	defer r.close()
+	r.bpOpen = bp
+	var v []byte
+	v, err = r.getValueInternal(handle, valLen)
+	if err != nil {
+		return nil, false, err
+	}
+	buf = append(buf[:0], v...)
+	return buf, true, nil
+}
+
+// doValueMangling attempts to uncover violations of the contract listed in
+// the declaration comment of LazyValue. It is expensive, hence only called
+// when invariants.Enabled.
+func (r *valueBlockReader) doValueMangling(v []byte) []byte {
+	// Randomly set the bytes in the previous retrieved value to 0, since
+	// property P1 only requires the valueBlockReader to maintain the memory of
+	// one fetched value.
+	if rand.Intn(2) == 0 {
+		for i := range r.bufToMangle {
+			r.bufToMangle[i] = 0
+		}
+	}
+	// Store the current value in a new buffer for future mangling.
+	r.bufToMangle = append([]byte(nil), v...)
+	return r.bufToMangle
+}
+
+func (r *valueBlockReader) getValueInternal(handle []byte, valLen int32) (val []byte, err error) {
+	vh := decodeRemainingValueHandle(handle)
+	vh.valueLen = uint32(valLen)
+	if r.vbiBlock == nil {
+		ch, err := r.bpOpen.readBlockForVBR(r.ctx, r.vbih.h, r.stats)
+		if err != nil {
+			return nil, err
+		}
+		r.vbiCache = ch
+		r.vbiBlock = ch.Get()
+	}
+	if r.valueBlock == nil || r.valueBlockNum != vh.blockNum {
+		vbh, err := r.getBlockHandle(vh.blockNum)
+		if err != nil {
+			return nil, err
+		}
+		vbCacheHandle, err := r.bpOpen.readBlockForVBR(r.ctx, vbh, r.stats)
+		if err != nil {
+			return nil, err
+		}
+		r.valueBlockNum = vh.blockNum
+		r.valueCache.Release()
+		r.valueCache = vbCacheHandle
+		r.valueBlock = vbCacheHandle.Get()
+		r.valueBlockPtr = unsafe.Pointer(&r.valueBlock[0])
+	}
+	if r.stats != nil {
+		r.stats.SeparatedPointValue.ValueBytesFetched += uint64(valLen)
+	}
+	return r.valueBlock[vh.offsetInBlock : vh.offsetInBlock+vh.valueLen], nil
+}
+
+func (r *valueBlockReader) getBlockHandle(blockNum uint32) (BlockHandle, error) {
+	indexEntryLen :=
+		int(r.vbih.blockNumByteLength + r.vbih.blockOffsetByteLength + r.vbih.blockLengthByteLength)
+	offsetInIndex := indexEntryLen * int(blockNum)
+	if len(r.vbiBlock) < offsetInIndex+indexEntryLen {
+		return BlockHandle{}, errors.Errorf(
+			"cannot read at offset %d and length %d from block of length %d",
+			offsetInIndex, indexEntryLen, len(r.vbiBlock))
+	}
+	b := r.vbiBlock[offsetInIndex : offsetInIndex+indexEntryLen]
+	n := int(r.vbih.blockNumByteLength)
+	bn := littleEndianGet(b, n)
+	if uint32(bn) != blockNum {
+		return BlockHandle{},
+			errors.Errorf("expected block num %d but found %d", blockNum, bn)
+	}
+	b = b[n:]
+	n = int(r.vbih.blockOffsetByteLength)
+	blockOffset := littleEndianGet(b, n)
+	b = b[n:]
+	n = int(r.vbih.blockLengthByteLength)
+	blockLen := littleEndianGet(b, n)
+	return BlockHandle{Offset: blockOffset, Length: blockLen}, nil
+}

vendor/github.com/cockroachdb/pebble/sstable/write_queue.go

@@ -0,0 +1,140 @@
+package sstable
+
+import (
+	"sync"
+
+	"github.com/cockroachdb/pebble/internal/base"
+)
+
+type writeTask struct {
+	// Since writeTasks are pooled, the compressionDone channel will be re-used.
+	// It is necessary that any writes to the channel have already been read,
+	// before adding the writeTask back to the pool.
+	compressionDone chan bool
+	buf             *dataBlockBuf
+	// If this is not nil, then this index block will be flushed.
+	flushableIndexBlock *indexBlockBuf
+	// currIndexBlock is the index block on which indexBlock.add must be called.
+	currIndexBlock *indexBlockBuf
+	indexEntrySep  InternalKey
+	// inflightIndexEntrySize is used to decrement Writer.indexBlock.sizeEstimate.inflightSize.
+	indexInflightSize int
+	// If the index block is finished, then we set the finishedIndexProps here.
+	finishedIndexProps []byte
+}
+
+// It is not the responsibility of the writeTask to clear the
+// task.flushableIndexBlock, and task.buf.
+func (task *writeTask) clear() {
+	*task = writeTask{
+		indexEntrySep:   base.InvalidInternalKey,
+		compressionDone: task.compressionDone,
+	}
+}
+
+// Note that only the Writer client goroutine will be adding tasks to the writeQueue.
+// Both the Writer client and the compression goroutines will be able to write to
+// writeTask.compressionDone to indicate that the compression job associated with
+// a writeTask has finished.
+type writeQueue struct {
+	tasks  chan *writeTask
+	wg     sync.WaitGroup
+	writer *Writer
+
+	// err represents an error which is encountered when the write queue attempts
+	// to write a block to disk. The error is stored here to skip unnecessary block
+	// writes once the first error is encountered.
+	err    error
+	closed bool
+}
+
+func newWriteQueue(size int, writer *Writer) *writeQueue {
+	w := &writeQueue{}
+	w.tasks = make(chan *writeTask, size)
+	w.writer = writer
+
+	w.wg.Add(1)
+	go w.runWorker()
+	return w
+}
+
+func (w *writeQueue) performWrite(task *writeTask) error {
+	var bh BlockHandle
+	var bhp BlockHandleWithProperties
+
+	var err error
+	if bh, err = w.writer.writeCompressedBlock(task.buf.compressed, task.buf.tmp[:]); err != nil {
+		return err
+	}
+
+	bhp = BlockHandleWithProperties{BlockHandle: bh, Props: task.buf.dataBlockProps}
+	if err = w.writer.addIndexEntry(
+		task.indexEntrySep, bhp, task.buf.tmp[:], task.flushableIndexBlock, task.currIndexBlock,
+		task.indexInflightSize, task.finishedIndexProps); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// It is necessary to ensure that none of the buffers in the writeTask,
+// dataBlockBuf, indexBlockBuf, are pointed to by another struct.
+func (w *writeQueue) releaseBuffers(task *writeTask) {
+	task.buf.clear()
+	dataBlockBufPool.Put(task.buf)
+
+	// This index block is no longer used by the Writer, so we can add it back
+	// to the pool.
+	if task.flushableIndexBlock != nil {
+		task.flushableIndexBlock.clear()
+		indexBlockBufPool.Put(task.flushableIndexBlock)
+	}
+
+	task.clear()
+	writeTaskPool.Put(task)
+}
+
+func (w *writeQueue) runWorker() {
+	for task := range w.tasks {
+		<-task.compressionDone
+
+		if w.err == nil {
+			w.err = w.performWrite(task)
+		}
+
+		w.releaseBuffers(task)
+	}
+	w.wg.Done()
+}
+
+func (w *writeQueue) add(task *writeTask) {
+	w.tasks <- task
+}
+
+// addSync will perform the writeTask synchronously with the caller goroutine. Calls to addSync
+// are no longer valid once writeQueue.add has been called at least once.
+func (w *writeQueue) addSync(task *writeTask) error {
+	// This should instantly return without blocking.
+	<-task.compressionDone
+
+	if w.err == nil {
+		w.err = w.performWrite(task)
+	}
+
+	w.releaseBuffers(task)
+
+	return w.err
+}
+
+// finish should only be called once no more tasks will be added to the writeQueue.
+// finish will return any error which was encountered while tasks were processed.
+func (w *writeQueue) finish() error {
+	if w.closed {
+		return w.err
+	}
+
+	close(w.tasks)
+	w.wg.Wait()
+	w.closed = true
+	return w.err
+}